maestro/src/engine/tools/web.test.ts

import * as fs from 'fs';
import * as path from 'path';
import { tmpdir } from 'os';
import { afterEach, describe, expect, it, vi } from 'vitest';
import { executeTool, sanitizeQuery, parseSearchResultsFromText, appendWebSearchHistory, searchViaSearxng, clearPersistentContexts } from './web.js';
import type { ToolContext } from './core.js';

function makeWorkspace(): string {
  return fs.mkdtempSync(path.join(tmpdir(), 'maestro-web-'));
}

function makeContext(workspacePath: string): ToolContext {
  return {
    workspacePath,
    editAllowed: false,
    toolsConfig: {
      webfetchAllowedHosts: ['example.com'],
    },
  };
}

describe('sanitizeQuery', () => {
  it('removes private IPv4 addresses', () => {
    expect(sanitizeQuery('deploy to 10.0.0.10 nginx', {})).toBe('deploy to nginx');
  });
  it('removes 10.x.x.x addresses', () => {
    expect(sanitizeQuery('access 10.0.0.1 server', {})).toBe('access server');
  });
  it('removes 172.16-31.x.x addresses', () => {
    expect(sanitizeQuery('host 172.16.0.5 config', {})).toBe('host config');
  });
  it('removes email addresses', () => {
    expect(sanitizeQuery('contact [email protected] for info', {})).toBe('contact for info');
  });
  it('removes Japanese phone numbers', () => {
    expect(sanitizeQuery('call 090-1234-5678 now', {})).toBe('call now');
    expect(sanitizeQuery('tel 03-1234-5678 office', {})).toBe('tel office');
  });
  it('removes internal domains (.local, .internal, .lan)', () => {
    expect(sanitizeQuery('check server.local status', {})).toBe('check status');
  });
  it('applies custom blocked patterns', () => {
    const config = { blockedPatterns: ['secret-project'] };
    expect(sanitizeQuery('details about secret-project release', config)).toBe('details about release');
  });
  it('returns null when query becomes empty', () => {
    expect(sanitizeQuery('10.0.0.10', {})).toBeNull();
  });
  it('respects autoBlock toggles', () => {
    const config = { autoBlock: { privateIp: false } };
    expect(sanitizeQuery('host 10.0.0.10 info', config)).toBe('host 10.0.0.10 info');
  });
  it('preserves public IPs', () => {
    expect(sanitizeQuery('query 8.8.8.8 dns', {})).toBe('query 8.8.8.8 dns');
  });
});

describe('web tools', () => {
  let workspacePath = '';

  afterEach(() => {
    if (workspacePath) {
      fs.rmSync(workspacePath, { recursive: true, force: true });
      workspacePath = '';
    }
    vi.unstubAllGlobals();
    vi.restoreAllMocks();
  });

  it('blocks WebFetch on PDF responses and suggests ReadPdf', async () => {
    workspacePath = makeWorkspace();
    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('%PDF-1.4', {
      status: 200,
      headers: { 'content-type': 'application/pdf' },
    })));

    const result = await executeTool('WebFetch', { url: 'https://example.com/manual.pdf' }, makeContext(workspacePath));

    expect(result).not.toBeNull();
    expect(result?.isError).toBe(true);
    expect(result?.output).toContain('ReadPdf');
    const history = fs.readFileSync(path.join(workspacePath, 'logs', 'webfetch-history.jsonl'), 'utf-8').trim().split('\n').map((line) => JSON.parse(line) as Record<string, unknown>);
    expect(history).toHaveLength(1);
    expect(history[0]?.url).toBe('https://example.com/manual.pdf');
    expect(history[0]?.outcome).toBe('pdf_blocked');
  });

  it('converts HTML responses to text', async () => {
    workspacePath = makeWorkspace();
    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('<html><body><h1>Hello</h1><p>World</p></body></html>', {
      status: 200,
      headers: { 'content-type': 'text/html; charset=utf-8' },
    })));

    const result = await executeTool('WebFetch', { url: 'https://example.com/page' }, makeContext(workspacePath));

    expect(result).not.toBeNull();
    expect(result?.isError).toBe(false);
    expect(result?.output).toContain('Hello');
    expect(result?.output).toContain('World');
    const history = fs.readFileSync(path.join(workspacePath, 'logs', 'webfetch-history.jsonl'), 'utf-8').trim().split('\n').map((line) => JSON.parse(line) as Record<string, unknown>);
    expect(history).toHaveLength(1);
    expect(history[0]?.url).toBe('https://example.com/page');
    expect(history[0]?.outcome).toBe('success');
    expect(history[0]?.contentType).toBe('text/html; charset=utf-8');
  });

  it('does not attach screenshot when vlmEnabled is false', async () => {
    workspacePath = makeWorkspace();
    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('<html><body><h1>Hi</h1></body></html>', {
      status: 200,
      headers: { 'content-type': 'text/html' },
    })));

    const ctx: ToolContext = {
      workspacePath,
      editAllowed: false,
      vlmEnabled: false,
      toolsConfig: { webfetchAllowedHosts: ['example.com'] },
    };
    const result = await executeTool('WebFetch', { url: 'https://example.com/' }, ctx);

    expect(result?.isError).toBe(false);
    expect(result?.images).toBeUndefined();
    expect(fs.existsSync(path.join(workspacePath, 'logs', 'webfetch-screenshots'))).toBe(false);
  });

  it('respects explicit opt-out via webfetchScreenshot=false even with vlmEnabled', async () => {
    workspacePath = makeWorkspace();
    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('<html><body>x</body></html>', {
      status: 200,
      headers: { 'content-type': 'text/html' },
    })));

    const ctx: ToolContext = {
      workspacePath,
      editAllowed: false,
      vlmEnabled: true,
      toolsConfig: {
        webfetchAllowedHosts: ['example.com'],
        webfetchScreenshot: false,
      },
    };
    const result = await executeTool('WebFetch', { url: 'https://example.com/' }, ctx);

    expect(result?.isError).toBe(false);
    expect(result?.images).toBeUndefined();
    expect(fs.existsSync(path.join(workspacePath, 'logs', 'webfetch-screenshots'))).toBe(false);
  });

  it('records invalid URL attempts before fetch', async () => {
    workspacePath = makeWorkspace();

    const result = await executeTool('WebFetch', { url: 'not a url' }, makeContext(workspacePath));

    expect(result).not.toBeNull();
    expect(result?.isError).toBe(true);
    const history = fs.readFileSync(path.join(workspacePath, 'logs', 'webfetch-history.jsonl'), 'utf-8').trim().split('\n').map((line) => JSON.parse(line) as Record<string, unknown>);
    expect(history).toHaveLength(1);
    expect(history[0]?.url).toBe('not a url');
    expect(history[0]?.outcome).toBe('invalid_url');
  });
});

describe('source library routing', () => {
  let workspacePath = '';

  afterEach(() => {
    if (workspacePath) {
      fs.rmSync(workspacePath, { recursive: true, force: true });
      workspacePath = '';
    }
    vi.unstubAllGlobals();
    vi.restoreAllMocks();
  });

  it('DownloadFile section=source saves under source/ and appends index.jsonl', async () => {
    workspacePath = makeWorkspace();
    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('PNGDATA', {
      status: 200,
      headers: { 'content-type': 'image/png' },
    })));

    const result = await executeTool(
      'DownloadFile',
      { url: 'https://example.com/chart.png', filename: 'chart.png', section: 'source' },
      makeContext(workspacePath),
    );

    expect(result?.isError).toBe(false);
    expect(fs.existsSync(path.join(workspacePath, 'source', 'chart.png'))).toBe(true);
    const idx = fs.readFileSync(path.join(workspacePath, 'source', 'index.jsonl'), 'utf-8')
      .trim().split('\n').map((l) => JSON.parse(l) as Record<string, unknown>);
    expect(idx).toHaveLength(1);
    expect(idx[0]?.url).toBe('https://example.com/chart.png');
    expect(idx[0]?.tool).toBe('DownloadFile');
    expect(idx[0]?.file).toBe('chart.png');
    expect(idx[0]?.bytes).toBe(Buffer.byteLength('PNGDATA'));
  });

  it('DownloadFile section=input does NOT write source/index.jsonl', async () => {
    workspacePath = makeWorkspace();
    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('DATA', {
      status: 200,
      headers: { 'content-type': 'application/octet-stream' },
    })));

    const result = await executeTool(
      'DownloadFile',
      { url: 'https://example.com/file.bin', filename: 'file.bin' },
      makeContext(workspacePath),
    );

    expect(result?.isError).toBe(false);
    expect(fs.existsSync(path.join(workspacePath, 'input', 'file.bin'))).toBe(true);
    expect(fs.existsSync(path.join(workspacePath, 'source', 'index.jsonl'))).toBe(false);
  });

  it('WebFetch persists source .md + index entry when ctx.spaceId is set', async () => {
    workspacePath = makeWorkspace();
    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response(
      '<html><body><h1>Title</h1><p>Body content here</p></body></html>',
      { status: 200, headers: { 'content-type': 'text/html; charset=utf-8' } },
    )));

    const ctx: ToolContext = {
      workspacePath,
      editAllowed: false,
      spaceId: 'space-123',
      toolsConfig: { webfetchAllowedHosts: ['example.com'] },
    };
    const result = await executeTool('WebFetch', { url: 'https://example.com/article' }, ctx);

    expect(result?.isError).toBe(false);
    const idx = fs.readFileSync(path.join(workspacePath, 'source', 'index.jsonl'), 'utf-8')
      .trim().split('\n').map((l) => JSON.parse(l) as Record<string, unknown>);
    expect(idx).toHaveLength(1);
    expect(idx[0]?.url).toBe('https://example.com/article');
    expect(idx[0]?.tool).toBe('WebFetch');
    const savedFile = idx[0]?.file as string;
    expect(savedFile.endsWith('.md')).toBe(true);
    const body = fs.readFileSync(path.join(workspacePath, 'source', savedFile), 'utf-8');
    expect(body).toContain('Body content here');
  });

  it('WebFetch dedupes the same URL within a space (no second source file)', async () => {
    workspacePath = makeWorkspace();
    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response(
      '<html><body><p>Stuff</p></body></html>',
      { status: 200, headers: { 'content-type': 'text/html' } },
    )));

    const ctx: ToolContext = {
      workspacePath,
      editAllowed: false,
      spaceId: 'space-1',
      toolsConfig: { webfetchAllowedHosts: ['example.com'] },
    };
    await executeTool('WebFetch', { url: 'https://example.com/dup' }, ctx);
    await executeTool('WebFetch', { url: 'https://example.com/dup' }, ctx);

    const idx = fs.readFileSync(path.join(workspacePath, 'source', 'index.jsonl'), 'utf-8')
      .trim().split('\n').filter(Boolean);
    expect(idx).toHaveLength(1);
  });

  it('WebFetch does NOT create source/ when ctx.spaceId is undefined (legacy)', async () => {
    workspacePath = makeWorkspace();
    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response(
      '<html><body><p>Non-space content</p></body></html>',
      { status: 200, headers: { 'content-type': 'text/html' } },
    )));

    const result = await executeTool('WebFetch', { url: 'https://example.com/page' }, makeContext(workspacePath));

    expect(result?.isError).toBe(false);
    expect(result?.output).toContain('Non-space content');
    expect(fs.existsSync(path.join(workspacePath, 'source'))).toBe(false);
  });

  it('WebFetch saves FULL clean markdown to source/ but returns TRUNCATED text to the agent', async () => {
    workspacePath = makeWorkspace();
    // 本文を 10k 文字を大きく超える長さにして、source(全文) と agent(切り詰め) の差を作る。
    const longBody = 'word '.repeat(6000); // ~30k chars
    const html = `<html><head><title>Long Article</title></head><body>
      <nav>NAV MENU LINKS</nav>
      <article><h1>Long Article Heading</h1><p>${longBody}</p></article>
      <footer>FOOTER JUNK</footer>
    </body></html>`;
    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response(
      html, { status: 200, headers: { 'content-type': 'text/html; charset=utf-8' } },
    )));

    const ctx: ToolContext = {
      workspacePath,
      editAllowed: false,
      spaceId: 'space-long',
      toolsConfig: { webfetchAllowedHosts: ['example.com'] },
    };
    const result = await executeTool('WebFetch', { url: 'https://example.com/long' }, ctx);
    expect(result?.isError).toBe(false);

    // agent への返却は切り詰め + source 案内ノート
    expect(result!.output.length).toBeLessThan(11000);
    expect(result!.output).toContain('full version saved to source/');
    // クリーン抽出されているので boilerplate は出ない
    expect(result!.output).not.toContain('NAV MENU LINKS');
    expect(result!.output).not.toContain('FOOTER JUNK');

    // source/ には全文（agent 返却より長い）が保存される
    const idx = fs.readFileSync(path.join(workspacePath, 'source', 'index.jsonl'), 'utf-8')
      .trim().split('\n').map((l) => JSON.parse(l) as Record<string, unknown>);
    expect(idx).toHaveLength(1);
    const savedFile = idx[0]?.file as string;
    const saved = fs.readFileSync(path.join(workspacePath, 'source', savedFile), 'utf-8');
    expect(saved.length).toBeGreaterThan(20000);
    expect(saved.length).toBeGreaterThan(result!.output.length);
    expect(saved).not.toContain('NAV MENU LINKS');
  });

  it('WebFetch (non-space) returns clean truncated text without a source note', async () => {
    workspacePath = makeWorkspace();
    const longBody = 'lorem '.repeat(6000);
    const html = `<html><head><title>NS</title></head><body>
      <article><h1>Heading</h1><p>${longBody}</p></article></body></html>`;
    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response(
      html, { status: 200, headers: { 'content-type': 'text/html' } },
    )));

    const result = await executeTool('WebFetch', { url: 'https://example.com/ns' }, makeContext(workspacePath));
    expect(result?.isError).toBe(false);
    expect(result!.output.length).toBeLessThan(11000);
    expect(result!.output).toContain('... (truncated)');
    expect(result!.output).not.toContain('saved to source');
    expect(fs.existsSync(path.join(workspacePath, 'source'))).toBe(false);
  });

  it('WebFetch falls back to naive extraction on non-article HTML (never hard-fails)', async () => {
    workspacePath = makeWorkspace();
    // readability が本文を見つけられない最小 HTML
    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response(
      '<html><body>bare text with no article structure</body></html>',
      { status: 200, headers: { 'content-type': 'text/html' } },
    )));

    const result = await executeTool('WebFetch', { url: 'https://example.com/bare' }, makeContext(workspacePath));
    expect(result?.isError).toBe(false);
    expect(result!.output).toContain('bare text');
  });
});

describe('parseSearchResultsFromText', () => {
  it('extracts URLs with surrounding text', () => {
    const text = [
      'Example Title',
      'https://example.com/page1',
      'This is a snippet about the page.',
      '',
      'Another Result',
      'https://another.example.com/page2',
      'Another snippet here.',
    ].join('\n');
    const results = parseSearchResultsFromText(text, 5);
    expect(results).toHaveLength(2);
    expect(results[0]).toEqual({
      title: 'Example Title',
      url: 'https://example.com/page1',
      snippet: 'This is a snippet about the page.',
    });
    expect(results[1]).toEqual({
      title: 'Another Result',
      url: 'https://another.example.com/page2',
      snippet: 'Another snippet here.',
    });
  });

  it('respects limit parameter', () => {
    const text = [
      'Title1', 'https://a.com', 'Snippet1',
      'Title2', 'https://b.com', 'Snippet2',
      'Title3', 'https://c.com', 'Snippet3',
    ].join('\n');
    const results = parseSearchResultsFromText(text, 2);
    expect(results).toHaveLength(2);
  });

  it('skips Google internal URLs', () => {
    const text = [
      'Internal', 'https://www.google.com/search?q=test', 'Skip this',
      'Real', 'https://real.example.com', 'Keep this',
    ].join('\n');
    const results = parseSearchResultsFromText(text, 5);
    expect(results).toHaveLength(1);
    expect(results[0]?.url).toBe('https://real.example.com');
  });

  it('returns empty array for text with no URLs', () => {
    const results = parseSearchResultsFromText('No urls here at all', 5);
    expect(results).toHaveLength(0);
  });
});

describe('appendWebSearchHistory', () => {
  let workspacePath = '';

  afterEach(() => {
    if (workspacePath) {
      fs.rmSync(workspacePath, { recursive: true, force: true });
      workspacePath = '';
    }
  });

  it('creates logs directory and writes JSONL record', () => {
    workspacePath = makeWorkspace();
    const ctx = makeContext(workspacePath);
    appendWebSearchHistory(ctx, {
      timestamp: '2026-03-23T00:00:00Z',
      query: 'test', limit: 5, method: 'browser',
      resultCount: 3, outcome: 'success', fallback: false,
    });
    const content = fs.readFileSync(
      path.join(workspacePath, 'logs', 'websearch-history.jsonl'), 'utf-8'
    );
    const record = JSON.parse(content.trim());
    expect(record.query).toBe('test');
    expect(record.method).toBe('browser');
    expect(record.resultCount).toBe(3);
  });

  it('appends multiple records', () => {
    workspacePath = makeWorkspace();
    const ctx = makeContext(workspacePath);
    appendWebSearchHistory(ctx, {
      timestamp: '2026-03-23T00:00:00Z',
      query: 'q1', limit: 5, method: 'browser',
      resultCount: 1, outcome: 'success', fallback: false,
    });
    appendWebSearchHistory(ctx, {
      timestamp: '2026-03-23T00:01:00Z',
      query: 'q2', limit: 5, method: 'searxng',
      resultCount: 2, outcome: 'success', fallback: true,
    });
    const lines = fs.readFileSync(
      path.join(workspacePath, 'logs', 'websearch-history.jsonl'), 'utf-8'
    ).trim().split('\n');
    expect(lines).toHaveLength(2);
  });
});

describe('searchViaSearxng', () => {
  afterEach(() => {
    vi.unstubAllGlobals();
    vi.restoreAllMocks();
  });

  it('returns parsed results from SearXNG API', async () => {
    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response(JSON.stringify({
      results: [
        { title: 'Result 1', url: 'https://example.com/1', content: 'Snippet 1' },
        { title: 'Result 2', url: 'https://example.com/2', content: 'Snippet 2' },
      ],
    }), { status: 200, headers: { 'content-type': 'application/json' } })));

    const ctx = makeContext(makeWorkspace());
    const results = await searchViaSearxng('test query', 5, ctx);
    expect(results).toHaveLength(2);
    expect(results[0]?.title).toBe('Result 1');
    expect(results[0]?.url).toBe('https://example.com/1');
  });

  it('throws on HTTP error', async () => {
    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('', { status: 500 })));

    const ctx = makeContext(makeWorkspace());
    await expect(searchViaSearxng('test', 5, ctx)).rejects.toThrow('HTTP 500');
  });

  it('respects limit parameter', async () => {
    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response(JSON.stringify({
      results: [
        { title: 'R1', url: 'https://a.com', content: 'S1' },
        { title: 'R2', url: 'https://b.com', content: 'S2' },
        { title: 'R3', url: 'https://c.com', content: 'S3' },
      ],
    }), { status: 200, headers: { 'content-type': 'application/json' } })));

    const ctx = makeContext(makeWorkspace());
    const results = await searchViaSearxng('test', 2, ctx);
    expect(results).toHaveLength(2);
  });
});

describe('WebSearch fallback history', () => {
  let workspacePath = '';

  afterEach(() => {
    if (workspacePath) {
      fs.rmSync(workspacePath, { recursive: true, force: true });
      workspacePath = '';
    }
  });

  it('records fallback history with correct fields', () => {
    workspacePath = makeWorkspace();
    const ctx = makeContext(workspacePath);
    appendWebSearchHistory(ctx, {
      timestamp: '2026-03-23T01:00:00Z',
      query: 'fallback test', limit: 5, method: 'searxng',
      resultCount: 1, outcome: 'success', fallback: true,
    });
    const content = fs.readFileSync(
      path.join(workspacePath, 'logs', 'websearch-history.jsonl'), 'utf-8'
    );
    const record = JSON.parse(content.trim()) as Record<string, unknown>;
    expect(record.method).toBe('searxng');
    expect(record.fallback).toBe(true);
    expect(record.outcome).toBe('success');
    expect(record.query).toBe('fallback test');
  });

  it('records captcha outcome', () => {
    workspacePath = makeWorkspace();
    const ctx = makeContext(workspacePath);
    appendWebSearchHistory(ctx, {
      timestamp: '2026-03-23T01:00:00Z',
      query: 'captcha test', limit: 5, method: 'browser',
      resultCount: 0, outcome: 'captcha', fallback: false,
    });
    const content = fs.readFileSync(
      path.join(workspacePath, 'logs', 'websearch-history.jsonl'), 'utf-8'
    );
    const record = JSON.parse(content.trim()) as Record<string, unknown>;
    expect(record.outcome).toBe('captcha');
    expect(record.method).toBe('browser');
    expect(record.resultCount).toBe(0);
  });

  it('records error outcome with error message', () => {
    workspacePath = makeWorkspace();
    const ctx = makeContext(workspacePath);
    appendWebSearchHistory(ctx, {
      timestamp: '2026-03-23T01:00:00Z',
      query: 'error test', limit: 5, method: 'browser',
      resultCount: 0, outcome: 'error', fallback: false,
      error: 'chromium not found',
    });
    const content = fs.readFileSync(
      path.join(workspacePath, 'logs', 'websearch-history.jsonl'), 'utf-8'
    );
    const record = JSON.parse(content.trim()) as Record<string, unknown>;
    expect(record.outcome).toBe('error');
    expect(record.error).toBe('chromium not found');
  });
});

describe('persistent context management', () => {
  afterEach(() => {
    clearPersistentContexts();
  });

  it('clearPersistentContexts resets internal state without error', () => {
    expect(() => clearPersistentContexts()).not.toThrow();
  });

  it('clearPersistentContexts is idempotent', () => {
    clearPersistentContexts();
    clearPersistentContexts();
  });
});