feat: initial public release (MAESTRO)

2026-06-03 05:08:00 +00:00
commit f5c7666f6b
823 changed files with 184150 additions and 0 deletions
@@ -0,0 +1,382 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { tmpdir } from 'os';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { executeTool, sanitizeQuery, parseSearchResultsFromText, appendWebSearchHistory, searchViaSearxng, clearPersistentContexts } from './web.js';
+import type { ToolContext } from './core.js';
+
+function makeWorkspace(): string {
+  return fs.mkdtempSync(path.join(tmpdir(), 'maestro-web-'));
+}
+
+function makeContext(workspacePath: string): ToolContext {
+  return {
+    workspacePath,
+    editAllowed: false,
+    toolsConfig: {
+      webfetchAllowedHosts: ['example.com'],
+    },
+  };
+}
+
+describe('sanitizeQuery', () => {
+  it('removes private IPv4 addresses', () => {
+    expect(sanitizeQuery('deploy to 10.0.0.10 nginx', {})).toBe('deploy to nginx');
+  });
+  it('removes 10.x.x.x addresses', () => {
+    expect(sanitizeQuery('access 10.0.0.1 server', {})).toBe('access server');
+  });
+  it('removes 172.16-31.x.x addresses', () => {
+    expect(sanitizeQuery('host 172.16.0.5 config', {})).toBe('host config');
+  });
+  it('removes email addresses', () => {
+    expect(sanitizeQuery('contact [email protected] for info', {})).toBe('contact for info');
+  });
+  it('removes Japanese phone numbers', () => {
+    expect(sanitizeQuery('call 090-1234-5678 now', {})).toBe('call now');
+    expect(sanitizeQuery('tel 03-1234-5678 office', {})).toBe('tel office');
+  });
+  it('removes internal domains (.local, .internal, .lan)', () => {
+    expect(sanitizeQuery('check server.local status', {})).toBe('check status');
+  });
+  it('applies custom blocked patterns', () => {
+    const config = { blockedPatterns: ['secret-project'] };
+    expect(sanitizeQuery('details about secret-project release', config)).toBe('details about release');
+  });
+  it('returns null when query becomes empty', () => {
+    expect(sanitizeQuery('10.0.0.10', {})).toBeNull();
+  });
+  it('respects autoBlock toggles', () => {
+    const config = { autoBlock: { privateIp: false } };
+    expect(sanitizeQuery('host 10.0.0.10 info', config)).toBe('host 10.0.0.10 info');
+  });
+  it('preserves public IPs', () => {
+    expect(sanitizeQuery('query 8.8.8.8 dns', {})).toBe('query 8.8.8.8 dns');
+  });
+});
+
+describe('web tools', () => {
+  let workspacePath = '';
+
+  afterEach(() => {
+    if (workspacePath) {
+      fs.rmSync(workspacePath, { recursive: true, force: true });
+      workspacePath = '';
+    }
+    vi.unstubAllGlobals();
+    vi.restoreAllMocks();
+  });
+
+  it('blocks WebFetch on PDF responses and suggests ReadPdf', async () => {
+    workspacePath = makeWorkspace();
+    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('%PDF-1.4', {
+      status: 200,
+      headers: { 'content-type': 'application/pdf' },
+    })));
+
+    const result = await executeTool('WebFetch', { url: 'https://example.com/manual.pdf' }, makeContext(workspacePath));
+
+    expect(result).not.toBeNull();
+    expect(result?.isError).toBe(true);
+    expect(result?.output).toContain('ReadPdf');
+    const history = fs.readFileSync(path.join(workspacePath, 'logs', 'webfetch-history.jsonl'), 'utf-8').trim().split('\n').map((line) => JSON.parse(line) as Record<string, unknown>);
+    expect(history).toHaveLength(1);
+    expect(history[0]?.url).toBe('https://example.com/manual.pdf');
+    expect(history[0]?.outcome).toBe('pdf_blocked');
+  });
+
+  it('converts HTML responses to text', async () => {
+    workspacePath = makeWorkspace();
+    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('<html><body><h1>Hello</h1><p>World</p></body></html>', {
+      status: 200,
+      headers: { 'content-type': 'text/html; charset=utf-8' },
+    })));
+
+    const result = await executeTool('WebFetch', { url: 'https://example.com/page' }, makeContext(workspacePath));
+
+    expect(result).not.toBeNull();
+    expect(result?.isError).toBe(false);
+    expect(result?.output).toContain('Hello');
+    expect(result?.output).toContain('World');
+    const history = fs.readFileSync(path.join(workspacePath, 'logs', 'webfetch-history.jsonl'), 'utf-8').trim().split('\n').map((line) => JSON.parse(line) as Record<string, unknown>);
+    expect(history).toHaveLength(1);
+    expect(history[0]?.url).toBe('https://example.com/page');
+    expect(history[0]?.outcome).toBe('success');
+    expect(history[0]?.contentType).toBe('text/html; charset=utf-8');
+  });
+
+  it('does not attach screenshot when vlmEnabled is false', async () => {
+    workspacePath = makeWorkspace();
+    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('<html><body><h1>Hi</h1></body></html>', {
+      status: 200,
+      headers: { 'content-type': 'text/html' },
+    })));
+
+    const ctx: ToolContext = {
+      workspacePath,
+      editAllowed: false,
+      vlmEnabled: false,
+      toolsConfig: { webfetchAllowedHosts: ['example.com'] },
+    };
+    const result = await executeTool('WebFetch', { url: 'https://example.com/' }, ctx);
+
+    expect(result?.isError).toBe(false);
+    expect(result?.images).toBeUndefined();
+    expect(fs.existsSync(path.join(workspacePath, 'logs', 'webfetch-screenshots'))).toBe(false);
+  });
+
+  it('respects explicit opt-out via webfetchScreenshot=false even with vlmEnabled', async () => {
+    workspacePath = makeWorkspace();
+    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('<html><body>x</body></html>', {
+      status: 200,
+      headers: { 'content-type': 'text/html' },
+    })));
+
+    const ctx: ToolContext = {
+      workspacePath,
+      editAllowed: false,
+      vlmEnabled: true,
+      toolsConfig: {
+        webfetchAllowedHosts: ['example.com'],
+        webfetchScreenshot: false,
+      },
+    };
+    const result = await executeTool('WebFetch', { url: 'https://example.com/' }, ctx);
+
+    expect(result?.isError).toBe(false);
+    expect(result?.images).toBeUndefined();
+    expect(fs.existsSync(path.join(workspacePath, 'logs', 'webfetch-screenshots'))).toBe(false);
+  });
+
+  it('records invalid URL attempts before fetch', async () => {
+    workspacePath = makeWorkspace();
+
+    const result = await executeTool('WebFetch', { url: 'not a url' }, makeContext(workspacePath));
+
+    expect(result).not.toBeNull();
+    expect(result?.isError).toBe(true);
+    const history = fs.readFileSync(path.join(workspacePath, 'logs', 'webfetch-history.jsonl'), 'utf-8').trim().split('\n').map((line) => JSON.parse(line) as Record<string, unknown>);
+    expect(history).toHaveLength(1);
+    expect(history[0]?.url).toBe('not a url');
+    expect(history[0]?.outcome).toBe('invalid_url');
+  });
+});
+
+describe('parseSearchResultsFromText', () => {
+  it('extracts URLs with surrounding text', () => {
+    const text = [
+      'Example Title',
+      'https://example.com/page1',
+      'This is a snippet about the page.',
+      '',
+      'Another Result',
+      'https://another.example.com/page2',
+      'Another snippet here.',
+    ].join('\n');
+    const results = parseSearchResultsFromText(text, 5);
+    expect(results).toHaveLength(2);
+    expect(results[0]).toEqual({
+      title: 'Example Title',
+      url: 'https://example.com/page1',
+      snippet: 'This is a snippet about the page.',
+    });
+    expect(results[1]).toEqual({
+      title: 'Another Result',
+      url: 'https://another.example.com/page2',
+      snippet: 'Another snippet here.',
+    });
+  });
+
+  it('respects limit parameter', () => {
+    const text = [
+      'Title1', 'https://a.com', 'Snippet1',
+      'Title2', 'https://b.com', 'Snippet2',
+      'Title3', 'https://c.com', 'Snippet3',
+    ].join('\n');
+    const results = parseSearchResultsFromText(text, 2);
+    expect(results).toHaveLength(2);
+  });
+
+  it('skips Google internal URLs', () => {
+    const text = [
+      'Internal', 'https://www.google.com/search?q=test', 'Skip this',
+      'Real', 'https://real.example.com', 'Keep this',
+    ].join('\n');
+    const results = parseSearchResultsFromText(text, 5);
+    expect(results).toHaveLength(1);
+    expect(results[0]?.url).toBe('https://real.example.com');
+  });
+
+  it('returns empty array for text with no URLs', () => {
+    const results = parseSearchResultsFromText('No urls here at all', 5);
+    expect(results).toHaveLength(0);
+  });
+});
+
+describe('appendWebSearchHistory', () => {
+  let workspacePath = '';
+
+  afterEach(() => {
+    if (workspacePath) {
+      fs.rmSync(workspacePath, { recursive: true, force: true });
+      workspacePath = '';
+    }
+  });
+
+  it('creates logs directory and writes JSONL record', () => {
+    workspacePath = makeWorkspace();
+    const ctx = makeContext(workspacePath);
+    appendWebSearchHistory(ctx, {
+      timestamp: '2026-03-23T00:00:00Z',
+      query: 'test', limit: 5, method: 'browser',
+      resultCount: 3, outcome: 'success', fallback: false,
+    });
+    const content = fs.readFileSync(
+      path.join(workspacePath, 'logs', 'websearch-history.jsonl'), 'utf-8'
+    );
+    const record = JSON.parse(content.trim());
+    expect(record.query).toBe('test');
+    expect(record.method).toBe('browser');
+    expect(record.resultCount).toBe(3);
+  });
+
+  it('appends multiple records', () => {
+    workspacePath = makeWorkspace();
+    const ctx = makeContext(workspacePath);
+    appendWebSearchHistory(ctx, {
+      timestamp: '2026-03-23T00:00:00Z',
+      query: 'q1', limit: 5, method: 'browser',
+      resultCount: 1, outcome: 'success', fallback: false,
+    });
+    appendWebSearchHistory(ctx, {
+      timestamp: '2026-03-23T00:01:00Z',
+      query: 'q2', limit: 5, method: 'searxng',
+      resultCount: 2, outcome: 'success', fallback: true,
+    });
+    const lines = fs.readFileSync(
+      path.join(workspacePath, 'logs', 'websearch-history.jsonl'), 'utf-8'
+    ).trim().split('\n');
+    expect(lines).toHaveLength(2);
+  });
+});
+
+describe('searchViaSearxng', () => {
+  afterEach(() => {
+    vi.unstubAllGlobals();
+    vi.restoreAllMocks();
+  });
+
+  it('returns parsed results from SearXNG API', async () => {
+    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response(JSON.stringify({
+      results: [
+        { title: 'Result 1', url: 'https://example.com/1', content: 'Snippet 1' },
+        { title: 'Result 2', url: 'https://example.com/2', content: 'Snippet 2' },
+      ],
+    }), { status: 200, headers: { 'content-type': 'application/json' } })));
+
+    const ctx = makeContext(makeWorkspace());
+    const results = await searchViaSearxng('test query', 5, ctx);
+    expect(results).toHaveLength(2);
+    expect(results[0]?.title).toBe('Result 1');
+    expect(results[0]?.url).toBe('https://example.com/1');
+  });
+
+  it('throws on HTTP error', async () => {
+    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('', { status: 500 })));
+
+    const ctx = makeContext(makeWorkspace());
+    await expect(searchViaSearxng('test', 5, ctx)).rejects.toThrow('HTTP 500');
+  });
+
+  it('respects limit parameter', async () => {
+    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response(JSON.stringify({
+      results: [
+        { title: 'R1', url: 'https://a.com', content: 'S1' },
+        { title: 'R2', url: 'https://b.com', content: 'S2' },
+        { title: 'R3', url: 'https://c.com', content: 'S3' },
+      ],
+    }), { status: 200, headers: { 'content-type': 'application/json' } })));
+
+    const ctx = makeContext(makeWorkspace());
+    const results = await searchViaSearxng('test', 2, ctx);
+    expect(results).toHaveLength(2);
+  });
+});
+
+describe('WebSearch fallback history', () => {
+  let workspacePath = '';
+
+  afterEach(() => {
+    if (workspacePath) {
+      fs.rmSync(workspacePath, { recursive: true, force: true });
+      workspacePath = '';
+    }
+  });
+
+  it('records fallback history with correct fields', () => {
+    workspacePath = makeWorkspace();
+    const ctx = makeContext(workspacePath);
+    appendWebSearchHistory(ctx, {
+      timestamp: '2026-03-23T01:00:00Z',
+      query: 'fallback test', limit: 5, method: 'searxng',
+      resultCount: 1, outcome: 'success', fallback: true,
+    });
+    const content = fs.readFileSync(
+      path.join(workspacePath, 'logs', 'websearch-history.jsonl'), 'utf-8'
+    );
+    const record = JSON.parse(content.trim()) as Record<string, unknown>;
+    expect(record.method).toBe('searxng');
+    expect(record.fallback).toBe(true);
+    expect(record.outcome).toBe('success');
+    expect(record.query).toBe('fallback test');
+  });
+
+  it('records captcha outcome', () => {
+    workspacePath = makeWorkspace();
+    const ctx = makeContext(workspacePath);
+    appendWebSearchHistory(ctx, {
+      timestamp: '2026-03-23T01:00:00Z',
+      query: 'captcha test', limit: 5, method: 'browser',
+      resultCount: 0, outcome: 'captcha', fallback: false,
+    });
+    const content = fs.readFileSync(
+      path.join(workspacePath, 'logs', 'websearch-history.jsonl'), 'utf-8'
+    );
+    const record = JSON.parse(content.trim()) as Record<string, unknown>;
+    expect(record.outcome).toBe('captcha');
+    expect(record.method).toBe('browser');
+    expect(record.resultCount).toBe(0);
+  });
+
+  it('records error outcome with error message', () => {
+    workspacePath = makeWorkspace();
+    const ctx = makeContext(workspacePath);
+    appendWebSearchHistory(ctx, {
+      timestamp: '2026-03-23T01:00:00Z',
+      query: 'error test', limit: 5, method: 'browser',
+      resultCount: 0, outcome: 'error', fallback: false,
+      error: 'chromium not found',
+    });
+    const content = fs.readFileSync(
+      path.join(workspacePath, 'logs', 'websearch-history.jsonl'), 'utf-8'
+    );
+    const record = JSON.parse(content.trim()) as Record<string, unknown>;
+    expect(record.outcome).toBe('error');
+    expect(record.error).toBe('chromium not found');
+  });
+});
+
+describe('persistent context management', () => {
+  afterEach(() => {
+    clearPersistentContexts();
+  });
+
+  it('clearPersistentContexts resets internal state without error', () => {
+    expect(() => clearPersistentContexts()).not.toThrow();
+  });
+
+  it('clearPersistentContexts is idempotent', () => {
+    clearPersistentContexts();
+    clearPersistentContexts();
+  });
+});