/** * app-test.ts — TestWorkspaceApp tool. * * Mints a scoped harness token, drives the /app-harness page headlessly via * runPageActions, checks expectations, returns a structured E2E result, and * revokes the token in `finally`. */ import * as fs from 'node:fs'; import * as path from 'node:path'; import { ToolDef } from '../../llm/openai-compat.js'; import type { ToolContext, ToolResult } from './tool-context.js'; import { resolveAndGuard } from './core.js'; import { logger } from '../../logger.js'; import { loadConfig } from '../../config.js'; import { resolveListenPort } from '../../server/config.js'; import { mintAppHarnessToken, revokeAppHarnessToken } from '../../bridge/app-harness-token.js'; import type { BrowseWebAction } from './browser.js'; // --------------------------------------------------------------------------- // Pure helpers (exported for unit testing) // --------------------------------------------------------------------------- /** * Build the localhost harness URL with all required query params. */ export function buildHarnessUrl( port: number, opts: { space: string; app: string; entry: string; token: string }, ): string { const params = new URLSearchParams({ space: opts.space, app: opts.app, entry: opts.entry, token: opts.token, }); return `http://localhost:${port}/app-harness?${params.toString()}`; } // --------------------------------------------------------------------------- // Canonical expect types // --------------------------------------------------------------------------- export interface ExpectItem { kind: 'text' | 'file'; /** kind=text: CSS selector (informational, page text searched); kind=file: workspace-relative path */ target: string; /** substring that must appear in page text or file content */ contains: string; } interface ValidatedInput { app: string; entry: string; steps: Record[]; seed_files: Array<{ path: string; content: string }>; expect: ExpectItem[]; timeout_ms: number; } /** * Validate raw tool input. Throws with a descriptive message on missing required * fields. Fills defaults for optional fields. */ export function validateInput(raw: Record): ValidatedInput { const app = raw['app']; if (typeof app !== 'string' || !app.trim()) { throw new Error('TestWorkspaceApp: "app" (string) is required'); } const entry = typeof raw['entry'] === 'string' && raw['entry'].trim() ? raw['entry'].trim() : `apps/${app}/index.html`; // Accept both "steps" (canonical) and legacy "actions" const stepsRaw = Array.isArray(raw['steps']) ? raw['steps'] : Array.isArray(raw['actions']) ? raw['actions'] : []; const steps = stepsRaw as Record[]; const seed_files = Array.isArray(raw['seed_files']) ? (raw['seed_files'] as Array<{ path: string; content: string }>) : []; const expectRaw: ExpectItem[] = []; if (Array.isArray(raw['expect'])) { for (const item of raw['expect'] as Record[]) { if (item['kind'] === 'text' || item['kind'] === 'file') { expectRaw.push({ kind: item['kind'] as 'text' | 'file', target: typeof item['target'] === 'string' ? item['target'] : '', contains: typeof item['contains'] === 'string' ? item['contains'] : '', }); } // Unknown kinds are silently dropped from validated list — // evaluateExpectations handles the "unknown kind → failure" path // when called with the raw array pre-validation. } } const timeout_ms = Math.min(120_000, typeof raw['timeout_ms'] === 'number' && raw['timeout_ms'] > 0 ? raw['timeout_ms'] : 30_000); return { app: app.trim(), entry, steps, seed_files, expect: expectRaw, timeout_ms }; } // --------------------------------------------------------------------------- // Matcher (exported pure function for unit testing) // --------------------------------------------------------------------------- export interface EvaluateExpectationsContext { /** Aggregated page text from all getText/text actions */ pageText: string; /** Absolute path to the workspace root */ workspacePath: string; targetedText?: Record; } export interface ExpectationFailure { index: number; message: string; } /** * Evaluate a list of ExpectItems against captured page text and workspace files. * Returns an array of failure messages (empty = all passed). * * Pure in terms of network/browser — only reads the local filesystem for kind="file". */ export function evaluateExpectations( expectList: Array<{ kind: string; target?: string; contains?: string }>, ctx: EvaluateExpectationsContext, ): string[] { const failures: string[] = []; for (let i = 0; i < expectList.length; i++) { const exp = expectList[i]; if (exp.kind === 'text') { // target is treated as a hint/CSS selector (informational), contains is checked against page text const needle = exp.contains ?? ''; const haystack = exp.target && exp.target !== 'body' ? (ctx.targetedText?.[exp.target] ?? '') : ctx.pageText; if (!haystack.includes(needle)) { failures.push( `text[${i}]: target="${exp.target ?? ''}" — "${needle}" not found in page text`, ); } } else if (exp.kind === 'file') { const filePath = exp.target ?? ''; if (!filePath) { failures.push(`file[${i}]: "target" (file path) is required`); continue; } try { const abs = resolveAndGuard(ctx.workspacePath, filePath); if (!fs.existsSync(abs)) { failures.push(`file[${i}]: "${filePath}" does not exist`); } else { const content = fs.readFileSync(abs, 'utf-8'); const needle = exp.contains ?? ''; if (!content.includes(needle)) { failures.push(`file[${i}]: "${filePath}" does not contain "${needle}"`); } } } catch (err) { failures.push(`file[${i}]: error reading "${filePath}": ${(err as Error).message}`); } } else { // Unknown kind — always record a failure, never silently pass failures.push(`expect[${i}]: unknown kind "${exp.kind}" (must be "text" or "file")`); } } return failures; } // --------------------------------------------------------------------------- // Tool definition // --------------------------------------------------------------------------- export const TOOL_DEFS: Record = { TestWorkspaceApp: { type: 'function', function: { name: 'TestWorkspaceApp', description: 'ワークスペースアプリをヘッドレスブラウザで実起動して操作・期待値を検証し、結果(ok/failures/screenshots/console_errors)を返す。詳細は ReadToolDoc({ name: "TestWorkspaceApp" }) で取得可能。', parameters: { type: 'object', properties: { app: { type: 'string', description: 'apps/ 配下のアプリフォルダ名(パスではなく名前のみ)', }, entry: { type: 'string', description: 'アプリのエントリーHTML パス(スペースファイルルートからの相対パス)。省略時は apps/{app}/index.html', }, seed_files: { type: 'array', description: 'テスト前にワークスペースに書き込むファイル一覧', items: { type: 'object', properties: { path: { type: 'string', description: 'ワークスペースルートからの相対パス' }, content: { type: 'string', description: 'ファイル内容(UTF-8 文字列)' }, }, required: ['path', 'content'], }, }, steps: { type: 'array', description: 'ハーネスページ上で実行するブラウザアクション列(BrowseWebAction と同じ形式)', items: { type: 'object', properties: { type: { type: 'string', enum: ['goto', 'click', 'fill', 'screenshot', 'getText', 'wait', 'dumpHtml'], }, selector: { type: 'string' }, ref: { type: 'string' }, value: { type: 'string' }, url: { type: 'string' }, ms: { type: 'number' }, }, required: ['type'], }, }, expect: { type: 'array', description: '検証する期待値リスト', items: { type: 'object', properties: { kind: { type: 'string', enum: ['text', 'file'], description: 'text: target(CSSセレクタ)のページテキストが contains を含む / file: target(ワークスペース相対パス)のファイル内容が contains を含む', }, target: { type: 'string', description: 'kind=text: CSS セレクタ(ページテキスト全体を検索) / kind=file: ワークスペース相対ファイルパス', }, contains: { type: 'string', description: 'ページテキストまたはファイル内容に含まれるべき文字列', }, }, required: ['kind', 'target', 'contains'], }, }, timeout_ms: { type: 'number', description: 'ブラウザ操作全体のタイムアウト(ミリ秒、デフォルト 30000)', }, }, required: ['app'], }, }, }, }; // --------------------------------------------------------------------------- // Result shape // --------------------------------------------------------------------------- interface AppTestResult { ok: boolean; screenshots: string[]; console_errors: string[]; bridge_calls: string[]; file_changes: Array<{ path: string; bytes: number }>; failures: string[]; } // --------------------------------------------------------------------------- // executeTool // --------------------------------------------------------------------------- export async function executeTool( name: string, input: Record, ctx: ToolContext, ): Promise { if (name !== 'TestWorkspaceApp') return null; // Validate required context const spaceId = ctx.spaceId; const userId = ctx.userId ?? (ctx as unknown as Record)['ownerId'] as string | undefined; if (!spaceId || !userId) { return { output: 'TestWorkspaceApp: space-scoped job required (ctx.spaceId and ctx.userId must be set)', isError: true, }; } // Validate input let validated: ValidatedInput; try { validated = validateInput(input); } catch (err) { return { output: (err as Error).message, isError: true }; } const { app, entry, steps, seed_files, expect: expectations, timeout_ms } = validated; const result: AppTestResult = { ok: false, screenshots: [], console_errors: [], bridge_calls: [], // headless observation not feasible; always [] file_changes: [], failures: [], }; // Write seed files into workspace (guarded by resolveAndGuard) for (const sf of seed_files) { try { const abs = resolveAndGuard(ctx.workspacePath, sf.path); fs.mkdirSync(path.dirname(abs), { recursive: true }); fs.writeFileSync(abs, sf.content, 'utf-8'); result.file_changes.push({ path: sf.path, bytes: Buffer.byteLength(sf.content) }); } catch (err) { result.failures.push(`seed_files: failed to write "${sf.path}": ${(err as Error).message}`); } } // Resolve server port const port = resolveListenPort(process.env['PORT'], loadConfig().server?.port); // Mint short-lived token const token = mintAppHarnessToken({ userId, spaceId }); logger.debug(`[TestWorkspaceApp] token minted space=${spaceId} app=${app}`); try { // Build harness URL const harnessUrl = buildHarnessUrl(port, { space: spaceId, app, entry, token }); // Dynamically import browser module (avoids Playwright load when unused) const { runPageActions } = await import('./browser.js'); // Build action list: // 1. goto the harness URL // 2. wait for the iframe to appear (ms-based wait as BrowseWebAction has no selector-wait) // 3. user-supplied steps // 4. final screenshot const builtActions: BrowseWebAction[] = [ { type: 'goto', url: harnessUrl }, // Wait for harness page to load — use ms wait since BrowseWebAction.wait uses ms not selector { type: 'wait', ms: 2000 }, // getText to capture iframe presence and page state { type: 'getText' }, // User steps ...(steps as unknown as BrowseWebAction[]), ...expectations.filter(item => item.kind === 'text' && item.target && item.target !== 'body') .map(item => ({ type: 'getText' as const, selector: item.target })), // Final screenshot { type: 'screenshot', value: `app-test-${app}-final.png` }, ]; // TestWorkspaceApp keeps single-image screenshots (design scope: do not adopt // BrowseWeb's default viewport segmentation here). Force full-page single // capture for any screenshot action unless a step explicitly opts in. for (const a of builtActions) { if (a.type === 'screenshot' && a.segments === undefined) a.segments = false; } const timeoutController = new AbortController(); const timeoutHandle = setTimeout(() => timeoutController.abort(), timeout_ms); let pageResult: Awaited>; try { pageResult = await runPageActions(harnessUrl, builtActions, ctx, { isolatedContext: true, extraAllowedHosts: ['localhost', '127.0.0.1'], defaultToFirstChildFrame: true, abortSignal: timeoutController.signal, }); if (timeoutController.signal.aborted) throw new Error(`timed out after ${timeout_ms}ms`); } catch (error) { if (timeoutController.signal.aborted) throw new Error(`timed out after ${timeout_ms}ms`); throw error; } finally { clearTimeout(timeoutHandle); } result.screenshots = pageResult.screenshots; result.console_errors = pageResult.consoleErrors; for (const error of result.console_errors) result.failures.push(`app diagnostic: ${error}`); // Evaluate expectations using the extracted pure function const pageText = pageResult.texts.join('\n'); const targetedText: Record = {}; const targeted = expectations.filter(item => item.kind === 'text' && item.target && item.target !== 'body'); const captured = pageResult.texts.slice(-targeted.length); targeted.forEach((item, index) => { targetedText[item.target] = captured[index] ?? ''; }); const expectFailures = evaluateExpectations(expectations, { pageText, workspacePath: ctx.workspacePath, targetedText, }); result.failures.push(...expectFailures); result.ok = result.failures.length === 0; const summary: Record = { ok: result.ok, failures: result.failures, screenshots: result.screenshots, console_errors: result.console_errors, bridge_calls: result.bridge_calls, file_changes: result.file_changes, }; return { output: JSON.stringify(summary, null, 2), isError: !result.ok, }; } catch (err) { const msg = (err as Error).message ?? String(err); logger.warn(`[TestWorkspaceApp] error: ${msg}`); return { output: JSON.stringify({ ok: false, failures: [`browser error: ${msg}`], screenshots: result.screenshots, console_errors: result.console_errors, bridge_calls: [], file_changes: result.file_changes, }, null, 2), isError: true, }; } finally { revokeAppHarnessToken(token); logger.debug(`[TestWorkspaceApp] token revoked`); } }