445 lines
16 KiB
TypeScript
445 lines
16 KiB
TypeScript
/**
|
|
* app-test.ts — TestWorkspaceApp tool.
|
|
*
|
|
* Mints a scoped harness token, drives the /app-harness page headlessly via
|
|
* runPageActions, checks expectations, returns a structured E2E result, and
|
|
* revokes the token in `finally`.
|
|
*/
|
|
import * as fs from 'node:fs';
|
|
import * as path from 'node:path';
|
|
import { ToolDef } from '../../llm/openai-compat.js';
|
|
import type { ToolContext, ToolResult } from './tool-context.js';
|
|
import { resolveAndGuard } from './core.js';
|
|
import { logger } from '../../logger.js';
|
|
import { loadConfig } from '../../config.js';
|
|
import { resolveListenPort } from '../../server/config.js';
|
|
import { mintAppHarnessToken, revokeAppHarnessToken } from '../../bridge/app-harness-token.js';
|
|
import type { BrowseWebAction } from './browser.js';
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Pure helpers (exported for unit testing)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Build the localhost harness URL with all required query params.
|
|
*/
|
|
export function buildHarnessUrl(
|
|
port: number,
|
|
opts: { space: string; app: string; entry: string; token: string },
|
|
): string {
|
|
const params = new URLSearchParams({
|
|
space: opts.space,
|
|
app: opts.app,
|
|
entry: opts.entry,
|
|
token: opts.token,
|
|
});
|
|
return `http://localhost:${port}/app-harness?${params.toString()}`;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Canonical expect types
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export interface ExpectItem {
|
|
kind: 'text' | 'file';
|
|
/** kind=text: CSS selector (informational, page text searched); kind=file: workspace-relative path */
|
|
target: string;
|
|
/** substring that must appear in page text or file content */
|
|
contains: string;
|
|
}
|
|
|
|
interface ValidatedInput {
|
|
app: string;
|
|
entry: string;
|
|
steps: Record<string, unknown>[];
|
|
seed_files: Array<{ path: string; content: string }>;
|
|
expect: ExpectItem[];
|
|
timeout_ms: number;
|
|
}
|
|
|
|
/**
|
|
* Validate raw tool input. Throws with a descriptive message on missing required
|
|
* fields. Fills defaults for optional fields.
|
|
*/
|
|
export function validateInput(raw: Record<string, unknown>): ValidatedInput {
|
|
const app = raw['app'];
|
|
if (typeof app !== 'string' || !app.trim()) {
|
|
throw new Error('TestWorkspaceApp: "app" (string) is required');
|
|
}
|
|
const entry =
|
|
typeof raw['entry'] === 'string' && raw['entry'].trim()
|
|
? raw['entry'].trim()
|
|
: `apps/${app}/index.html`;
|
|
|
|
// Accept both "steps" (canonical) and legacy "actions"
|
|
const stepsRaw = Array.isArray(raw['steps'])
|
|
? raw['steps']
|
|
: Array.isArray(raw['actions'])
|
|
? raw['actions']
|
|
: [];
|
|
const steps = stepsRaw as Record<string, unknown>[];
|
|
|
|
const seed_files = Array.isArray(raw['seed_files'])
|
|
? (raw['seed_files'] as Array<{ path: string; content: string }>)
|
|
: [];
|
|
|
|
const expectRaw: ExpectItem[] = [];
|
|
if (Array.isArray(raw['expect'])) {
|
|
for (const item of raw['expect'] as Record<string, unknown>[]) {
|
|
if (item['kind'] === 'text' || item['kind'] === 'file') {
|
|
expectRaw.push({
|
|
kind: item['kind'] as 'text' | 'file',
|
|
target: typeof item['target'] === 'string' ? item['target'] : '',
|
|
contains: typeof item['contains'] === 'string' ? item['contains'] : '',
|
|
});
|
|
}
|
|
// Unknown kinds are silently dropped from validated list —
|
|
// evaluateExpectations handles the "unknown kind → failure" path
|
|
// when called with the raw array pre-validation.
|
|
}
|
|
}
|
|
|
|
const timeout_ms = Math.min(120_000,
|
|
typeof raw['timeout_ms'] === 'number' && raw['timeout_ms'] > 0
|
|
? raw['timeout_ms']
|
|
: 30_000);
|
|
|
|
return { app: app.trim(), entry, steps, seed_files, expect: expectRaw, timeout_ms };
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Matcher (exported pure function for unit testing)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export interface EvaluateExpectationsContext {
|
|
/** Aggregated page text from all getText/text actions */
|
|
pageText: string;
|
|
/** Absolute path to the workspace root */
|
|
workspacePath: string;
|
|
targetedText?: Record<string, string>;
|
|
}
|
|
|
|
export interface ExpectationFailure {
|
|
index: number;
|
|
message: string;
|
|
}
|
|
|
|
/**
|
|
* Evaluate a list of ExpectItems against captured page text and workspace files.
|
|
* Returns an array of failure messages (empty = all passed).
|
|
*
|
|
* Pure in terms of network/browser — only reads the local filesystem for kind="file".
|
|
*/
|
|
export function evaluateExpectations(
|
|
expectList: Array<{ kind: string; target?: string; contains?: string }>,
|
|
ctx: EvaluateExpectationsContext,
|
|
): string[] {
|
|
const failures: string[] = [];
|
|
|
|
for (let i = 0; i < expectList.length; i++) {
|
|
const exp = expectList[i];
|
|
|
|
if (exp.kind === 'text') {
|
|
// target is treated as a hint/CSS selector (informational), contains is checked against page text
|
|
const needle = exp.contains ?? '';
|
|
const haystack = exp.target && exp.target !== 'body'
|
|
? (ctx.targetedText?.[exp.target] ?? '')
|
|
: ctx.pageText;
|
|
if (!haystack.includes(needle)) {
|
|
failures.push(
|
|
`text[${i}]: target="${exp.target ?? ''}" — "${needle}" not found in page text`,
|
|
);
|
|
}
|
|
} else if (exp.kind === 'file') {
|
|
const filePath = exp.target ?? '';
|
|
if (!filePath) {
|
|
failures.push(`file[${i}]: "target" (file path) is required`);
|
|
continue;
|
|
}
|
|
try {
|
|
const abs = resolveAndGuard(ctx.workspacePath, filePath);
|
|
if (!fs.existsSync(abs)) {
|
|
failures.push(`file[${i}]: "${filePath}" does not exist`);
|
|
} else {
|
|
const content = fs.readFileSync(abs, 'utf-8');
|
|
const needle = exp.contains ?? '';
|
|
if (!content.includes(needle)) {
|
|
failures.push(`file[${i}]: "${filePath}" does not contain "${needle}"`);
|
|
}
|
|
}
|
|
} catch (err) {
|
|
failures.push(`file[${i}]: error reading "${filePath}": ${(err as Error).message}`);
|
|
}
|
|
} else {
|
|
// Unknown kind — always record a failure, never silently pass
|
|
failures.push(`expect[${i}]: unknown kind "${exp.kind}" (must be "text" or "file")`);
|
|
}
|
|
}
|
|
|
|
return failures;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Tool definition
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export const TOOL_DEFS: Record<string, ToolDef> = {
|
|
TestWorkspaceApp: {
|
|
type: 'function',
|
|
function: {
|
|
name: 'TestWorkspaceApp',
|
|
description:
|
|
'ワークスペースアプリをヘッドレスブラウザで実起動して操作・期待値を検証し、結果（ok/failures/screenshots/console_errors）を返す。詳細は ReadToolDoc({ name: "TestWorkspaceApp" }) で取得可能。',
|
|
parameters: {
|
|
type: 'object',
|
|
properties: {
|
|
app: {
|
|
type: 'string',
|
|
description: 'apps/ 配下のアプリフォルダ名（パスではなく名前のみ）',
|
|
},
|
|
entry: {
|
|
type: 'string',
|
|
description:
|
|
'アプリのエントリーHTML パス（スペースファイルルートからの相対パス）。省略時は apps/{app}/index.html',
|
|
},
|
|
seed_files: {
|
|
type: 'array',
|
|
description: 'テスト前にワークスペースに書き込むファイル一覧',
|
|
items: {
|
|
type: 'object',
|
|
properties: {
|
|
path: { type: 'string', description: 'ワークスペースルートからの相対パス' },
|
|
content: { type: 'string', description: 'ファイル内容（UTF-8 文字列）' },
|
|
},
|
|
required: ['path', 'content'],
|
|
},
|
|
},
|
|
steps: {
|
|
type: 'array',
|
|
description: 'ハーネスページ上で実行するブラウザアクション列（BrowseWebAction と同じ形式）',
|
|
items: {
|
|
type: 'object',
|
|
properties: {
|
|
type: {
|
|
type: 'string',
|
|
enum: ['goto', 'click', 'fill', 'screenshot', 'getText', 'wait', 'dumpHtml'],
|
|
},
|
|
selector: { type: 'string' },
|
|
ref: { type: 'string' },
|
|
value: { type: 'string' },
|
|
url: { type: 'string' },
|
|
ms: { type: 'number' },
|
|
},
|
|
required: ['type'],
|
|
},
|
|
},
|
|
expect: {
|
|
type: 'array',
|
|
description: '検証する期待値リスト',
|
|
items: {
|
|
type: 'object',
|
|
properties: {
|
|
kind: {
|
|
type: 'string',
|
|
enum: ['text', 'file'],
|
|
description:
|
|
'text: target(CSSセレクタ)のページテキストが contains を含む / file: target(ワークスペース相対パス)のファイル内容が contains を含む',
|
|
},
|
|
target: {
|
|
type: 'string',
|
|
description: 'kind=text: CSS セレクタ（ページテキスト全体を検索） / kind=file: ワークスペース相対ファイルパス',
|
|
},
|
|
contains: {
|
|
type: 'string',
|
|
description: 'ページテキストまたはファイル内容に含まれるべき文字列',
|
|
},
|
|
},
|
|
required: ['kind', 'target', 'contains'],
|
|
},
|
|
},
|
|
timeout_ms: {
|
|
type: 'number',
|
|
description: 'ブラウザ操作全体のタイムアウト（ミリ秒、デフォルト 30000）',
|
|
},
|
|
},
|
|
required: ['app'],
|
|
},
|
|
},
|
|
},
|
|
};
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Result shape
|
|
// ---------------------------------------------------------------------------
|
|
|
|
interface AppTestResult {
|
|
ok: boolean;
|
|
screenshots: string[];
|
|
console_errors: string[];
|
|
bridge_calls: string[];
|
|
file_changes: Array<{ path: string; bytes: number }>;
|
|
failures: string[];
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// executeTool
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export async function executeTool(
|
|
name: string,
|
|
input: Record<string, unknown>,
|
|
ctx: ToolContext,
|
|
): Promise<ToolResult | null> {
|
|
if (name !== 'TestWorkspaceApp') return null;
|
|
|
|
// Validate required context
|
|
const spaceId = ctx.spaceId;
|
|
const userId = ctx.userId ?? (ctx as unknown as Record<string, unknown>)['ownerId'] as string | undefined;
|
|
if (!spaceId || !userId) {
|
|
return {
|
|
output: 'TestWorkspaceApp: space-scoped job required (ctx.spaceId and ctx.userId must be set)',
|
|
isError: true,
|
|
};
|
|
}
|
|
|
|
// Validate input
|
|
let validated: ValidatedInput;
|
|
try {
|
|
validated = validateInput(input);
|
|
} catch (err) {
|
|
return { output: (err as Error).message, isError: true };
|
|
}
|
|
|
|
const { app, entry, steps, seed_files, expect: expectations, timeout_ms } = validated;
|
|
|
|
const result: AppTestResult = {
|
|
ok: false,
|
|
screenshots: [],
|
|
console_errors: [],
|
|
bridge_calls: [], // headless observation not feasible; always []
|
|
file_changes: [],
|
|
failures: [],
|
|
};
|
|
|
|
// Write seed files into workspace (guarded by resolveAndGuard)
|
|
for (const sf of seed_files) {
|
|
try {
|
|
const abs = resolveAndGuard(ctx.workspacePath, sf.path);
|
|
fs.mkdirSync(path.dirname(abs), { recursive: true });
|
|
fs.writeFileSync(abs, sf.content, 'utf-8');
|
|
result.file_changes.push({ path: sf.path, bytes: Buffer.byteLength(sf.content) });
|
|
} catch (err) {
|
|
result.failures.push(`seed_files: failed to write "${sf.path}": ${(err as Error).message}`);
|
|
}
|
|
}
|
|
|
|
// Resolve server port
|
|
const port = resolveListenPort(process.env['PORT'], loadConfig().server?.port);
|
|
|
|
// Mint short-lived token
|
|
const token = mintAppHarnessToken({ userId, spaceId });
|
|
logger.debug(`[TestWorkspaceApp] token minted space=${spaceId} app=${app}`);
|
|
|
|
try {
|
|
// Build harness URL
|
|
const harnessUrl = buildHarnessUrl(port, { space: spaceId, app, entry, token });
|
|
|
|
// Dynamically import browser module (avoids Playwright load when unused)
|
|
const { runPageActions } = await import('./browser.js');
|
|
|
|
// Build action list:
|
|
// 1. goto the harness URL
|
|
// 2. wait for the iframe to appear (ms-based wait as BrowseWebAction has no selector-wait)
|
|
// 3. user-supplied steps
|
|
// 4. final screenshot
|
|
const builtActions: BrowseWebAction[] = [
|
|
{ type: 'goto', url: harnessUrl },
|
|
// Wait for harness page to load — use ms wait since BrowseWebAction.wait uses ms not selector
|
|
{ type: 'wait', ms: 2000 },
|
|
// getText to capture iframe presence and page state
|
|
{ type: 'getText' },
|
|
// User steps
|
|
...(steps as unknown as BrowseWebAction[]),
|
|
...expectations.filter(item => item.kind === 'text' && item.target && item.target !== 'body')
|
|
.map(item => ({ type: 'getText' as const, selector: item.target })),
|
|
// Final screenshot
|
|
{ type: 'screenshot', value: `app-test-${app}-final.png` },
|
|
];
|
|
|
|
// TestWorkspaceApp keeps single-image screenshots (design scope: do not adopt
|
|
// BrowseWeb's default viewport segmentation here). Force full-page single
|
|
// capture for any screenshot action unless a step explicitly opts in.
|
|
for (const a of builtActions) {
|
|
if (a.type === 'screenshot' && a.segments === undefined) a.segments = false;
|
|
}
|
|
|
|
const timeoutController = new AbortController();
|
|
const timeoutHandle = setTimeout(() => timeoutController.abort(), timeout_ms);
|
|
let pageResult: Awaited<ReturnType<typeof runPageActions>>;
|
|
try {
|
|
pageResult = await runPageActions(harnessUrl, builtActions, ctx, {
|
|
isolatedContext: true,
|
|
extraAllowedHosts: ['localhost', '127.0.0.1'],
|
|
defaultToFirstChildFrame: true,
|
|
abortSignal: timeoutController.signal,
|
|
});
|
|
if (timeoutController.signal.aborted) throw new Error(`timed out after ${timeout_ms}ms`);
|
|
} catch (error) {
|
|
if (timeoutController.signal.aborted) throw new Error(`timed out after ${timeout_ms}ms`);
|
|
throw error;
|
|
} finally {
|
|
clearTimeout(timeoutHandle);
|
|
}
|
|
|
|
result.screenshots = pageResult.screenshots;
|
|
result.console_errors = pageResult.consoleErrors;
|
|
for (const error of result.console_errors) result.failures.push(`app diagnostic: ${error}`);
|
|
|
|
// Evaluate expectations using the extracted pure function
|
|
const pageText = pageResult.texts.join('\n');
|
|
const targetedText: Record<string, string> = {};
|
|
const targeted = expectations.filter(item => item.kind === 'text' && item.target && item.target !== 'body');
|
|
const captured = pageResult.texts.slice(-targeted.length);
|
|
targeted.forEach((item, index) => { targetedText[item.target] = captured[index] ?? ''; });
|
|
const expectFailures = evaluateExpectations(expectations, {
|
|
pageText,
|
|
workspacePath: ctx.workspacePath,
|
|
targetedText,
|
|
});
|
|
result.failures.push(...expectFailures);
|
|
|
|
result.ok = result.failures.length === 0;
|
|
|
|
const summary: Record<string, unknown> = {
|
|
ok: result.ok,
|
|
failures: result.failures,
|
|
screenshots: result.screenshots,
|
|
console_errors: result.console_errors,
|
|
bridge_calls: result.bridge_calls,
|
|
file_changes: result.file_changes,
|
|
};
|
|
|
|
return {
|
|
output: JSON.stringify(summary, null, 2),
|
|
isError: !result.ok,
|
|
};
|
|
} catch (err) {
|
|
const msg = (err as Error).message ?? String(err);
|
|
logger.warn(`[TestWorkspaceApp] error: ${msg}`);
|
|
return {
|
|
output: JSON.stringify({
|
|
ok: false,
|
|
failures: [`browser error: ${msg}`],
|
|
screenshots: result.screenshots,
|
|
console_errors: result.console_errors,
|
|
bridge_calls: [],
|
|
file_changes: result.file_changes,
|
|
}, null, 2),
|
|
isError: true,
|
|
};
|
|
} finally {
|
|
revokeAppHarnessToken(token);
|
|
logger.debug(`[TestWorkspaceApp] token revoked`);
|
|
}
|
|
}
|