This commit is contained in:
oss-sync
@@ -0,0 +1,422 @@
|
||||
/**
|
||||
* app-test.ts — TestWorkspaceApp tool.
|
||||
*
|
||||
* Mints a scoped harness token, drives the /app-harness page headlessly via
|
||||
* runPageActions, checks expectations, returns a structured E2E result, and
|
||||
* revokes the token in `finally`.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { ToolDef } from '../../llm/openai-compat.js';
|
||||
import type { ToolContext, ToolResult } from './core.js';
|
||||
import { resolveAndGuard } from './core.js';
|
||||
import { logger } from '../../logger.js';
|
||||
import { loadConfig } from '../../config.js';
|
||||
import { resolveListenPort } from '../../server/config.js';
|
||||
import { mintAppHarnessToken, revokeAppHarnessToken } from '../../bridge/app-harness-token.js';
|
||||
import type { BrowseWebAction } from './browser.js';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Pure helpers (exported for unit testing)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Build the localhost harness URL with all required query params.
|
||||
*/
|
||||
export function buildHarnessUrl(
|
||||
port: number,
|
||||
opts: { space: string; app: string; entry: string; token: string },
|
||||
): string {
|
||||
const params = new URLSearchParams({
|
||||
space: opts.space,
|
||||
app: opts.app,
|
||||
entry: opts.entry,
|
||||
token: opts.token,
|
||||
});
|
||||
return `http://localhost:${port}/app-harness?${params.toString()}`;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Canonical expect types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface ExpectItem {
|
||||
kind: 'text' | 'file';
|
||||
/** kind=text: CSS selector (informational, page text searched); kind=file: workspace-relative path */
|
||||
target: string;
|
||||
/** substring that must appear in page text or file content */
|
||||
contains: string;
|
||||
}
|
||||
|
||||
interface ValidatedInput {
|
||||
space: string;
|
||||
app: string;
|
||||
entry: string;
|
||||
steps: Record<string, unknown>[];
|
||||
seed_files: Array<{ path: string; content: string }>;
|
||||
expect: ExpectItem[];
|
||||
timeout_ms: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate raw tool input. Throws with a descriptive message on missing required
|
||||
* fields. Fills defaults for optional fields.
|
||||
*/
|
||||
export function validateInput(raw: Record<string, unknown>): ValidatedInput {
|
||||
const app = raw['app'];
|
||||
if (typeof app !== 'string' || !app.trim()) {
|
||||
throw new Error('TestWorkspaceApp: "app" (string) is required');
|
||||
}
|
||||
const space = raw['space'];
|
||||
if (typeof space !== 'string' || !space.trim()) {
|
||||
throw new Error('TestWorkspaceApp: "space" (string) is required');
|
||||
}
|
||||
|
||||
const entry =
|
||||
typeof raw['entry'] === 'string' && raw['entry'].trim()
|
||||
? raw['entry'].trim()
|
||||
: `apps/${app}/index.html`;
|
||||
|
||||
// Accept both "steps" (canonical) and legacy "actions"
|
||||
const stepsRaw = Array.isArray(raw['steps'])
|
||||
? raw['steps']
|
||||
: Array.isArray(raw['actions'])
|
||||
? raw['actions']
|
||||
: [];
|
||||
const steps = stepsRaw as Record<string, unknown>[];
|
||||
|
||||
const seed_files = Array.isArray(raw['seed_files'])
|
||||
? (raw['seed_files'] as Array<{ path: string; content: string }>)
|
||||
: [];
|
||||
|
||||
const expectRaw: ExpectItem[] = [];
|
||||
if (Array.isArray(raw['expect'])) {
|
||||
for (const item of raw['expect'] as Record<string, unknown>[]) {
|
||||
if (item['kind'] === 'text' || item['kind'] === 'file') {
|
||||
expectRaw.push({
|
||||
kind: item['kind'] as 'text' | 'file',
|
||||
target: typeof item['target'] === 'string' ? item['target'] : '',
|
||||
contains: typeof item['contains'] === 'string' ? item['contains'] : '',
|
||||
});
|
||||
}
|
||||
// Unknown kinds are silently dropped from validated list —
|
||||
// evaluateExpectations handles the "unknown kind → failure" path
|
||||
// when called with the raw array pre-validation.
|
||||
}
|
||||
}
|
||||
|
||||
const timeout_ms =
|
||||
typeof raw['timeout_ms'] === 'number' && raw['timeout_ms'] > 0
|
||||
? raw['timeout_ms']
|
||||
: 30_000;
|
||||
|
||||
return { space: space.trim(), app: app.trim(), entry, steps, seed_files, expect: expectRaw, timeout_ms };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Matcher (exported pure function for unit testing)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface EvaluateExpectationsContext {
|
||||
/** Aggregated page text from all getText/text actions */
|
||||
pageText: string;
|
||||
/** Absolute path to the workspace root */
|
||||
workspacePath: string;
|
||||
}
|
||||
|
||||
export interface ExpectationFailure {
|
||||
index: number;
|
||||
message: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluate a list of ExpectItems against captured page text and workspace files.
|
||||
* Returns an array of failure messages (empty = all passed).
|
||||
*
|
||||
* Pure in terms of network/browser — only reads the local filesystem for kind="file".
|
||||
*/
|
||||
export function evaluateExpectations(
|
||||
expectList: Array<{ kind: string; target?: string; contains?: string }>,
|
||||
ctx: EvaluateExpectationsContext,
|
||||
): string[] {
|
||||
const failures: string[] = [];
|
||||
|
||||
for (let i = 0; i < expectList.length; i++) {
|
||||
const exp = expectList[i];
|
||||
|
||||
if (exp.kind === 'text') {
|
||||
// target is treated as a hint/CSS selector (informational), contains is checked against page text
|
||||
const needle = exp.contains ?? '';
|
||||
if (!ctx.pageText.includes(needle)) {
|
||||
failures.push(
|
||||
`text[${i}]: target="${exp.target ?? ''}" — "${needle}" not found in page text`,
|
||||
);
|
||||
}
|
||||
} else if (exp.kind === 'file') {
|
||||
const filePath = exp.target ?? '';
|
||||
if (!filePath) {
|
||||
failures.push(`file[${i}]: "target" (file path) is required`);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const abs = resolveAndGuard(ctx.workspacePath, filePath);
|
||||
if (!fs.existsSync(abs)) {
|
||||
failures.push(`file[${i}]: "${filePath}" does not exist`);
|
||||
} else {
|
||||
const content = fs.readFileSync(abs, 'utf-8');
|
||||
const needle = exp.contains ?? '';
|
||||
if (!content.includes(needle)) {
|
||||
failures.push(`file[${i}]: "${filePath}" does not contain "${needle}"`);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
failures.push(`file[${i}]: error reading "${filePath}": ${(err as Error).message}`);
|
||||
}
|
||||
} else {
|
||||
// Unknown kind — always record a failure, never silently pass
|
||||
failures.push(`expect[${i}]: unknown kind "${exp.kind}" (must be "text" or "file")`);
|
||||
}
|
||||
}
|
||||
|
||||
return failures;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tool definition
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const TOOL_DEFS: Record<string, ToolDef> = {
|
||||
TestWorkspaceApp: {
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'TestWorkspaceApp',
|
||||
description:
|
||||
'ワークスペースアプリをヘッドレスブラウザで実起動して操作・期待値を検証し、結果（ok/failures/screenshots/console_errors）を返す。詳細は ReadToolDoc({ name: "TestWorkspaceApp" }) で取得可能。',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
space: {
|
||||
type: 'string',
|
||||
description: 'テスト対象スペースの ID',
|
||||
},
|
||||
app: {
|
||||
type: 'string',
|
||||
description: 'apps/ 配下のアプリフォルダ名（パスではなく名前のみ）',
|
||||
},
|
||||
entry: {
|
||||
type: 'string',
|
||||
description:
|
||||
'アプリのエントリーHTML パス（スペースファイルルートからの相対パス）。省略時は apps/{app}/index.html',
|
||||
},
|
||||
seed_files: {
|
||||
type: 'array',
|
||||
description: 'テスト前にワークスペースに書き込むファイル一覧',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
path: { type: 'string', description: 'ワークスペースルートからの相対パス' },
|
||||
content: { type: 'string', description: 'ファイル内容（UTF-8 文字列）' },
|
||||
},
|
||||
required: ['path', 'content'],
|
||||
},
|
||||
},
|
||||
steps: {
|
||||
type: 'array',
|
||||
description: 'ハーネスページ上で実行するブラウザアクション列（BrowseWebAction と同じ形式）',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
type: {
|
||||
type: 'string',
|
||||
enum: ['goto', 'click', 'fill', 'screenshot', 'getText', 'wait', 'dumpHtml'],
|
||||
},
|
||||
selector: { type: 'string' },
|
||||
ref: { type: 'string' },
|
||||
value: { type: 'string' },
|
||||
url: { type: 'string' },
|
||||
ms: { type: 'number' },
|
||||
},
|
||||
required: ['type'],
|
||||
},
|
||||
},
|
||||
expect: {
|
||||
type: 'array',
|
||||
description: '検証する期待値リスト',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
kind: {
|
||||
type: 'string',
|
||||
enum: ['text', 'file'],
|
||||
description:
|
||||
'text: target(CSSセレクタ)のページテキストが contains を含む / file: target(ワークスペース相対パス)のファイル内容が contains を含む',
|
||||
},
|
||||
target: {
|
||||
type: 'string',
|
||||
description: 'kind=text: CSS セレクタ（ページテキスト全体を検索） / kind=file: ワークスペース相対ファイルパス',
|
||||
},
|
||||
contains: {
|
||||
type: 'string',
|
||||
description: 'ページテキストまたはファイル内容に含まれるべき文字列',
|
||||
},
|
||||
},
|
||||
required: ['kind', 'target', 'contains'],
|
||||
},
|
||||
},
|
||||
timeout_ms: {
|
||||
type: 'number',
|
||||
description: 'ブラウザ操作全体のタイムアウト（ミリ秒、デフォルト 30000）',
|
||||
},
|
||||
},
|
||||
required: ['space', 'app'],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Result shape
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface AppTestResult {
|
||||
ok: boolean;
|
||||
screenshots: string[];
|
||||
console_errors: string[];
|
||||
bridge_calls: string[];
|
||||
file_changes: Array<{ path: string; bytes: number }>;
|
||||
failures: string[];
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// executeTool
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export async function executeTool(
|
||||
name: string,
|
||||
input: Record<string, unknown>,
|
||||
ctx: ToolContext,
|
||||
): Promise<ToolResult | null> {
|
||||
if (name !== 'TestWorkspaceApp') return null;
|
||||
|
||||
// Validate required context
|
||||
const spaceId = ctx.spaceId;
|
||||
const userId = ctx.userId ?? (ctx as unknown as Record<string, unknown>)['ownerId'] as string | undefined;
|
||||
if (!spaceId || !userId) {
|
||||
return {
|
||||
output: 'TestWorkspaceApp: space-scoped job required (ctx.spaceId and ctx.userId must be set)',
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
|
||||
// Validate input
|
||||
let validated: ValidatedInput;
|
||||
try {
|
||||
validated = validateInput(input);
|
||||
} catch (err) {
|
||||
return { output: (err as Error).message, isError: true };
|
||||
}
|
||||
|
||||
const { space, app, entry, steps, seed_files, expect: expectations, timeout_ms } = validated;
|
||||
|
||||
const result: AppTestResult = {
|
||||
ok: false,
|
||||
screenshots: [],
|
||||
console_errors: [],
|
||||
bridge_calls: [], // headless observation not feasible; always []
|
||||
file_changes: [],
|
||||
failures: [],
|
||||
};
|
||||
|
||||
// Write seed files into workspace (guarded by resolveAndGuard)
|
||||
for (const sf of seed_files) {
|
||||
try {
|
||||
const abs = resolveAndGuard(ctx.workspacePath, sf.path);
|
||||
fs.mkdirSync(path.dirname(abs), { recursive: true });
|
||||
fs.writeFileSync(abs, sf.content, 'utf-8');
|
||||
result.file_changes.push({ path: sf.path, bytes: Buffer.byteLength(sf.content) });
|
||||
} catch (err) {
|
||||
result.failures.push(`seed_files: failed to write "${sf.path}": ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve server port
|
||||
const port = resolveListenPort(process.env['PORT'], loadConfig().server?.port);
|
||||
|
||||
// Mint short-lived token
|
||||
const token = mintAppHarnessToken({ userId, spaceId });
|
||||
logger.debug(`[TestWorkspaceApp] token minted space=${spaceId} app=${app}`);
|
||||
|
||||
try {
|
||||
// Build harness URL
|
||||
const harnessUrl = buildHarnessUrl(port, { space, app, entry, token });
|
||||
|
||||
// Dynamically import browser module (avoids Playwright load when unused)
|
||||
const { runPageActions } = await import('./browser.js');
|
||||
|
||||
// Build action list:
|
||||
// 1. goto the harness URL
|
||||
// 2. wait for the iframe to appear (ms-based wait as BrowseWebAction has no selector-wait)
|
||||
// 3. user-supplied steps
|
||||
// 4. final screenshot
|
||||
const builtActions: BrowseWebAction[] = [
|
||||
{ type: 'goto', url: harnessUrl },
|
||||
// Wait for harness page to load — use ms wait since BrowseWebAction.wait uses ms not selector
|
||||
{ type: 'wait', ms: 2000 },
|
||||
// getText to capture iframe presence and page state
|
||||
{ type: 'getText' },
|
||||
// User steps
|
||||
...(steps as unknown as BrowseWebAction[]),
|
||||
// Final screenshot
|
||||
{ type: 'screenshot', value: `app-test-${app}-final.png` },
|
||||
];
|
||||
|
||||
const pageResult = await runPageActions(harnessUrl, builtActions, ctx, {
|
||||
isolatedContext: true,
|
||||
extraAllowedHosts: ['localhost', '127.0.0.1'],
|
||||
});
|
||||
|
||||
result.screenshots = pageResult.screenshots;
|
||||
result.console_errors = pageResult.consoleErrors;
|
||||
|
||||
// Evaluate expectations using the extracted pure function
|
||||
const pageText = pageResult.texts.join('\n');
|
||||
const expectFailures = evaluateExpectations(expectations, {
|
||||
pageText,
|
||||
workspacePath: ctx.workspacePath,
|
||||
});
|
||||
result.failures.push(...expectFailures);
|
||||
|
||||
result.ok = result.failures.length === 0;
|
||||
|
||||
const summary: Record<string, unknown> = {
|
||||
ok: result.ok,
|
||||
failures: result.failures,
|
||||
screenshots: result.screenshots,
|
||||
console_errors: result.console_errors,
|
||||
bridge_calls: result.bridge_calls,
|
||||
file_changes: result.file_changes,
|
||||
};
|
||||
|
||||
return {
|
||||
output: JSON.stringify(summary, null, 2),
|
||||
isError: !result.ok,
|
||||
};
|
||||
} catch (err) {
|
||||
const msg = (err as Error).message ?? String(err);
|
||||
logger.warn(`[TestWorkspaceApp] error: ${msg}`);
|
||||
return {
|
||||
output: JSON.stringify({
|
||||
ok: false,
|
||||
failures: [`browser error: ${msg}`],
|
||||
screenshots: result.screenshots,
|
||||
console_errors: result.console_errors,
|
||||
bridge_calls: [],
|
||||
file_changes: result.file_changes,
|
||||
}, null, 2),
|
||||
isError: true,
|
||||
};
|
||||
} finally {
|
||||
revokeAppHarnessToken(token);
|
||||
logger.debug(`[TestWorkspaceApp] token revoked`);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user