maestro/src/engine/piece-runner.delegate.test.ts
oss-sync 63d34d7cf6
All checks were successful
CI / build-and-test (push) Successful in 6m22s
sync: update from private repo (061c701c)
2026-07-09 03:01:06 +00:00

555 lines
22 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/**
* Phase B Task 4 — E2E integration test: runPiece with delegate tool.
*
* Exercises the real runPiece → executeMovement → runDelegateSubAgent path.
* Only tools/index.js is mocked.
*
* Assertions:
* (a) The sub-agent's complete.result propagates back to the parent as the
* delegate tool's output (appears in parent conversation + final result).
* (b) ISOLATION: The sub-agent's own seed messages (system/user) and
* intermediate turns do NOT appear as standalone parent conversation
* messages. We use SpyFakeClient to snapshot the parent's messages array
* at each LLM call — the sub's turns are never added there.
* (c) The delegate tool result equals the sub's complete.result exactly.
*/
import { afterEach, describe, expect, it, vi } from 'vitest';
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
import { join } from 'node:path';
import { tmpdir } from 'node:os';
import type { LLMEvent, ToolDef } from '../llm/openai-compat.js';
import type { PieceDef } from './piece-runner.js';
import type { ToolContext } from './tools/index.js';
// ---------------------------------------------------------------------------
// Hoist mocks before any imports that use tools/index.js
// ---------------------------------------------------------------------------
const { executeToolMock, getToolDefsMock } = vi.hoisted(() => ({
executeToolMock: vi.fn(),
getToolDefsMock: vi.fn(),
}));
vi.mock('./tools/index.js', () => ({
executeTool: executeToolMock,
getToolDefs: getToolDefsMock,
}));
// Import after mocking
import { runPiece } from './piece-runner.js';
import { Conversation } from './context/conversation.js';
import { reconstructDelegateRuns } from '../progress/delegate-runs.js';
// ---------------------------------------------------------------------------
// Unique sentinel strings — must survive the full test without false positives
// ---------------------------------------------------------------------------
const SUB_PROMPT_MARKER = 'SUBTASK_PROMPT_MARKER_x9 analyze the data';
const SUB_RESULT_MARKER = 'SUB_RESULT_MARKER_q7';
const PARENT_TASK_INSTRUCTION = 'PARENT_TASK_E2E_DELEGATE_TEST_7f3a';
// ---------------------------------------------------------------------------
// SpyFakeClient — takes snapshots of the parent messages array at each call.
// Snapshots are taken at call time so later mutations don't affect them.
// ---------------------------------------------------------------------------
class SpyFakeClient {
private index = 0;
/** Deep-shallow copies of messages at the moment of each LLM call. */
readonly snapshots: Array<
Array<{
role: string;
content?: unknown;
tool_calls?: Array<{ id: string; function: { name: string } }>;
tool_call_id?: string;
}>
> = [];
constructor(private readonly responses: LLMEvent[][]) {}
async *chat(
messages: unknown,
_tools?: unknown,
_signal?: AbortSignal,
): AsyncGenerator<LLMEvent> {
// Snapshot NOW (before sub-agent appends its own messages)
this.snapshots.push(
(messages as Array<Record<string, unknown>>).map((m) => ({ ...m })) as never,
);
const response = this.responses[this.index++] ?? [];
for (const event of response) {
yield event;
}
}
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function makeToolDefs(names: string[]): ToolDef[] {
return names.map((name) => ({
type: 'function' as const,
function: {
name,
description: name,
parameters: { type: 'object', properties: {}, required: [] },
},
}));
}
function makeTmpWorkspace(): { workspacePath: string; logsRoot: string } {
const workspacePath = mkdtempSync(join(tmpdir(), 'piece-runner-delegate-e2e-'));
const logsRoot = join(workspacePath, 'logs');
mkdirSync(logsRoot, { recursive: true });
return { workspacePath, logsRoot };
}
/**
* A one-movement piece whose single movement has 'delegate' in allowed_tools
* and no rules (so only 'complete' is offered as a terminal — no transition
* tool is built). The movement calls delegate, then calls complete.
*/
function delegatePiece(): PieceDef {
return {
name: 'delegate-e2e-test-piece',
description: 'E2E test piece for delegate tool',
max_movements: 5,
initial_movement: 'main',
movements: [
{
name: 'main',
edit: false,
persona: 'orchestrator',
instruction: 'Delegate a subtask and then complete.',
allowed_tools: ['delegate'],
rules: [],
// No default_next — complete is the only exit.
},
],
};
}
// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------
describe('runPiece — delegate E2E (Phase B Task 4)', () => {
let workspacePath = '';
afterEach(() => {
executeToolMock.mockReset();
getToolDefsMock.mockReset();
if (workspacePath) {
rmSync(workspacePath, { recursive: true, force: true });
workspacePath = '';
}
});
it(
'(a)(b)(c) sub result propagates to parent, sub turns absent from parent messages, transcript isolation',
async () => {
const tmp = makeTmpWorkspace();
workspacePath = tmp.workspacePath;
const { logsRoot } = tmp;
// getToolDefs is called for both parent and sub-agent runs.
// Return 'delegate' for parent, return nothing (or complete-only) for sub
// — but since the real code always adds 'complete' as a meta-tool for
// sub (rules:[]), returning an empty list here is fine: getToolDefs is
// only consulted for the explicit allowed_tools list.
getToolDefsMock.mockResolvedValue(makeToolDefs(['delegate']));
// executeToolMock: forward 'delegate' calls to ctx.runDelegate (same
// pattern as agent-loop.delegate.test.ts — this is what orchestration.ts
// does in production).
executeToolMock.mockImplementation(
async (name: string, input: Record<string, unknown>, ctx: ToolContext) => {
if (name === 'delegate') {
if (!ctx.runDelegate) {
return { output: 'runDelegate not injected', isError: true };
}
const { result, status } = await ctx.runDelegate({
description: String(input['description'] ?? 'sub-task'),
prompt: String(input['prompt'] ?? ''),
});
if (status === 'aborted') {
return { output: `[delegate aborted] ${result}`, isError: true };
}
return { output: result, isError: false };
}
return { output: 'ok', isError: false };
},
);
// Scripted call order (single shared client — sub-agent shares the same
// client instance as the parent via runDelegateSubAgent):
// [0] parent turn 1 → emits delegate tool_call with SUB_PROMPT_MARKER
// [1] sub turn 1 → emits complete({status:'success', result: SUB_RESULT_MARKER})
// [2] parent turn 2 → emits complete({status:'success', result: 'parent finished with: SUB_RESULT_MARKER_q7'})
const client = new SpyFakeClient([
// parent turn 1 — delegate
[
{
type: 'tool_use',
id: 'del-e2e-1',
name: 'delegate',
input: {
description: 'analyze data subtask',
prompt: SUB_PROMPT_MARKER,
},
},
{ type: 'done', usage: { prompt_tokens: 100, completion_tokens: 10 } },
],
// sub turn 1 — complete (sub-agent finishes)
[
{
type: 'tool_use',
id: 'sub-e2e-complete-1',
name: 'complete',
input: { status: 'success', result: SUB_RESULT_MARKER },
},
{ type: 'done', usage: { prompt_tokens: 80, completion_tokens: 5 } },
],
// parent turn 2 — complete (parent references sub result)
[
{
type: 'tool_use',
id: 'par-e2e-complete-1',
name: 'complete',
input: {
status: 'success',
result: `parent finished with: ${SUB_RESULT_MARKER}`,
},
},
{ type: 'done', usage: { prompt_tokens: 120, completion_tokens: 10 } },
],
]);
const piece = delegatePiece();
const pieceResult = await runPiece(
piece,
PARENT_TASK_INSTRUCTION,
client as never,
workspacePath,
undefined,
undefined,
// PR B: tool availability comes from the workspace policy, not the piece.
// 'delegate' must be in the resolved set for ctx.runDelegate to be wired.
{ runtimeDir: logsRoot, workspaceTools: { allowedTools: ['delegate'], editAllowed: true } },
);
// --- Basic completion ---
expect(pieceResult.status).toBe('completed');
expect(pieceResult.finalOutput).toContain(SUB_RESULT_MARKER);
// --- (c) delegate tool result equals sub's complete.result ---
// The parent's final output must contain SUB_RESULT_MARKER (set by sub's
// complete.result). This proves the result was threaded through unchanged.
expect(pieceResult.finalOutput).toBe(`parent finished with: ${SUB_RESULT_MARKER}`);
// --- (a) Sub result appears in the parent conversation ---
// The transcript.jsonl must exist.
const transcriptPath = join(logsRoot, 'transcript.jsonl');
expect(existsSync(transcriptPath)).toBe(true);
// Load the parent transcript (all messages written by runPiece).
const loaded = Conversation.loadFrom(transcriptPath);
const loadedJson = JSON.stringify(loaded);
// The sub result must appear somewhere in the parent thread — as the
// tool_result content for the delegate tool call.
expect(loadedJson).toContain(SUB_RESULT_MARKER);
// --- (b) ISOLATION: sub-agent's own seed/intermediate turns must NOT
// appear as standalone parent messages ---
//
// The sub-agent was launched with SUB_PROMPT_MARKER as its
// taskInstruction. Inside runDelegateSubAgent, the sub's Conversation
// is seeded with a user message whose content IS SUB_PROMPT_MARKER.
// That message must NEVER appear as a user-role message in the parent
// conversation — it lives only inside the sub's isolated Conversation.
//
// IMPORTANT — what is NOT a leak:
// SUB_PROMPT_MARKER legitimately appears inside the parent's assistant
// message as the arguments to the delegate tool_call (parent's own
// message [3] above). That is correct and expected.
// The isolation check is: SUB_PROMPT_MARKER must NOT appear as the
// *content* of a parent `user` message.
const parentUserMessages = loaded.filter((m) => m.role === 'user');
const subPromptLeakedIntoParentUser = parentUserMessages.some(
(m) =>
typeof m.content === 'string' &&
m.content.includes(SUB_PROMPT_MARKER),
);
expect(subPromptLeakedIntoParentUser).toBe(false);
// The sub's system messages (its own preamble + movement guidance) must
// NOT appear in the parent's system messages either.
const parentSystemMessages = loaded.filter((m) => m.role === 'system');
const subSystemLeakedIntoParent = parentSystemMessages.some(
(m) =>
typeof m.content === 'string' &&
m.content.includes(SUB_PROMPT_MARKER),
);
expect(subSystemLeakedIntoParent).toBe(false);
// Cross-check via SpyFakeClient snapshots:
// The client was called 3 times in total:
// snapshot[0] = parent messages at turn 1 (before delegate runs)
// snapshot[1] = SUB-AGENT messages at turn 1 (sub's own Conversation)
// snapshot[2] = parent messages at turn 2 (after delegate returns)
//
// In snapshot[0] and snapshot[2] — the parent's message arrays —
// SUB_PROMPT_MARKER must NOT appear as the content of any `user` message.
// (It may appear inside an assistant tool_call's `arguments` string at
// snapshot[2], which is the parent's OWN delegate tool_call — not a leak.)
expect(client.snapshots.length).toBe(3);
for (const snapshotIndex of [0, 2]) {
const snapshot = client.snapshots[snapshotIndex]!;
const userMsgsInSnapshot = snapshot.filter((m) => m.role === 'user');
const leaked = userMsgsInSnapshot.some(
(m) =>
typeof m.content === 'string' &&
m.content.includes(SUB_PROMPT_MARKER),
);
expect(
leaked,
`SUB_PROMPT_MARKER leaked as user message in parent snapshot[${snapshotIndex}]`,
).toBe(false);
}
// Snapshot[1] is the sub-agent's own Conversation — it IS expected to
// contain SUB_PROMPT_MARKER as its task instruction (user message).
const subSnapshot = client.snapshots[1]!;
const subHasOwnInstruction = subSnapshot.some(
(m) =>
m.role === 'user' &&
typeof m.content === 'string' &&
m.content.includes(SUB_PROMPT_MARKER),
);
expect(
subHasOwnInstruction,
'Sub-agent snapshot should contain its own task instruction',
).toBe(true);
// The parent task instruction must appear in parent snapshots but NOT in
// the sub snapshot (proving the two Conversations are fully isolated).
const subHasParentInstruction = subSnapshot.some(
(m) =>
m.role === 'user' &&
typeof m.content === 'string' &&
m.content.includes(PARENT_TASK_INSTRUCTION),
);
expect(
subHasParentInstruction,
'Parent task instruction must not bleed into sub-agent snapshot',
).toBe(false);
},
);
it(
'delegate events carry a stable delegateRunId; sub internal events share same correlationId',
async () => {
const tmp = makeTmpWorkspace();
workspacePath = tmp.workspacePath;
const { logsRoot } = tmp;
getToolDefsMock.mockResolvedValue(makeToolDefs(['delegate']));
executeToolMock.mockImplementation(
async (name: string, input: Record<string, unknown>, ctx: ToolContext) => {
if (name === 'delegate') {
if (!ctx.runDelegate) {
return { output: 'runDelegate not injected', isError: true };
}
const { result, status } = await ctx.runDelegate({
description: String(input['description'] ?? 'sub-task'),
prompt: String(input['prompt'] ?? ''),
});
if (status === 'aborted') {
return { output: `[delegate aborted] ${result}`, isError: true };
}
return { output: result, isError: false };
}
return { output: 'ok', isError: false };
},
);
const client = new SpyFakeClient([
// parent turn 1 — delegate
[
{
type: 'tool_use',
id: 'del-obs-1',
name: 'delegate',
input: {
description: 'observability sub-task',
prompt: SUB_PROMPT_MARKER,
},
},
{ type: 'done', usage: { prompt_tokens: 100, completion_tokens: 10 } },
],
// sub turn 1 — complete
[
{
type: 'tool_use',
id: 'sub-obs-complete-1',
name: 'complete',
input: { status: 'success', result: SUB_RESULT_MARKER },
},
{ type: 'done', usage: { prompt_tokens: 80, completion_tokens: 5 } },
],
// parent turn 2 — complete
[
{
type: 'tool_use',
id: 'par-obs-complete-1',
name: 'complete',
input: {
status: 'success',
result: `parent finished with: ${SUB_RESULT_MARKER}`,
},
},
{ type: 'done', usage: { prompt_tokens: 120, completion_tokens: 10 } },
],
]);
const piece = delegatePiece();
await runPiece(
piece,
PARENT_TASK_INSTRUCTION,
client as never,
workspacePath,
undefined,
undefined,
// PR B: tool availability comes from the workspace policy, not the piece.
// 'delegate' must be in the resolved set for ctx.runDelegate to be wired.
{ runtimeDir: logsRoot, workspaceTools: { allowedTools: ['delegate'], editAllowed: true } },
);
const { parseEventLine: pel } = await import('../progress/event-log.js');
const eventsJsonlPath = join(logsRoot, 'events.jsonl');
const raw = readFileSync(eventsJsonlPath, 'utf-8');
const events = raw
.trim()
.split('\n')
.map((l) => pel(l))
.filter((r) => r.kind === 'ok')
// eslint-disable-next-line @typescript-eslint/no-explicit-any
.map((r: any) => r.event);
const start = events.find((e: { kind: string }) => e.kind === 'delegate_start');
expect(start, 'delegate_start event must exist').toBeTruthy();
// eslint-disable-next-line @typescript-eslint/no-explicit-any
expect(typeof (start as any).payload.delegateRunId).toBe('string');
// eslint-disable-next-line @typescript-eslint/no-explicit-any
expect((start as any).payload.parentRunId).toBeNull();
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const runId = (start as any).payload.delegateRunId as string;
const complete = events.find((e: { kind: string }) => e.kind === 'delegate_complete');
expect(complete, 'delegate_complete event must exist').toBeTruthy();
// eslint-disable-next-line @typescript-eslint/no-explicit-any
expect((complete as any).payload.delegateRunId).toBe(runId);
// eslint-disable-next-line @typescript-eslint/no-explicit-any
expect((complete as any).payload.status).toBe('success');
// Sub-internal events must carry the delegateRunId attribution tag.
// NOTE: correlationId は tool_call が per-tool UUID で上書きするため
// 帰属キーには使えない。delegateRunId フィールドで束ねるのが正。
const subEvent = events.find(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(e: any) =>
e.delegateRunId === runId &&
e.kind !== 'delegate_start' &&
e.kind !== 'delegate_complete',
);
expect(subEvent, 'at least one sub-internal event must carry delegateRunId === runId').toBeTruthy();
},
);
it(
'回帰: sub-agent の通常ツール呼び出しが reconstructDelegateRuns で toolCalls に数えられる',
async () => {
// 「ツール 0 回」バグの真のエンドツーエンド再発防止。
// sub-agent が通常ツールReadThingを呼ぶと tool-execution が
// per-tool correlationId を発行して上書きする。それでも scope の
// delegateRunId で束ねられ、toolCalls が加算されねばならない。
const tmp = makeTmpWorkspace();
workspacePath = tmp.workspacePath;
const { logsRoot } = tmp;
getToolDefsMock.mockResolvedValue(makeToolDefs(['delegate', 'ReadThing']));
executeToolMock.mockImplementation(
async (name: string, input: Record<string, unknown>, ctx: ToolContext) => {
if (name === 'delegate') {
if (!ctx.runDelegate) {
return { output: 'runDelegate not injected', isError: true };
}
const { result, status } = await ctx.runDelegate({
description: String(input['description'] ?? 'sub-task'),
prompt: String(input['prompt'] ?? ''),
});
if (status === 'aborted') {
return { output: `[delegate aborted] ${result}`, isError: true };
}
return { output: result, isError: false };
}
return { output: 'ok', isError: false };
},
);
const client = new SpyFakeClient([
// parent turn 1 — delegate
[
{ type: 'tool_use', id: 'del-tc-1', name: 'delegate', input: { description: 'read a thing', prompt: SUB_PROMPT_MARKER } },
{ type: 'done', usage: { prompt_tokens: 100, completion_tokens: 10 } },
],
// sub turn 1 — regular tool call (ReadThing)
[
{ type: 'tool_use', id: 'sub-read-1', name: 'ReadThing', input: { file_path: 'x.txt' } },
{ type: 'done', usage: { prompt_tokens: 80, completion_tokens: 5 } },
],
// sub turn 2 — complete
[
{ type: 'tool_use', id: 'sub-complete-1', name: 'complete', input: { status: 'success', result: SUB_RESULT_MARKER } },
{ type: 'done', usage: { prompt_tokens: 60, completion_tokens: 5 } },
],
// parent turn 2 — complete
[
{ type: 'tool_use', id: 'par-complete-1', name: 'complete', input: { status: 'success', result: `parent: ${SUB_RESULT_MARKER}` } },
{ type: 'done', usage: { prompt_tokens: 120, completion_tokens: 10 } },
],
]);
const piece = delegatePiece();
await runPiece(
piece,
PARENT_TASK_INSTRUCTION,
client as never,
workspacePath,
undefined,
undefined,
{ runtimeDir: logsRoot, workspaceTools: { allowedTools: ['delegate', 'ReadThing'], editAllowed: true } },
);
const { parseEventLine: pel } = await import('../progress/event-log.js');
const raw = readFileSync(join(logsRoot, 'events.jsonl'), 'utf-8');
const events = raw
.trim()
.split('\n')
.map((l) => pel(l))
.filter((r) => r.kind === 'ok')
// eslint-disable-next-line @typescript-eslint/no-explicit-any
.map((r: any) => r.event);
const runs = reconstructDelegateRuns(events);
expect(runs, 'exactly one delegate run reconstructed').toHaveLength(1);
expect(runs[0].toolCalls, 'ReadThing tool_call must be counted (not 0)').toBe(1);
expect(runs[0].toolSummary).toEqual([{ tool: 'ReadThing', count: 1 }]);
},
);
});