/** * Phase B Task 4 — E2E integration test: runPiece with delegate tool. * * Exercises the real runPiece → executeMovement → runDelegateSubAgent path. * Only tools/index.js is mocked. * * Assertions: * (a) The sub-agent's complete.result propagates back to the parent as the * delegate tool's output (appears in parent conversation + final result). * (b) ISOLATION: The sub-agent's own seed messages (system/user) and * intermediate turns do NOT appear as standalone parent conversation * messages. We use SpyFakeClient to snapshot the parent's messages array * at each LLM call — the sub's turns are never added there. * (c) The delegate tool result equals the sub's complete.result exactly. */ import { afterEach, describe, expect, it, vi } from 'vitest'; import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; import type { LLMEvent, ToolDef } from '../llm/openai-compat.js'; import type { PieceDef } from './piece-runner.js'; import type { ToolContext } from './tools/index.js'; // --------------------------------------------------------------------------- // Hoist mocks before any imports that use tools/index.js // --------------------------------------------------------------------------- const { executeToolMock, getToolDefsMock } = vi.hoisted(() => ({ executeToolMock: vi.fn(), getToolDefsMock: vi.fn(), })); vi.mock('./tools/index.js', () => ({ executeTool: executeToolMock, getToolDefs: getToolDefsMock, })); // Import after mocking import { runPiece } from './piece-runner.js'; import { Conversation } from './context/conversation.js'; import { reconstructDelegateRuns } from '../progress/delegate-runs.js'; // --------------------------------------------------------------------------- // Unique sentinel strings — must survive the full test without false positives // --------------------------------------------------------------------------- const SUB_PROMPT_MARKER = 'SUBTASK_PROMPT_MARKER_x9 analyze the data'; const SUB_RESULT_MARKER = 'SUB_RESULT_MARKER_q7'; const PARENT_TASK_INSTRUCTION = 'PARENT_TASK_E2E_DELEGATE_TEST_7f3a'; // --------------------------------------------------------------------------- // SpyFakeClient — takes snapshots of the parent messages array at each call. // Snapshots are taken at call time so later mutations don't affect them. // --------------------------------------------------------------------------- class SpyFakeClient { private index = 0; /** Deep-shallow copies of messages at the moment of each LLM call. */ readonly snapshots: Array< Array<{ role: string; content?: unknown; tool_calls?: Array<{ id: string; function: { name: string } }>; tool_call_id?: string; }> > = []; constructor(private readonly responses: LLMEvent[][]) {} async *chat( messages: unknown, _tools?: unknown, _signal?: AbortSignal, ): AsyncGenerator { // Snapshot NOW (before sub-agent appends its own messages) this.snapshots.push( (messages as Array>).map((m) => ({ ...m })) as never, ); const response = this.responses[this.index++] ?? []; for (const event of response) { yield event; } } } // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- function makeToolDefs(names: string[]): ToolDef[] { return names.map((name) => ({ type: 'function' as const, function: { name, description: name, parameters: { type: 'object', properties: {}, required: [] }, }, })); } function makeTmpWorkspace(): { workspacePath: string; logsRoot: string } { const workspacePath = mkdtempSync(join(tmpdir(), 'piece-runner-delegate-e2e-')); const logsRoot = join(workspacePath, 'logs'); mkdirSync(logsRoot, { recursive: true }); return { workspacePath, logsRoot }; } /** * A one-movement piece whose single movement has 'delegate' in allowed_tools * and no rules (so only 'complete' is offered as a terminal — no transition * tool is built). The movement calls delegate, then calls complete. */ function delegatePiece(): PieceDef { return { name: 'delegate-e2e-test-piece', description: 'E2E test piece for delegate tool', max_movements: 5, initial_movement: 'main', movements: [ { name: 'main', edit: false, persona: 'orchestrator', instruction: 'Delegate a subtask and then complete.', allowed_tools: ['delegate'], rules: [], // No default_next — complete is the only exit. }, ], }; } // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- describe('runPiece — delegate E2E (Phase B Task 4)', () => { let workspacePath = ''; afterEach(() => { executeToolMock.mockReset(); getToolDefsMock.mockReset(); if (workspacePath) { rmSync(workspacePath, { recursive: true, force: true }); workspacePath = ''; } }); it( '(a)(b)(c) sub result propagates to parent, sub turns absent from parent messages, transcript isolation', async () => { const tmp = makeTmpWorkspace(); workspacePath = tmp.workspacePath; const { logsRoot } = tmp; // getToolDefs is called for both parent and sub-agent runs. // Return 'delegate' for parent, return nothing (or complete-only) for sub // — but since the real code always adds 'complete' as a meta-tool for // sub (rules:[]), returning an empty list here is fine: getToolDefs is // only consulted for the explicit allowed_tools list. getToolDefsMock.mockResolvedValue(makeToolDefs(['delegate'])); // executeToolMock: forward 'delegate' calls to ctx.runDelegate (same // pattern as agent-loop.delegate.test.ts — this is what orchestration.ts // does in production). executeToolMock.mockImplementation( async (name: string, input: Record, ctx: ToolContext) => { if (name === 'delegate') { if (!ctx.runDelegate) { return { output: 'runDelegate not injected', isError: true }; } const { result, status } = await ctx.runDelegate({ description: String(input['description'] ?? 'sub-task'), prompt: String(input['prompt'] ?? ''), }); if (status === 'aborted') { return { output: `[delegate aborted] ${result}`, isError: true }; } return { output: result, isError: false }; } return { output: 'ok', isError: false }; }, ); // Scripted call order (single shared client — sub-agent shares the same // client instance as the parent via runDelegateSubAgent): // [0] parent turn 1 → emits delegate tool_call with SUB_PROMPT_MARKER // [1] sub turn 1 → emits complete({status:'success', result: SUB_RESULT_MARKER}) // [2] parent turn 2 → emits complete({status:'success', result: 'parent finished with: SUB_RESULT_MARKER_q7'}) const client = new SpyFakeClient([ // parent turn 1 — delegate [ { type: 'tool_use', id: 'del-e2e-1', name: 'delegate', input: { description: 'analyze data subtask', prompt: SUB_PROMPT_MARKER, }, }, { type: 'done', usage: { prompt_tokens: 100, completion_tokens: 10 } }, ], // sub turn 1 — complete (sub-agent finishes) [ { type: 'tool_use', id: 'sub-e2e-complete-1', name: 'complete', input: { status: 'success', result: SUB_RESULT_MARKER }, }, { type: 'done', usage: { prompt_tokens: 80, completion_tokens: 5 } }, ], // parent turn 2 — complete (parent references sub result) [ { type: 'tool_use', id: 'par-e2e-complete-1', name: 'complete', input: { status: 'success', result: `parent finished with: ${SUB_RESULT_MARKER}`, }, }, { type: 'done', usage: { prompt_tokens: 120, completion_tokens: 10 } }, ], ]); const piece = delegatePiece(); const pieceResult = await runPiece( piece, PARENT_TASK_INSTRUCTION, client as never, workspacePath, undefined, undefined, // PR B: tool availability comes from the workspace policy, not the piece. // 'delegate' must be in the resolved set for ctx.runDelegate to be wired. { runtimeDir: logsRoot, workspaceTools: { allowedTools: ['delegate'], editAllowed: true } }, ); // --- Basic completion --- expect(pieceResult.status).toBe('completed'); expect(pieceResult.finalOutput).toContain(SUB_RESULT_MARKER); // --- (c) delegate tool result equals sub's complete.result --- // The parent's final output must contain SUB_RESULT_MARKER (set by sub's // complete.result). This proves the result was threaded through unchanged. expect(pieceResult.finalOutput).toBe(`parent finished with: ${SUB_RESULT_MARKER}`); // --- (a) Sub result appears in the parent conversation --- // The transcript.jsonl must exist. const transcriptPath = join(logsRoot, 'transcript.jsonl'); expect(existsSync(transcriptPath)).toBe(true); // Load the parent transcript (all messages written by runPiece). const loaded = Conversation.loadFrom(transcriptPath); const loadedJson = JSON.stringify(loaded); // The sub result must appear somewhere in the parent thread — as the // tool_result content for the delegate tool call. expect(loadedJson).toContain(SUB_RESULT_MARKER); // --- (b) ISOLATION: sub-agent's own seed/intermediate turns must NOT // appear as standalone parent messages --- // // The sub-agent was launched with SUB_PROMPT_MARKER as its // taskInstruction. Inside runDelegateSubAgent, the sub's Conversation // is seeded with a user message whose content IS SUB_PROMPT_MARKER. // That message must NEVER appear as a user-role message in the parent // conversation — it lives only inside the sub's isolated Conversation. // // IMPORTANT — what is NOT a leak: // SUB_PROMPT_MARKER legitimately appears inside the parent's assistant // message as the arguments to the delegate tool_call (parent's own // message [3] above). That is correct and expected. // The isolation check is: SUB_PROMPT_MARKER must NOT appear as the // *content* of a parent `user` message. const parentUserMessages = loaded.filter((m) => m.role === 'user'); const subPromptLeakedIntoParentUser = parentUserMessages.some( (m) => typeof m.content === 'string' && m.content.includes(SUB_PROMPT_MARKER), ); expect(subPromptLeakedIntoParentUser).toBe(false); // The sub's system messages (its own preamble + movement guidance) must // NOT appear in the parent's system messages either. const parentSystemMessages = loaded.filter((m) => m.role === 'system'); const subSystemLeakedIntoParent = parentSystemMessages.some( (m) => typeof m.content === 'string' && m.content.includes(SUB_PROMPT_MARKER), ); expect(subSystemLeakedIntoParent).toBe(false); // Cross-check via SpyFakeClient snapshots: // The client was called 3 times in total: // snapshot[0] = parent messages at turn 1 (before delegate runs) // snapshot[1] = SUB-AGENT messages at turn 1 (sub's own Conversation) // snapshot[2] = parent messages at turn 2 (after delegate returns) // // In snapshot[0] and snapshot[2] — the parent's message arrays — // SUB_PROMPT_MARKER must NOT appear as the content of any `user` message. // (It may appear inside an assistant tool_call's `arguments` string at // snapshot[2], which is the parent's OWN delegate tool_call — not a leak.) expect(client.snapshots.length).toBe(3); for (const snapshotIndex of [0, 2]) { const snapshot = client.snapshots[snapshotIndex]!; const userMsgsInSnapshot = snapshot.filter((m) => m.role === 'user'); const leaked = userMsgsInSnapshot.some( (m) => typeof m.content === 'string' && m.content.includes(SUB_PROMPT_MARKER), ); expect( leaked, `SUB_PROMPT_MARKER leaked as user message in parent snapshot[${snapshotIndex}]`, ).toBe(false); } // Snapshot[1] is the sub-agent's own Conversation — it IS expected to // contain SUB_PROMPT_MARKER as its task instruction (user message). const subSnapshot = client.snapshots[1]!; const subHasOwnInstruction = subSnapshot.some( (m) => m.role === 'user' && typeof m.content === 'string' && m.content.includes(SUB_PROMPT_MARKER), ); expect( subHasOwnInstruction, 'Sub-agent snapshot should contain its own task instruction', ).toBe(true); // The parent task instruction must appear in parent snapshots but NOT in // the sub snapshot (proving the two Conversations are fully isolated). const subHasParentInstruction = subSnapshot.some( (m) => m.role === 'user' && typeof m.content === 'string' && m.content.includes(PARENT_TASK_INSTRUCTION), ); expect( subHasParentInstruction, 'Parent task instruction must not bleed into sub-agent snapshot', ).toBe(false); }, ); it( 'delegate events carry a stable delegateRunId; sub internal events share same correlationId', async () => { const tmp = makeTmpWorkspace(); workspacePath = tmp.workspacePath; const { logsRoot } = tmp; getToolDefsMock.mockResolvedValue(makeToolDefs(['delegate'])); executeToolMock.mockImplementation( async (name: string, input: Record, ctx: ToolContext) => { if (name === 'delegate') { if (!ctx.runDelegate) { return { output: 'runDelegate not injected', isError: true }; } const { result, status } = await ctx.runDelegate({ description: String(input['description'] ?? 'sub-task'), prompt: String(input['prompt'] ?? ''), }); if (status === 'aborted') { return { output: `[delegate aborted] ${result}`, isError: true }; } return { output: result, isError: false }; } return { output: 'ok', isError: false }; }, ); const client = new SpyFakeClient([ // parent turn 1 — delegate [ { type: 'tool_use', id: 'del-obs-1', name: 'delegate', input: { description: 'observability sub-task', prompt: SUB_PROMPT_MARKER, }, }, { type: 'done', usage: { prompt_tokens: 100, completion_tokens: 10 } }, ], // sub turn 1 — complete [ { type: 'tool_use', id: 'sub-obs-complete-1', name: 'complete', input: { status: 'success', result: SUB_RESULT_MARKER }, }, { type: 'done', usage: { prompt_tokens: 80, completion_tokens: 5 } }, ], // parent turn 2 — complete [ { type: 'tool_use', id: 'par-obs-complete-1', name: 'complete', input: { status: 'success', result: `parent finished with: ${SUB_RESULT_MARKER}`, }, }, { type: 'done', usage: { prompt_tokens: 120, completion_tokens: 10 } }, ], ]); const piece = delegatePiece(); await runPiece( piece, PARENT_TASK_INSTRUCTION, client as never, workspacePath, undefined, undefined, // PR B: tool availability comes from the workspace policy, not the piece. // 'delegate' must be in the resolved set for ctx.runDelegate to be wired. { runtimeDir: logsRoot, workspaceTools: { allowedTools: ['delegate'], editAllowed: true } }, ); const { parseEventLine: pel } = await import('../progress/event-log.js'); const eventsJsonlPath = join(logsRoot, 'events.jsonl'); const raw = readFileSync(eventsJsonlPath, 'utf-8'); const events = raw .trim() .split('\n') .map((l) => pel(l)) .filter((r) => r.kind === 'ok') // eslint-disable-next-line @typescript-eslint/no-explicit-any .map((r: any) => r.event); const start = events.find((e: { kind: string }) => e.kind === 'delegate_start'); expect(start, 'delegate_start event must exist').toBeTruthy(); // eslint-disable-next-line @typescript-eslint/no-explicit-any expect(typeof (start as any).payload.delegateRunId).toBe('string'); // eslint-disable-next-line @typescript-eslint/no-explicit-any expect((start as any).payload.parentRunId).toBeNull(); // eslint-disable-next-line @typescript-eslint/no-explicit-any const runId = (start as any).payload.delegateRunId as string; const complete = events.find((e: { kind: string }) => e.kind === 'delegate_complete'); expect(complete, 'delegate_complete event must exist').toBeTruthy(); // eslint-disable-next-line @typescript-eslint/no-explicit-any expect((complete as any).payload.delegateRunId).toBe(runId); // eslint-disable-next-line @typescript-eslint/no-explicit-any expect((complete as any).payload.status).toBe('success'); // Sub-internal events must carry the delegateRunId attribution tag. // NOTE: correlationId は tool_call が per-tool UUID で上書きするため // 帰属キーには使えない。delegateRunId フィールドで束ねるのが正。 const subEvent = events.find( // eslint-disable-next-line @typescript-eslint/no-explicit-any (e: any) => e.delegateRunId === runId && e.kind !== 'delegate_start' && e.kind !== 'delegate_complete', ); expect(subEvent, 'at least one sub-internal event must carry delegateRunId === runId').toBeTruthy(); }, ); it( '回帰: sub-agent の通常ツール呼び出しが reconstructDelegateRuns で toolCalls に数えられる', async () => { // 「ツール 0 回」バグの真のエンドツーエンド再発防止。 // sub-agent が通常ツール(ReadThing)を呼ぶと tool-execution が // per-tool correlationId を発行して上書きする。それでも scope の // delegateRunId で束ねられ、toolCalls が加算されねばならない。 const tmp = makeTmpWorkspace(); workspacePath = tmp.workspacePath; const { logsRoot } = tmp; getToolDefsMock.mockResolvedValue(makeToolDefs(['delegate', 'ReadThing'])); executeToolMock.mockImplementation( async (name: string, input: Record, ctx: ToolContext) => { if (name === 'delegate') { if (!ctx.runDelegate) { return { output: 'runDelegate not injected', isError: true }; } const { result, status } = await ctx.runDelegate({ description: String(input['description'] ?? 'sub-task'), prompt: String(input['prompt'] ?? ''), }); if (status === 'aborted') { return { output: `[delegate aborted] ${result}`, isError: true }; } return { output: result, isError: false }; } return { output: 'ok', isError: false }; }, ); const client = new SpyFakeClient([ // parent turn 1 — delegate [ { type: 'tool_use', id: 'del-tc-1', name: 'delegate', input: { description: 'read a thing', prompt: SUB_PROMPT_MARKER } }, { type: 'done', usage: { prompt_tokens: 100, completion_tokens: 10 } }, ], // sub turn 1 — regular tool call (ReadThing) [ { type: 'tool_use', id: 'sub-read-1', name: 'ReadThing', input: { file_path: 'x.txt' } }, { type: 'done', usage: { prompt_tokens: 80, completion_tokens: 5 } }, ], // sub turn 2 — complete [ { type: 'tool_use', id: 'sub-complete-1', name: 'complete', input: { status: 'success', result: SUB_RESULT_MARKER } }, { type: 'done', usage: { prompt_tokens: 60, completion_tokens: 5 } }, ], // parent turn 2 — complete [ { type: 'tool_use', id: 'par-complete-1', name: 'complete', input: { status: 'success', result: `parent: ${SUB_RESULT_MARKER}` } }, { type: 'done', usage: { prompt_tokens: 120, completion_tokens: 10 } }, ], ]); const piece = delegatePiece(); await runPiece( piece, PARENT_TASK_INSTRUCTION, client as never, workspacePath, undefined, undefined, { runtimeDir: logsRoot, workspaceTools: { allowedTools: ['delegate', 'ReadThing'], editAllowed: true } }, ); const { parseEventLine: pel } = await import('../progress/event-log.js'); const raw = readFileSync(join(logsRoot, 'events.jsonl'), 'utf-8'); const events = raw .trim() .split('\n') .map((l) => pel(l)) .filter((r) => r.kind === 'ok') // eslint-disable-next-line @typescript-eslint/no-explicit-any .map((r: any) => r.event); const runs = reconstructDelegateRuns(events); expect(runs, 'exactly one delegate run reconstructed').toHaveLength(1); expect(runs[0].toolCalls, 'ReadThing tool_call must be counted (not 0)').toBe(1); expect(runs[0].toolSummary).toEqual([{ tool: 'ReadThing', count: 1 }]); }, ); });