sync: update from private repo (d31b280)
CI / build-and-test (push) Has been cancelled

This commit is contained in:
oss-sync
2026-06-11 11:28:40 +00:00
parent d061ad08d8
commit 3b1645cc91
96 changed files with 4627 additions and 287 deletions
+2 -1
View File
@@ -1963,7 +1963,7 @@ export async function executeMovement(
{ role: 'user', content: taskInstruction },
];
const runIsolatedLlm = (isolatedMessages: Message[]): Promise<string> =>
runIsolatedLlmHelper(client, isolatedMessages, cancelSignal);
runIsolatedLlmHelper(client, isolatedMessages, cancelSignal, { userId: ctx.userId });
// Traceability T-1: ensure eventLogger is non-undefined for the
// duration of the movement. Production callers (piece-runner) always
@@ -2166,6 +2166,7 @@ export async function executeMovement(
},
},
`movement=${movement.name} `,
{ userId: ctx.userId },
);
const llmDurationMs = Date.now() - llmStartedAt;
let { accumulatedText } = consumed;
+5 -2
View File
@@ -4,6 +4,7 @@ import type {
ToolCall,
OpenAICompatClient,
LLMEvent,
LlmCallContext,
} from '../llm/openai-compat.js';
import { logger } from '../logger.js';
import { stripThinkingTokens } from './strip-thinking.js';
@@ -22,9 +23,10 @@ export async function runIsolatedLlm(
client: OpenAICompatClient,
messages: Message[],
cancelSignal?: AbortSignal,
context?: LlmCallContext,
): Promise<string> {
let output = '';
for await (const event of client.chat(messages, undefined, cancelSignal)) {
for await (const event of client.chat(messages, undefined, cancelSignal, context)) {
if (event.type === 'text') {
output += event.text;
continue;
@@ -107,8 +109,9 @@ export async function consumeLlmStream(
idleTimeoutMs: number,
callbacks: ConsumeStreamCallbacks = {},
contextLabel: string = '',
context?: LlmCallContext,
): Promise<ConsumedLLMResponse> {
const stream = client.chat(messages, tools, cancelSignal);
const stream = client.chat(messages, tools, cancelSignal, context);
const accumulator: ConsumedLLMResponse = {
accumulatedText: '',
pendingToolCalls: [],
+2 -1
View File
@@ -79,6 +79,7 @@ export async function classifyPiece(
pieces: PieceDescription[],
fileNames: string[],
timeoutMs: number = 8000,
userId?: string,
): Promise<string | null> {
const prompt = buildClassificationPrompt(taskText, pieces, fileNames);
logger.debug(`[piece-classifier] candidates=[${pieces.map(p => p.name).join(', ')}] textLen=${taskText.length}`);
@@ -87,7 +88,7 @@ export async function classifyPiece(
const llmCall = async (): Promise<string | null> => {
let result = '';
try {
for await (const event of client.chat(messages)) {
for await (const event of client.chat(messages, undefined, undefined, { userId })) {
if (event.type === 'text') result += event.text;
else if (event.type === 'error') return null;
else if (event.type === 'done') break;
+97 -50
View File
@@ -13,26 +13,57 @@ const validResult = {
reasoning: 'x',
};
const okResponse = {
ok: true,
json: () => Promise.resolve({
choices: [
{
message: {
tool_calls: [
{
function: {
name: 'submit_reflection',
arguments: JSON.stringify(validResult),
},
},
],
/**
* The reflection client now routes through OpenAICompatClient, which speaks
* streaming SSE. Build a fake streaming `Response` that emits the given SSE
* `data:` payloads, then `[DONE]`.
*/
function sseResponse(chunks: unknown[]): Response {
const lines = chunks.map((c) => `data: ${JSON.stringify(c)}\n\n`);
lines.push('data: [DONE]\n\n');
const encoder = new TextEncoder();
let i = 0;
return {
ok: true,
status: 200,
headers: { get: () => null },
body: {
getReader: () => ({
read: async () =>
i < lines.length
? { done: false, value: encoder.encode(lines[i++]) }
: { done: true, value: undefined },
releaseLock: () => {},
}),
},
} as unknown as Response;
}
/** A complete, valid submit_reflection tool-call stream with usage. */
function okStream(args: unknown = validResult): Response {
return sseResponse([
{
model: 'test-model',
choices: [
{
delta: { tool_calls: [{ index: 0, id: 'c1', function: { name: 'submit_reflection', arguments: JSON.stringify(args) } }] },
finish_reason: null,
},
},
],
usage: { prompt_tokens: 42, completion_tokens: 17 },
}),
};
],
},
{ choices: [{ delta: {}, finish_reason: 'tool_calls' }] },
{ choices: [], usage: { prompt_tokens: 42, completion_tokens: 17 } },
]);
}
function httpError(status: number, bodyText: string): Response {
return {
ok: false,
status,
headers: { get: () => null },
text: () => Promise.resolve(bodyText),
} as unknown as Response;
}
beforeEach(() => {
// No real backoff sleeps in tests.
@@ -45,7 +76,7 @@ afterEach(() => {
describe('callReflectionLlm', () => {
it('happy path: parses tool_call arguments and extracts token usage', async () => {
vi.stubGlobal('fetch', vi.fn().mockResolvedValue(okResponse));
vi.stubGlobal('fetch', vi.fn().mockResolvedValue(okStream()));
const result = await callReflectionLlm(cfg, 'system prompt', 'user prompt');
@@ -59,12 +90,8 @@ describe('callReflectionLlm', () => {
it('retries a 5xx (backend tool-call parse failure) and succeeds on resample', async () => {
const fetchMock = vi.fn()
.mockResolvedValueOnce({
ok: false,
status: 500,
text: () => Promise.resolve('{"error":{"message":"Failed to parse input at pos 41: <tool_call>..."}}'),
})
.mockResolvedValueOnce(okResponse);
.mockResolvedValueOnce(httpError(500, '{"error":{"message":"Failed to parse input at pos 41"}}'))
.mockResolvedValueOnce(okStream());
vi.stubGlobal('fetch', fetchMock);
const result = await callReflectionLlm(cfg, 's', 'u');
@@ -73,11 +100,7 @@ describe('callReflectionLlm', () => {
});
it('gives up after 3 attempts of persistent 5xx', async () => {
const fetchMock = vi.fn().mockResolvedValue({
ok: false,
status: 500,
text: () => Promise.resolve('parse error'),
});
const fetchMock = vi.fn().mockResolvedValue(httpError(500, 'parse error'));
vi.stubGlobal('fetch', fetchMock);
await expect(callReflectionLlm(cfg, 's', 'u')).rejects.toThrow('HTTP 500');
@@ -85,38 +108,62 @@ describe('callReflectionLlm', () => {
});
it('does NOT retry a 4xx (deterministic config error, e.g. invalid api key)', async () => {
const fetchMock = vi.fn().mockResolvedValue({
ok: false,
status: 401,
text: () => Promise.resolve('invalid api key'),
});
const fetchMock = vi.fn().mockResolvedValue(httpError(401, 'invalid api key'));
vi.stubGlobal('fetch', fetchMock);
await expect(callReflectionLlm(cfg, 's', 'u')).rejects.toThrow('HTTP 401');
expect(fetchMock).toHaveBeenCalledTimes(1);
});
it('retries when no tool_calls present, then throws after exhaustion', async () => {
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: () => Promise.resolve({ choices: [{ message: {} }] }),
});
it('retries when no tool_call present, then throws after exhaustion', async () => {
// A stream that yields only text and finishes — no submit_reflection call.
const noToolStream = () => sseResponse([
{ choices: [{ delta: { content: 'just text' }, finish_reason: 'stop' }] },
]);
const fetchMock = vi.fn().mockImplementation(async () => noToolStream());
vi.stubGlobal('fetch', fetchMock);
await expect(callReflectionLlm(cfg, 'system prompt', 'user prompt'))
.rejects.toThrow('no tool_call');
.rejects.toThrow('no submit_reflection tool_call');
expect(fetchMock).toHaveBeenCalledTimes(3);
});
it('retries malformed tool_call arguments JSON', async () => {
it('does NOT retry a budget_exhausted gateway sentinel (fail fast)', async () => {
// SSE sentinel error → client yields gatewayErrorType=budget_exhausted.
const fetchMock = vi.fn().mockResolvedValue(
sseResponse([{ error: { type: 'budget_exhausted', message: 'over quota' } }]),
);
vi.stubGlobal('fetch', fetchMock);
await expect(callReflectionLlm(cfg, 's', 'u')).rejects.toThrow('budget_exhausted');
expect(fetchMock).toHaveBeenCalledTimes(1);
});
it('does NOT retry a prompt-size preflight block (fail fast)', async () => {
// Tiny context window forces the client preflight guard to block before
// any fetch; resampling the identical prompt cannot help.
const fetchMock = vi.fn().mockResolvedValue(okStream());
vi.stubGlobal('fetch', fetchMock);
await expect(callReflectionLlm({ ...cfg, contextLimitTokens: 1 }, 'system', 'user'))
.rejects.toThrow('blocked before send');
expect(fetchMock).not.toHaveBeenCalled();
});
it('retries malformed tool_call arguments (client yields empty input)', async () => {
// First stream carries broken JSON args → client parses to {} → structural
// guard treats it as malformed → resample. Second stream is valid.
const brokenStream = () => sseResponse([
{
choices: [
{ delta: { tool_calls: [{ index: 0, id: 'c1', function: { name: 'submit_reflection', arguments: '{broken' } }] }, finish_reason: null },
],
},
{ choices: [{ delta: {}, finish_reason: 'tool_calls' }] },
]);
const fetchMock = vi.fn()
.mockResolvedValueOnce({
ok: true,
json: () => Promise.resolve({
choices: [{ message: { tool_calls: [{ function: { name: 'submit_reflection', arguments: '{broken' } }] } }],
}),
})
.mockResolvedValueOnce(okResponse);
.mockImplementationOnce(async () => brokenStream())
.mockImplementationOnce(async () => okStream());
vi.stubGlobal('fetch', fetchMock);
const result = await callReflectionLlm(cfg, 's', 'u');
+100 -37
View File
@@ -1,4 +1,6 @@
import { logger } from '../../logger.js';
import { getDefaultProviderRetryConfig } from '../../config.js';
import { OpenAICompatClient, type LLMEvent, type Message, type ToolDef } from '../../llm/openai-compat.js';
import type { ReflectionResult } from './types.js';
import { REFLECTION_TOOL_SCHEMA } from './reflection-schema.js';
@@ -6,6 +8,17 @@ export interface ReflectionLlmConfig {
endpoint: string;
model: string | undefined;
apiKey?: string;
/** True when the reflection worker routes through the AAO Gateway (proxy). */
proxy?: boolean;
/** Reflection target user — recorded as the usage owner. */
userId?: string;
/**
* Model context window in tokens. Passed to the shared client's
* prompt-size preflight guard. Reflection prompts can be large (uncapped
* memory snapshot), so use the worker's real limit rather than the
* client's conservative 32k default, which would block valid prompts.
*/
contextLimitTokens?: number;
}
export interface ReflectionLlmResult {
@@ -62,54 +75,104 @@ export async function callReflectionLlm(
throw lastErr ?? new Error('reflection LLM failed');
}
/**
* Classify an OpenAICompatClient error for the reflection resample loop.
* - HTTP 5xx (incl. tool-call parse errors on malformed model output) and
* gateway_shutdown / gateway_timeout: transient → resample.
* - HTTP 4xx (bad key / request shape), budget_exhausted / rate_limited
* (won't pass until the period resets), and the client-side
* "blocked before send" prompt-size guard: deterministic → fail fast.
* - Everything else (transport / parse / idle timeout): stochastic → resample.
*/
function classifyClientError(message: string, gatewayErrorType?: string): Error {
if (gatewayErrorType === 'budget_exhausted' || gatewayErrorType === 'rate_limited') {
return new Error(message);
}
if (gatewayErrorType === 'gateway_shutdown' || gatewayErrorType === 'gateway_timeout') {
return new RetryableLlmError(message);
}
// Client-side preflight rejection — the prompt is too large; resampling the
// identical prompt cannot help.
if (message.includes('blocked before send')) {
return new Error(message);
}
const m = /HTTP (\d{3})/.exec(message);
if (m) {
const status = Number(m[1]);
if (status >= 500) return new RetryableLlmError(message);
return new Error(message);
}
return new RetryableLlmError(message);
}
async function callOnce(
cfg: ReflectionLlmConfig,
systemPrompt: string,
userPrompt: string,
start: number,
): Promise<ReflectionLlmResult> {
const body: Record<string, unknown> = {
messages: [
{ role: 'system', content: systemPrompt },
{ role: 'user', content: userPrompt },
],
tools: [REFLECTION_TOOL_SCHEMA],
tool_choice: { type: 'function', function: { name: 'submit_reflection' } },
temperature: 0.2,
};
if (cfg.model) {
body['model'] = cfg.model;
// Route through the shared client so usage lands in the single
// per-user ledger (gateway + direct) like every other LLM call.
// maxAttempts=1: the outer callReflectionLlm loop owns resampling.
const client = new OpenAICompatClient(
cfg.endpoint,
cfg.model,
cfg.apiKey,
{ ...getDefaultProviderRetryConfig(), maxAttempts: 1 },
undefined,
cfg.contextLimitTokens, // real model window; avoid the 32k default blocking large reflection prompts
undefined,
undefined,
{ proxy: cfg.proxy === true },
);
const messages: Message[] = [
{ role: 'system', content: systemPrompt },
{ role: 'user', content: userPrompt },
];
let parsed: ReflectionResult | null = null;
let usage: { prompt_tokens: number; completion_tokens: number } | undefined;
let errorMsg: string | null = null;
let errorGatewayType: string | undefined;
for await (const event of client.chat(
messages,
[REFLECTION_TOOL_SCHEMA as unknown as ToolDef],
undefined,
{ userId: cfg.userId },
{ temperature: 0.2, toolChoice: { type: 'function', function: { name: 'submit_reflection' } } },
) as AsyncGenerator<LLMEvent>) {
if (event.type === 'tool_use') {
if (event.name === 'submit_reflection' && parsed === null) {
parsed = event.input as unknown as ReflectionResult;
}
} else if (event.type === 'done') {
usage = event.usage;
} else if (event.type === 'error') {
errorMsg = event.error;
errorGatewayType = event.gatewayErrorType;
}
}
const resp = await fetch(`${cfg.endpoint}/chat/completions`, {
method: 'POST',
headers: {
'content-type': 'application/json',
...(cfg.apiKey ? { authorization: `Bearer ${cfg.apiKey}` } : {}),
},
body: JSON.stringify(body),
});
if (!resp.ok) {
const text = await resp.text();
const msg = `reflection LLM HTTP ${resp.status}: ${text}`;
// 5xx: backend-side failure (incl. tool-call parse errors on malformed
// model output) — resample. 4xx: deterministic config error — fail fast.
if (resp.status >= 500) throw new RetryableLlmError(msg);
throw new Error(msg);
if (errorMsg !== null) {
throw classifyClientError(`reflection LLM ${errorMsg}`, errorGatewayType);
}
const data = await resp.json() as any;
const toolCall = data.choices?.[0]?.message?.tool_calls?.[0];
if (!toolCall) throw new RetryableLlmError('reflection LLM returned no tool_call');
let parsed: ReflectionResult;
try {
parsed = JSON.parse(toolCall.function.arguments) as ReflectionResult;
} catch {
throw new RetryableLlmError('reflection LLM tool_call arguments were not valid JSON');
if (parsed === null) {
throw new RetryableLlmError('reflection LLM returned no submit_reflection tool_call');
}
// The shared client swallows tool-argument JSON parse errors and yields an
// empty `{}` input. Preserve the old resample-on-malformed behaviour with a
// shallow structural check against the tool schema's required fields — a
// genuinely-empty object means the model emitted broken tool markup.
const p = parsed as unknown as Record<string, unknown>;
if (p['piece_changes'] === undefined || p['reasoning'] === undefined) {
throw new RetryableLlmError('reflection LLM tool_call arguments were malformed or incomplete');
}
return {
parsed,
tokensIn: data.usage?.prompt_tokens ?? 0,
tokensOut: data.usage?.completion_tokens ?? 0,
tokensIn: usage?.prompt_tokens ?? 0,
tokensOut: usage?.completion_tokens ?? 0,
durationMs: Date.now() - start,
raw: data,
raw: { usage },
};
}
@@ -24,6 +24,10 @@ export interface RunReflectionDeps {
* (normal task calls always send the worker's key — reflection must too).
*/
llmApiKey?: string;
/** True when the reflection worker routes through the AAO Gateway (proxy). */
llmProxy?: boolean;
/** Reflection worker's model context window (tokens) for the prompt guard. */
llmContextLimitTokens?: number;
}
export async function runReflectionJob(
@@ -86,6 +90,9 @@ export async function runReflectionJob(
endpoint: deps.llmEndpoint,
model: deps.llmModel,
apiKey: deps.llmApiKey,
proxy: deps.llmProxy === true,
userId: meta.userId,
contextLimitTokens: deps.llmContextLimitTokens,
};
let llmResult;
Binary file not shown.
+1
View File
@@ -51,6 +51,7 @@ export interface ToolsConfig {
officePdfMaxSizeMb?: number; // ReadPdf の最大ファイルサイズ (default: 10)
officePptxMaxSizeMb?: number; // ReadPPTX の最大ファイルサイズ (default: 50)
officePptxMaxUncompressedMb?: number; // ReadPPTX の ZIP 展開後サイズ上限 (default: 200)
officeMsgMaxSizeMb?: number; // ReadMsg の最大ファイルサイズ (default: 25)
webfetchScreenshot?: boolean; // WebFetch で vlmEnabled 時にスクショを添付するか (default: true)
webfetchScreenshotTimeoutMs?: number; // スクショ取得のタイムアウト (default: 15000)
}
+1
View File
@@ -52,6 +52,7 @@ const TOOL_DOC_ALIASES: Record<string, string> = {
readexcel: 'office',
readdocx: 'office',
readpptx: 'office',
readmsg: 'office',
pdftoimages: 'office',
splitexcelsheets: 'office',
splitdocxsections: 'office',
+324
View File
@@ -0,0 +1,324 @@
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { fileURLToPath } from 'url';
import {
formatAddress,
stripHtml,
selectMsgBody,
sanitizeAttachmentName,
formatMsgOutput,
assembleMsgOutput,
pickEmail,
isParsedMsgValid,
executeReadMsg,
type MsgView,
} from './msg.js';
import type { ToolContext } from './core.js';
import { executeTool as officeExecuteTool, TOOL_DEFS as OFFICE_TOOL_DEFS } from './office.js';
const FIXTURE = path.join(
path.dirname(fileURLToPath(import.meta.url)),
'__fixtures__',
'attachmentFiles.msg',
);
describe('formatAddress', () => {
it('renders name and email together', () => {
expect(formatAddress({ name: 'Alice', email: '[email protected]' })).toBe(
'Alice <[email protected]>',
);
});
it('renders name only when email is missing', () => {
expect(formatAddress({ name: 'Alice' })).toBe('Alice');
});
it('renders email only when name is missing', () => {
expect(formatAddress({ email: '[email protected]' })).toBe('[email protected]');
});
it('falls back to a placeholder when both are missing', () => {
expect(formatAddress({})).toBe('(unknown)');
});
});
describe('stripHtml', () => {
it('removes tags and decodes entities', () => {
expect(stripHtml('<p>Hello&nbsp;<b>world</b> &amp; co</p>')).toBe('Hello world & co');
});
it('drops script and style content', () => {
const html = '<style>.x{color:red}</style><p>Keep</p><script>alert(1)</script>';
expect(stripHtml(html)).toBe('Keep');
});
it('turns block boundaries into newlines', () => {
expect(stripHtml('<div>line1</div><div>line2</div>')).toBe('line1\nline2');
});
it('decodes valid numeric entities', () => {
expect(stripHtml('<p>&#65;&#x42;</p>')).toBe('AB');
});
it('does not throw on out-of-range numeric entities', () => {
expect(() => stripHtml('<p>&#999999999;</p>')).not.toThrow();
expect(stripHtml('A&#x110000;B')).toBe('A&#x110000;B');
});
});
describe('selectMsgBody', () => {
it('prefers the plain-text body', () => {
expect(selectMsgBody({ body: 'plain text', bodyHtml: '<p>html</p>' })).toEqual({
text: 'plain text',
format: 'plain',
});
});
it('falls back to stripped HTML when no plain body exists', () => {
expect(selectMsgBody({ bodyHtml: '<p>html body</p>' })).toEqual({
text: 'html body',
format: 'html',
});
});
it('reports none when no body is present', () => {
expect(selectMsgBody({})).toEqual({ text: '', format: 'none' });
});
it('decodes PidTagHtml (html) when body and bodyHtml are absent', () => {
const html = new TextEncoder().encode('<p>from pidtag</p>');
expect(selectMsgBody({ html })).toEqual({ text: 'from pidtag', format: 'html' });
});
it('prefers plain body over the PidTagHtml field', () => {
const html = new TextEncoder().encode('<p>html</p>');
expect(selectMsgBody({ body: 'plain', html })).toEqual({ text: 'plain', format: 'plain' });
});
it('falls back to PidTagHtml when bodyHtml is empty/whitespace', () => {
const html = new TextEncoder().encode('<p>pidtag body</p>');
expect(selectMsgBody({ bodyHtml: ' ', html })).toEqual({
text: 'pidtag body',
format: 'html',
});
});
});
describe('pickEmail', () => {
it('prefers a real SMTP address over a legacy EX DN', () => {
expect(pickEmail('/O=EX/OU=x/CN=alice', '[email protected]')).toBe('[email protected]');
expect(pickEmail('[email protected]', '/O=EX/OU=x/CN=alice')).toBe('[email protected]');
});
it('falls back to the EX DN when no SMTP-looking address exists', () => {
expect(pickEmail(undefined, '/O=EX/OU=x/CN=alice')).toBe('/O=EX/OU=x/CN=alice');
});
it('returns undefined when nothing usable is provided', () => {
expect(pickEmail(undefined, undefined)).toBeUndefined();
expect(pickEmail('', ' ')).toBeUndefined();
});
});
describe('isParsedMsgValid', () => {
it('accepts a parsed Outlook message', () => {
expect(isParsedMsgValid({ dataType: 'msg' })).toBe(true);
});
it('rejects an unsupported CFBF result (old .doc/.xls, corrupted compound file)', () => {
expect(isParsedMsgValid({ error: 'Unsupported file type!', dataType: null })).toBe(false);
expect(isParsedMsgValid({ dataType: null })).toBe(false);
expect(isParsedMsgValid({ dataType: 'attachment' })).toBe(false);
});
it('treats a whitespace-only plain body as empty and uses HTML', () => {
expect(selectMsgBody({ body: ' \n ', bodyHtml: '<p>real</p>' })).toEqual({
text: 'real',
format: 'html',
});
});
});
describe('sanitizeAttachmentName', () => {
it('keeps a normal filename unchanged', () => {
expect(sanitizeAttachmentName('report.pdf', 0)).toBe('report.pdf');
});
it('strips directory components to prevent path traversal', () => {
expect(sanitizeAttachmentName('../../etc/passwd', 0)).toBe('passwd');
expect(sanitizeAttachmentName('foo/bar/baz.txt', 0)).toBe('baz.txt');
expect(sanitizeAttachmentName('a\\b\\c.doc', 0)).toBe('c.doc');
});
it('removes control characters and null bytes', () => {
expect(sanitizeAttachmentName('na\x00me.txt', 0)).toBe('name.txt');
expect(sanitizeAttachmentName('tab\tname.txt', 0)).toBe('tabname.txt');
});
it('preserves spaces inside the filename', () => {
expect(sanitizeAttachmentName('my report.pdf', 0)).toBe('my report.pdf');
});
it('falls back to an indexed name when the result is empty', () => {
expect(sanitizeAttachmentName('', 2)).toBe('attachment-3');
expect(sanitizeAttachmentName('...', 0)).toBe('attachment-1');
});
});
describe('formatMsgOutput', () => {
const baseView: MsgView = {
subject: 'Quarterly report',
from: { name: 'Alice', email: '[email protected]' },
to: [{ name: 'Bob', email: '[email protected]' }],
cc: [],
date: 'Mon, 1 Jun 2026 10:00:00 +0900',
body: { text: 'See attached.', format: 'plain' },
attachments: [],
};
it('renders the header block and body', () => {
const out = formatMsgOutput(baseView);
expect(out).toContain('Subject: Quarterly report');
expect(out).toContain('From: Alice <[email protected]>');
expect(out).toContain('To: Bob <[email protected]>');
expect(out).toContain('Date: Mon, 1 Jun 2026 10:00:00 +0900');
expect(out).toContain('See attached.');
});
it('lists saved attachments with their paths and sizes', () => {
const out = formatMsgOutput({
...baseView,
attachments: [{ fileName: 'report.pdf', contentLength: 2048, savedPath: 'input/report.pdf' }],
});
expect(out).toContain('Attachments (1)');
expect(out).toContain('report.pdf');
expect(out).toContain('input/report.pdf');
expect(out).toContain('2048');
});
it('shows a skip reason for attachments that were not saved', () => {
const out = formatMsgOutput({
...baseView,
attachments: [{ fileName: 'huge.bin', skipped: 'exceeds size limit' }],
});
expect(out).toContain('huge.bin');
expect(out).toContain('exceeds size limit');
});
it('notes when the body could not be extracted', () => {
const out = formatMsgOutput({ ...baseView, body: { text: '', format: 'none' } });
expect(out).toContain('(no text body)');
});
it('omits the CC line when there are no CC recipients', () => {
expect(formatMsgOutput(baseView)).not.toContain('Cc:');
});
it('keeps the attachment list when the body is truncated to budget', () => {
const longBody = 'word '.repeat(20000);
const out = assembleMsgOutput(
{
...baseView,
body: { text: longBody, format: 'plain' },
attachments: [{ fileName: 'a.pdf', contentLength: 10, savedPath: 'input/a.pdf' }],
},
100,
'mail.msg',
);
expect(out).toContain('input/a.pdf');
expect(out).toContain('Subject: Quarterly report');
expect(out.length).toBeLessThan(longBody.length);
});
it('includes the CC line when CC recipients exist', () => {
const out = formatMsgOutput({ ...baseView, cc: [{ email: '[email protected]' }] });
expect(out).toContain('Cc: [email protected]');
});
});
describe('executeReadMsg (integration)', () => {
let workspace: string;
const ctx = (): ToolContext => ({ workspacePath: workspace, editAllowed: true });
beforeEach(() => {
workspace = fs.mkdtempSync(path.join(os.tmpdir(), 'readmsg-'));
fs.copyFileSync(FIXTURE, path.join(workspace, 'mail.msg'));
});
afterEach(() => {
fs.rmSync(workspace, { recursive: true, force: true });
});
it('extracts headers and body from a real .msg file', async () => {
const result = await executeReadMsg({ file_path: 'mail.msg' }, ctx());
expect(result.isError).toBeFalsy();
expect(result.output).toContain('Subject: attachmentFiles');
expect(result.output).toContain('From: hmailuser <[email protected]>');
expect(result.output).toContain('To: [email protected]');
expect(result.output).toContain('attachmentFiles');
});
it('saves attachments to input/ and lists them', async () => {
const result = await executeReadMsg({ file_path: 'mail.msg' }, ctx());
expect(result.output).toContain('Attachments (3)');
for (const [name, size] of [
['jpg.jpg', 726],
['png.png', 134],
['tif.tif', 664],
] as const) {
const saved = path.join(workspace, 'input', name);
expect(fs.existsSync(saved)).toBe(true);
expect(fs.statSync(saved).size).toBe(size);
expect(result.output).toContain(path.join('input', name));
}
});
it('rejects paths outside the workspace', async () => {
const result = await executeReadMsg({ file_path: '../../etc/passwd' }, ctx());
expect(result.isError).toBe(true);
});
it('reports a clear error for a non-.msg file', async () => {
fs.writeFileSync(path.join(workspace, 'junk.msg'), 'not a real msg file');
const result = await executeReadMsg({ file_path: 'junk.msg' }, ctx());
expect(result.isError).toBe(true);
expect(result.output).toContain('ReadMsg');
});
it('does not write attachments in a read-only phase', async () => {
const result = await executeReadMsg(
{ file_path: 'mail.msg' },
{ workspacePath: workspace, editAllowed: false },
);
expect(result.isError).toBeFalsy();
expect(fs.existsSync(path.join(workspace, 'input', 'jpg.jpg'))).toBe(false);
expect(result.output).toContain('read-only');
});
it('does not overwrite an existing file in input/', async () => {
fs.mkdirSync(path.join(workspace, 'input'), { recursive: true });
fs.writeFileSync(path.join(workspace, 'input', 'jpg.jpg'), 'pre-existing');
const result = await executeReadMsg({ file_path: 'mail.msg' }, ctx());
expect(fs.readFileSync(path.join(workspace, 'input', 'jpg.jpg'), 'utf8')).toBe('pre-existing');
expect(fs.existsSync(path.join(workspace, 'input', 'jpg-1.jpg'))).toBe(true);
expect(result.output).toContain('jpg-1.jpg');
});
it('rejects files exceeding the configured size limit', async () => {
const result = await executeReadMsg(
{ file_path: 'mail.msg' },
{ workspacePath: workspace, editAllowed: true, toolsConfig: { officeMsgMaxSizeMb: 0.001 } },
);
expect(result.isError).toBe(true);
expect(result.output).toMatch(/size|limit|too large/i);
});
it('is registered and routed through the office module dispatch', async () => {
expect(OFFICE_TOOL_DEFS.ReadMsg).toBeDefined();
const result = await officeExecuteTool('ReadMsg', { file_path: 'mail.msg' }, ctx());
expect(result?.isError).toBeFalsy();
expect(result?.output).toContain('Subject: attachmentFiles');
});
});
+416
View File
@@ -0,0 +1,416 @@
import * as fs from 'fs';
import * as path from 'path';
import MsgReaderImport from '@kenjiuno/msgreader';
import { ToolDef } from '../../llm/openai-compat.js';
import type { ToolContext, ToolResult } from './core.js';
import { resolveAndGuard, truncateToBudget, getToolOutputBudgetTokens } from './core.js';
import { logger } from '../../logger.js';
// CJS/ESM interop: under native Node ESM (the built dist), a default import of
// this CommonJS package resolves to the module.exports namespace object, not the
// class — so `new MsgReaderImport()` throws "is not a constructor". Vitest/tsx
// hide this via __esModule interop. Pick the real constructor for both worlds.
const MsgReader = (
typeof MsgReaderImport === 'function'
? MsgReaderImport
: (MsgReaderImport as unknown as { default: typeof MsgReaderImport }).default
) as typeof MsgReaderImport;
type MsgReaderInstance = InstanceType<typeof MsgReader>;
const DEFAULT_MSG_MAX_SIZE_MB = 25;
export interface MsgAddress {
name?: string;
email?: string;
}
export interface MsgAttachmentMeta {
fileName: string;
contentLength?: number;
/** Relative path the attachment was written to (when saved). */
savedPath?: string;
/** Reason the attachment was not saved (mutually exclusive with savedPath). */
skipped?: string;
}
export interface MsgView {
subject?: string;
from?: MsgAddress;
to: MsgAddress[];
cc: MsgAddress[];
date?: string;
body: { text: string; format: 'plain' | 'html' | 'none' };
attachments: MsgAttachmentMeta[];
}
/** Render a single address as `Name <email>`, falling back gracefully. */
export function formatAddress(a: MsgAddress): string {
const name = a.name?.trim();
const email = a.email?.trim();
if (name && email) return `${name} <${email}>`;
if (name) return name;
if (email) return email;
return '(unknown)';
}
const NAMED_ENTITIES: Record<string, string> = {
'&nbsp;': ' ',
'&amp;': '&',
'&lt;': '<',
'&gt;': '>',
'&quot;': '"',
'&#39;': "'",
'&apos;': "'",
};
// Decode a numeric character reference, preserving the original entity if the
// code point is out of range (broken email HTML must not crash the whole read).
function safeFromCodePoint(code: number, original: string): string {
if (!Number.isFinite(code) || code < 0 || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff)) {
return original;
}
try {
return String.fromCodePoint(code);
} catch {
return original;
}
}
function decodeEntities(s: string): string {
let out = s.replace(/&nbsp;|&amp;|&lt;|&gt;|&quot;|&#39;|&apos;/g, (m) => NAMED_ENTITIES[m]);
out = out.replace(/&#(\d+);/g, (m, code) => safeFromCodePoint(Number(code), m));
out = out.replace(/&#x([0-9a-fA-F]+);/g, (m, code) => safeFromCodePoint(parseInt(code, 16), m));
return out;
}
/** Convert an HTML fragment into readable plain text. */
export function stripHtml(html: string): string {
let s = html;
// Drop script/style blocks including their contents.
s = s.replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1>/gi, '');
// Treat <br> and block-level boundaries as newlines.
s = s.replace(/<br\s*\/?>/gi, '\n');
s = s.replace(/<\/(p|div|li|tr|h[1-6]|ul|ol|table|blockquote|section|article)\s*>/gi, '\n');
// Remove all remaining tags.
s = s.replace(/<[^>]+>/g, '');
s = decodeEntities(s);
// Normalize whitespace: collapse intra-line runs, trim each line, collapse blank runs.
s = s
.split('\n')
.map((line) => line.replace(/[^\S\n]+/g, ' ').trim())
.join('\n')
.replace(/\n{3,}/g, '\n\n')
.trim();
return s;
}
/** Choose the best available body text, preferring plain over HTML. */
export function selectMsgBody(fields: {
body?: string;
bodyHtml?: string;
// PidTagHtml: some HTML-only messages carry the body here as raw bytes.
html?: Uint8Array | string;
}): {
text: string;
format: 'plain' | 'html' | 'none';
} {
const plain = fields.body?.trim();
if (plain) return { text: plain, format: 'plain' };
// Try each HTML source in order; an empty/whitespace bodyHtml must not block
// the PidTagHtml fallback, so we check the stripped result of each.
const htmlSources = [
fields.bodyHtml,
fields.html != null
? typeof fields.html === 'string'
? fields.html
: Buffer.from(fields.html).toString('utf8')
: undefined,
];
for (const source of htmlSources) {
if (!source) continue;
const stripped = stripHtml(source);
if (stripped) return { text: stripped, format: 'html' };
}
return { text: '', format: 'none' };
}
/**
* Pick the most usable email address from candidates, preferring a real SMTP
* address (contains '@') over an Exchange legacy EX DN (`/O=.../CN=...`).
*/
export function pickEmail(...candidates: (string | undefined)[]): string | undefined {
const valid = candidates.map((c) => c?.trim()).filter((c): c is string => !!c);
return valid.find((c) => c.includes('@')) ?? valid[0];
}
/** Reduce an attachment name to a safe basename, never escaping the target dir. */
export function sanitizeAttachmentName(name: string, index: number): string {
// Take the last path segment across both separators (defends path traversal).
const base = name.split(/[/\\]/).pop() ?? '';
// Strip control characters and null bytes.
// eslint-disable-next-line no-control-regex
const cleaned = base.replace(/[\x00-\x1f\x7f]/g, '').trim();
// Reject names that are empty or consist only of dots/spaces.
if (!cleaned || /^[.\s]*$/.test(cleaned)) {
return `attachment-${index + 1}`;
}
return cleaned;
}
/** Find a filename that collides with neither this run nor existing files on disk. */
function resolveFreeName(dir: string, name: string, used: Set<string>): string {
const ext = path.extname(name);
const stem = name.slice(0, name.length - ext.length);
let candidate = name;
let n = 1;
while (used.has(candidate) || fs.existsSync(path.join(dir, candidate))) {
candidate = `${stem}-${n}${ext}`;
n += 1;
}
return candidate;
}
/** Build the human-readable text output for a parsed message. */
export function formatMsgOutput(view: MsgView): string {
const lines: string[] = [];
if (view.subject) lines.push(`Subject: ${view.subject}`);
if (view.from) lines.push(`From: ${formatAddress(view.from)}`);
if (view.to.length) lines.push(`To: ${view.to.map(formatAddress).join(', ')}`);
if (view.cc.length) lines.push(`Cc: ${view.cc.map(formatAddress).join(', ')}`);
if (view.date) lines.push(`Date: ${view.date}`);
const parts: string[] = [lines.join('\n')];
parts.push(view.body.format === 'none' ? '(no text body)' : view.body.text);
if (view.attachments.length) {
const attLines = [`Attachments (${view.attachments.length}):`];
for (const att of view.attachments) {
if (att.savedPath) {
const size = att.contentLength != null ? ` (${att.contentLength} bytes)` : '';
attLines.push(`- ${att.fileName}${size} -> ${att.savedPath}`);
} else {
attLines.push(`- ${att.fileName} - skipped: ${att.skipped ?? 'not saved'}`);
}
}
parts.push(attLines.join('\n'));
}
return parts.join('\n\n');
}
/**
* Assemble the final output, truncating ONLY the body to the token budget.
* Headers and the attachment list (with saved input/ paths) always survive —
* attachments are already written to disk and the caller needs their paths.
*/
export function assembleMsgOutput(view: MsgView, budgetTokens: number, sourceLabel: string): string {
const shell = formatMsgOutput({ ...view, body: { text: '', format: 'plain' } });
const reserveTokens = Math.ceil(shell.length / 4) + 64;
const bodyBudget = Math.max(500, budgetTokens - reserveTokens);
const bodyText = view.body.format === 'none' ? '' : view.body.text;
const truncatedBody = truncateToBudget(bodyText, bodyBudget, { sourceLabel }).text;
return formatMsgOutput({ ...view, body: { text: truncatedBody, format: view.body.format } });
}
/**
* msgreader's getFileData() returns `{ error: 'Unsupported file type!' }` (not a
* throw) for CFBF files that aren't Outlook messages (legacy .doc/.xls, broken
* compound files). Treat anything whose root isn't a 'msg' as a read failure.
*/
export function isParsedMsgValid(fields: { error?: string; dataType?: string | null }): boolean {
return !fields.error && fields.dataType === 'msg';
}
export const READ_MSG_DEF: ToolDef = {
type: 'function',
function: {
name: 'ReadMsg',
description:
'Read an Outlook .msg email file, extracting subject/sender/recipients/body and saving attachments to input/. 詳細は ReadToolDoc({ name: "ReadMsg" }) で取得可能。',
parameters: {
type: 'object',
properties: {
file_path: { type: 'string', description: 'Path to the .msg file' },
},
required: ['file_path'],
},
},
};
interface FieldsLike {
error?: string;
dataType?: string | null;
subject?: string;
senderName?: string;
senderEmail?: string;
senderSmtpAddress?: string;
body?: string;
bodyHtml?: string;
html?: Uint8Array;
messageDeliveryTime?: string;
clientSubmitTime?: string;
recipients?: { name?: string; email?: string; smtpAddress?: string; recipType?: string }[];
attachments?: {
fileName?: string;
fileNameShort?: string;
contentLength?: number;
innerMsgContent?: boolean;
dataType?: string | null;
}[];
}
export async function executeReadMsg(
input: Record<string, unknown>,
ctx: ToolContext,
): Promise<ToolResult> {
const filePath = String(input.file_path ?? '');
if (!filePath) {
return { output: 'ReadMsg: file_path is required', isError: true };
}
let resolved: string;
try {
resolved = resolveAndGuard(ctx.workspacePath, filePath);
} catch (e) {
return { output: `ReadMsg: ${(e as Error).message}`, isError: true };
}
// Enforce a size cap before loading the whole file into memory (matches the
// other office tools, which each guard against oversized inputs).
const maxMb =
typeof ctx.toolsConfig?.officeMsgMaxSizeMb === 'number' &&
Number.isFinite(ctx.toolsConfig.officeMsgMaxSizeMb) &&
ctx.toolsConfig.officeMsgMaxSizeMb > 0
? ctx.toolsConfig.officeMsgMaxSizeMb
: DEFAULT_MSG_MAX_SIZE_MB;
try {
const sizeMb = fs.statSync(resolved).size / 1024 / 1024;
if (sizeMb > maxMb) {
return {
output: `ReadMsg: file size ${sizeMb.toFixed(1)}MB exceeds limit of ${maxMb}MB`,
isError: true,
};
}
} catch (e) {
return { output: `ReadMsg: cannot stat file: ${(e as Error).message}`, isError: true };
}
let buffer: Buffer;
try {
buffer = fs.readFileSync(resolved);
} catch (e) {
return { output: `ReadMsg: cannot read file: ${(e as Error).message}`, isError: true };
}
// .msg is an OLE2 / CFBF compound file. Validate the magic header up front:
// MsgReader silently returns an empty result for non-CFBF data instead of throwing.
const CFBF_MAGIC = Buffer.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]);
if (buffer.length < 8 || !buffer.subarray(0, 8).equals(CFBF_MAGIC)) {
return {
output: `ReadMsg: not a valid Outlook .msg file (bad signature): ${path.basename(resolved)}`,
isError: true,
};
}
let reader: MsgReaderInstance;
let fields: FieldsLike;
try {
// Copy into a standalone ArrayBuffer (MsgReader rejects Node Buffers).
const arrayBuffer = new Uint8Array(buffer).buffer;
reader = new MsgReader(arrayBuffer);
fields = reader.getFileData() as unknown as FieldsLike;
} catch (e) {
return {
output: `ReadMsg: failed to parse .msg (is this a valid Outlook message?): ${(e as Error).message}`,
isError: true,
};
}
if (!isParsedMsgValid(fields)) {
return {
output: `ReadMsg: not a parseable Outlook message${fields.error ? ` (${fields.error})` : ''}: ${path.basename(resolved)}`,
isError: true,
};
}
const recipients = fields.recipients ?? [];
const to = recipients
.filter((r) => (r.recipType ?? 'to') === 'to')
.map((r) => ({ name: r.name, email: pickEmail(r.smtpAddress, r.email) }));
const cc = recipients
.filter((r) => r.recipType === 'cc')
.map((r) => ({ name: r.name, email: pickEmail(r.smtpAddress, r.email) }));
const inputDir = path.join(ctx.workspacePath, 'input');
const attachments: MsgAttachmentMeta[] = [];
const rawAttachments = fields.attachments ?? [];
const usedNames = new Set<string>();
rawAttachments.forEach((att, i) => {
const rawName = att.fileName || att.fileNameShort || '';
const baseName = sanitizeAttachmentName(rawName, i);
// Read-only movements (verify etc.) must not mutate the workspace.
if (!ctx.editAllowed) {
attachments.push({
fileName: baseName,
contentLength: att.contentLength,
skipped: 'read-only phase (attachment not saved)',
});
return;
}
if (att.innerMsgContent) {
attachments.push({
fileName: baseName,
contentLength: att.contentLength,
skipped: 'embedded message (open separately)',
});
return;
}
// Resolve a name that collides with neither an earlier attachment this run
// nor a file already present in input/ (user uploads, prior extractions).
const name = resolveFreeName(inputDir, baseName, usedNames);
usedNames.add(name);
try {
const data = reader.getAttachment(att as never);
fs.mkdirSync(inputDir, { recursive: true });
const dest = path.join(inputDir, name);
fs.writeFileSync(dest, Buffer.from(data.content));
attachments.push({
fileName: name,
contentLength: data.content.length,
savedPath: path.join('input', name),
});
} catch (e) {
logger.warn(`[ReadMsg] failed to save attachment ${name}: ${(e as Error).message}`);
attachments.push({
fileName: name,
contentLength: att.contentLength,
skipped: `extraction failed: ${(e as Error).message}`,
});
}
});
const view: MsgView = {
subject: fields.subject,
from: (() => {
const email = pickEmail(fields.senderSmtpAddress, fields.senderEmail);
return fields.senderName || email ? { name: fields.senderName, email } : undefined;
})(),
to,
cc,
date: fields.messageDeliveryTime || fields.clientSubmitTime,
body: selectMsgBody(fields),
attachments,
};
const output = assembleMsgOutput(view, getToolOutputBudgetTokens(ctx), path.basename(resolved));
logger.info(
`[ReadMsg] ${path.basename(resolved)}: attachments=${attachments.length} bodyFormat=${view.body.format}`,
);
return { output, isError: false };
}
+4
View File
@@ -10,6 +10,7 @@ import { ToolDef } from '../../llm/openai-compat.js';
import type { ToolContext, ToolResult } from './core.js';
import { resolveAndGuard, resolveOutputPathWithin, truncateToBudget, getToolOutputBudgetTokens } from './core.js';
import { resolveThemePalette, extractSheetStyles } from './excel-styles.js';
import { READ_MSG_DEF, executeReadMsg } from './msg.js';
import { logger } from '../../logger.js';
import { callVisionModel, resolveImagePath } from './image.js';
import type {
@@ -351,6 +352,7 @@ export const TOOL_DEFS: Record<string, ToolDef> = {
ReadDocx: READ_DOCX_DEF,
ReadPdf: READ_PDF_DEF,
ReadPPTX: READ_PPTX_DEF,
ReadMsg: READ_MSG_DEF,
SplitExcelSheets: SPLIT_EXCEL_SHEETS_DEF,
SplitDocxSections: SPLIT_DOCX_SECTIONS_DEF,
PdfToImages: PDF_TO_IMAGES_DEF,
@@ -2284,6 +2286,8 @@ export async function executeTool(
return executeReadPdf(input, ctx);
case 'ReadPPTX':
return executeReadPptx(input, ctx);
case 'ReadMsg':
return executeReadMsg(input, ctx);
case 'SplitExcelSheets':
return executeSplitExcelSheets(input, ctx);
case 'SplitDocxSections':