This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
import type { IncomingMessage, Server as HttpServer } from 'node:http';
|
||||
import type { Server as HttpsServer } from 'node:https';
|
||||
import type { Socket } from 'node:net';
|
||||
import { WebSocketServer, type WebSocket } from 'ws';
|
||||
import { Router, json, type Request, type Response } from 'express';
|
||||
@@ -81,7 +82,9 @@ const PATH_RE = /^\/+api\/local\/tasks\/([^/]+)\/console\/ws$/;
|
||||
* (the client gets a 1006 abnormal close) so we don't leak failure
|
||||
* reasons over the upgrade channel. The reason is always logged.
|
||||
*/
|
||||
export function attachConsoleWs(server: HttpServer, deps: ConsoleWsDeps): void {
|
||||
// Both http.Server and https.Server emit the 'upgrade' event used for WSS,
|
||||
// so either type is a valid host for the console WebSocket upgrade handler.
|
||||
export function attachConsoleWs(server: HttpServer | HttpsServer, deps: ConsoleWsDeps): void {
|
||||
const wss = new WebSocketServer({ noServer: true });
|
||||
|
||||
server.on('upgrade', async (req, socket, head) => {
|
||||
|
||||
@@ -37,7 +37,11 @@ function makeUser(overrides: Partial<Express.User> = {}): Express.User {
|
||||
};
|
||||
}
|
||||
|
||||
function makeApp(repo: Repository, user?: Express.User): express.Application {
|
||||
function makeApp(
|
||||
repo: Repository,
|
||||
user?: Express.User,
|
||||
opts: { authActive?: boolean } = {},
|
||||
): express.Application {
|
||||
const app = express();
|
||||
if (user) {
|
||||
app.use((req, _res, next) => {
|
||||
@@ -45,7 +49,7 @@ function makeApp(repo: Repository, user?: Express.User): express.Application {
|
||||
next();
|
||||
});
|
||||
}
|
||||
mountLocalFilesApi(app, repo);
|
||||
mountLocalFilesApi(app, repo, { authActive: opts.authActive ?? true });
|
||||
return app;
|
||||
}
|
||||
|
||||
@@ -55,6 +59,7 @@ beforeEach(() => {
|
||||
mkdirSync(join(ws, 'output', 'sub'), { recursive: true });
|
||||
writeFileSync(join(ws, 'input', 'data.csv'), 'a,b\n1,2\n');
|
||||
writeFileSync(join(ws, 'output', 'report.md'), '# report');
|
||||
writeFileSync(join(ws, 'output', 'report.html'), '<!doctype html><script>window.__ran = true</script><h1>report</h1>');
|
||||
writeFileSync(join(ws, 'output', 'sub', 'nested.txt'), 'nested');
|
||||
// A file just outside the workspace that traversal must never reach.
|
||||
writeFileSync(join(ws, '..', `outside-${process.pid}.txt`), 'secret');
|
||||
@@ -153,6 +158,80 @@ describe('GET /api/local/tasks/:taskId/files/raw', () => {
|
||||
expect(res.headers['content-type']).toContain('markdown');
|
||||
});
|
||||
|
||||
it('sandboxes raw HTML by default', async () => {
|
||||
const res = await request(makeApp(makeRepo(), makeUser()))
|
||||
.get('/api/local/tasks/1/files/raw?section=output&path=report.html');
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers['content-security-policy']).toBe('sandbox');
|
||||
});
|
||||
|
||||
it('allows the task OWNER to open trusted raw HTML without the sandbox header', async () => {
|
||||
const res = await request(makeApp(makeRepo(), makeUser()))
|
||||
.get('/api/local/tasks/1/files/raw?section=output&path=report.html&trusted=1');
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers['content-security-policy']).toBeUndefined();
|
||||
expect(res.headers['content-type']).toContain('html');
|
||||
});
|
||||
|
||||
it("keeps trusted raw HTML sandboxed even for an admin on another user's task (user→admin lure)", async () => {
|
||||
const res = await request(makeApp(makeRepo(), makeUser({ id: 'admin-9', role: 'admin' })))
|
||||
.get('/api/local/tasks/1/files/raw?section=output&path=report.html&trusted=1');
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers['content-security-policy']).toBe('sandbox');
|
||||
});
|
||||
|
||||
it('keeps trusted raw HTML sandboxed for a NON-owner viewer of a shared task', async () => {
|
||||
const repo = makeRepo({
|
||||
getLocalTask: vi.fn().mockResolvedValue({
|
||||
id: 1,
|
||||
ownerId: 'user-1',
|
||||
visibility: 'public',
|
||||
workspacePath: ws,
|
||||
}),
|
||||
} as Partial<Repository>);
|
||||
const res = await request(makeApp(repo, makeUser({ id: 'user-2' })))
|
||||
.get('/api/local/tasks/1/files/raw?section=output&path=report.html&trusted=1');
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers['content-security-policy']).toBe('sandbox');
|
||||
});
|
||||
|
||||
it('keeps trusted raw HTML sandboxed for an ownerless task even when authenticated', async () => {
|
||||
const repo = makeRepo({
|
||||
getLocalTask: vi.fn().mockResolvedValue({
|
||||
id: 1,
|
||||
ownerId: null,
|
||||
visibility: 'public',
|
||||
workspacePath: ws,
|
||||
}),
|
||||
} as Partial<Repository>);
|
||||
const res = await request(makeApp(repo, makeUser()))
|
||||
.get('/api/local/tasks/1/files/raw?section=output&path=report.html&trusted=1');
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers['content-security-policy']).toBe('sandbox');
|
||||
});
|
||||
|
||||
it('keeps trusted raw HTML sandboxed when auth is on but no user is present', async () => {
|
||||
const res = await request(makeApp(makeRepo()))
|
||||
.get('/api/local/tasks/1/files/raw?section=output&path=report.html&trusted=1');
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers['content-security-policy']).toBe('sandbox');
|
||||
});
|
||||
|
||||
it('allows trusted raw HTML in no-auth mode (sole operator owns every task)', async () => {
|
||||
const res = await request(makeApp(makeRepo(), undefined, { authActive: false }))
|
||||
.get('/api/local/tasks/1/files/raw?section=output&path=report.html&trusted=1');
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers['content-security-policy']).toBeUndefined();
|
||||
expect(res.headers['content-type']).toContain('html');
|
||||
});
|
||||
|
||||
it('still sandboxes non-HTML in no-auth mode even with trusted=1', async () => {
|
||||
const res = await request(makeApp(makeRepo(), undefined, { authActive: false }))
|
||||
.get('/api/local/tasks/1/files/raw?section=output&path=report.md&trusted=1');
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers['content-security-policy']).toBe('sandbox');
|
||||
});
|
||||
|
||||
it('rejects traversal reads with 400 and never serves outside files', async () => {
|
||||
const res = await request(makeApp(makeRepo(), makeUser()))
|
||||
.get(`/api/local/tasks/1/files/raw?section=input&path=..%2F..%2Foutside-${process.pid}.txt`);
|
||||
|
||||
@@ -6,7 +6,20 @@ import { logger } from '../logger.js';
|
||||
import { parseTaskId } from './validation.js';
|
||||
import { ensurePathWithin, isPathEscapeError, serializeLocalFileEntry, checkTaskOwnership, canViewTask, setUntrustedFileResponseHeaders } from './local-api-helpers.js';
|
||||
|
||||
export function mountLocalFilesApi(app: Application, repo: Repository): void {
|
||||
export interface LocalFilesApiOptions {
|
||||
/** Whether the auth subsystem is wired. When false (no-auth single-user
|
||||
* deployment) there is no req.user, so the sole local operator owns every
|
||||
* task and is allowed to open their own generated HTML with trusted=1.
|
||||
* Defaults to true (owner identity comes from req.user). */
|
||||
authActive?: boolean;
|
||||
}
|
||||
|
||||
export function mountLocalFilesApi(
|
||||
app: Application,
|
||||
repo: Repository,
|
||||
opts: LocalFilesApiOptions = {},
|
||||
): void {
|
||||
const authActive = opts.authActive ?? true;
|
||||
|
||||
app.get('/api/local/tasks/:taskId/files', async (req: Request, res: Response) => {
|
||||
try {
|
||||
@@ -125,7 +138,23 @@ export function mountLocalFilesApi(app: Application, repo: Repository): void {
|
||||
res.status(400).json({ error: 'path must point to a file' });
|
||||
return;
|
||||
}
|
||||
setUntrustedFileResponseHeaders(res);
|
||||
// trusted=1 drops the CSP sandbox so the owner's own generated HTML can
|
||||
// run on the app origin. STRICTLY owner-only — self-XSS at worst:
|
||||
// - org/public visibility lets other users VIEW the task, but serving
|
||||
// someone else's HTML unsandboxed here would be stored XSS against
|
||||
// the viewer;
|
||||
// - admins are excluded too: another user's HTML running in an ADMIN
|
||||
// session would be a user→admin privilege-escalation lure.
|
||||
// No-auth single-user mode has no req.user; the sole operator owns every
|
||||
// task, so they are the owner for this purpose (self-XSS only — there is
|
||||
// no second principal to attack).
|
||||
const trustedAllowed = authActive
|
||||
? !!viewer && task.ownerId != null && viewer.id === task.ownerId
|
||||
: true;
|
||||
const trustedHtml = req.query.trusted === '1' && /\.html?$/i.test(filePath) && trustedAllowed;
|
||||
if (!trustedHtml) {
|
||||
setUntrustedFileResponseHeaders(res);
|
||||
}
|
||||
res.type(extname(filePath) || 'application/octet-stream');
|
||||
res.send(readFileSync(filePath));
|
||||
} catch (err) {
|
||||
|
||||
@@ -7,6 +7,7 @@ import { tmpdir } from 'os';
|
||||
import { Repository, localTaskRepoName } from '../db/repository.js';
|
||||
import { BrowserSessionRepo } from '../db/browser-session-repo.js';
|
||||
import { mountLocalTasksApi } from './local-tasks-api.js';
|
||||
import { buildLocalConversationContext } from '../engine/local-context.js';
|
||||
|
||||
describe('POST /api/local/tasks with visibility', () => {
|
||||
let tempDir = '';
|
||||
@@ -888,6 +889,41 @@ describe('POST /api/local/tasks/:id/continue', () => {
|
||||
expect(handoff?.body).toContain('ssh-ops');
|
||||
});
|
||||
|
||||
it('persists the switch instruction as the latest user request so the agent follows it', async () => {
|
||||
const { task } = await setupTaskWithTerminalJob();
|
||||
const res = await request(app)
|
||||
.post(`/api/local/tasks/${task.id}/continue`)
|
||||
.send({ piece: 'ssh-ops', instruction: 'use output/manual.md to set up foo' });
|
||||
expect(res.status).toBe(201);
|
||||
const comments = await repo.listLocalTaskComments(task.id);
|
||||
// The switch text must exist as a user 'request' comment...
|
||||
const userRequests = comments.filter((c) => c.author === 'user' && c.kind === 'request');
|
||||
const switchComment = userRequests.find((c) => c.body === 'use output/manual.md to set up foo');
|
||||
expect(switchComment).toBeTruthy();
|
||||
// ...and be the LATEST user instruction (newer than the original 'b' body and
|
||||
// the prior agent result), which is what buildLocalConversationContext keys on.
|
||||
const userInstructionKinds = ['comment', 'request', 'interjection'];
|
||||
const latestUserInstruction = [...comments]
|
||||
.reverse()
|
||||
.find((c) => c.author === 'user' && userInstructionKinds.includes(c.kind));
|
||||
expect(latestUserInstruction?.body).toBe('use output/manual.md to set up foo');
|
||||
|
||||
// End-to-end: feeding the resulting comments + the continued job's
|
||||
// instruction into the worker's context builder must put the switch text
|
||||
// under the active "## タスク" heading, NOT the demoted
|
||||
// "## オリジナルタスク (参考、対応済みの可能性あり)" slot that caused the
|
||||
// agent to re-follow earlier instructions.
|
||||
const ctx = buildLocalConversationContext({
|
||||
comments,
|
||||
jobInstruction: 'use output/manual.md to set up foo',
|
||||
inputFiles: [],
|
||||
outputFiles: [],
|
||||
});
|
||||
expect(ctx).toContain('## タスク');
|
||||
expect(ctx).toContain('use output/manual.md to set up foo');
|
||||
expect(ctx).not.toContain('## オリジナルタスク');
|
||||
});
|
||||
|
||||
it('returns 409 job_in_progress when prev job is running', async () => {
|
||||
const { task } = await setupTaskWithTerminalJob({ status: 'running' });
|
||||
const res = await request(app)
|
||||
|
||||
@@ -8,11 +8,12 @@ import { resolveJobScheduling } from '../scheduling.js';
|
||||
import { parseTaskId, validateCreateTaskBody, validateCommentBody, validateFeedbackBody } from './validation.js';
|
||||
import { getLocalWorkspacePath, checkTaskOwnership, canViewTask } from './local-api-helpers.js';
|
||||
import { jobEventBus, type JobStreamEvent } from './job-events.js';
|
||||
import { buildTitleFallback } from '../title-generation.js';
|
||||
|
||||
export interface LocalTasksApiOptions {
|
||||
repo: Repository;
|
||||
worktreeDir?: string;
|
||||
generateTitle?: (body: string) => Promise<string>;
|
||||
generateTitle?: (body: string, ownerId?: string) => Promise<string>;
|
||||
selectPiece?: (body: string, fileNames: string[], userId?: string) => Promise<string>;
|
||||
/**
|
||||
* Server-side validator for piece names accepted by the
|
||||
@@ -126,28 +127,23 @@ export function mountLocalTasksApi(app: Application, opts: LocalTasksApiOptions)
|
||||
browserSessionProfileId = n;
|
||||
}
|
||||
|
||||
let taskTitle = (body.title ?? '').trim();
|
||||
const userTitle = (body.title ?? '').trim();
|
||||
const rawPiece = (body.piece ?? 'auto').trim();
|
||||
const attachmentNames = (body.attachments ?? []).map((a: { name?: string }) => a.name).filter(Boolean) as string[];
|
||||
|
||||
// タイトル生成と piece 分類を並列実行
|
||||
const [generatedTitle, autoSelectedPiece] = await Promise.all([
|
||||
// タイトル生成
|
||||
(!taskTitle && opts.generateTitle)
|
||||
? Promise.race([
|
||||
opts.generateTitle(body.body.trim()),
|
||||
new Promise<string>((_, reject) => setTimeout(() => reject(new Error('timeout')), 8000)),
|
||||
]).catch((e: unknown) => { logger.warn(`Title generation failed: ${e}`); return ''; })
|
||||
: Promise.resolve(''),
|
||||
// piece 分類('auto' の場合のみ); userId を渡し per-user カタログを使用
|
||||
(rawPiece === 'auto' && opts.selectPiece)
|
||||
? opts.selectPiece(body.body.trim(), attachmentNames, (req.user as Express.User | undefined)?.id).catch((e: unknown) => { logger.warn(`Piece classification failed: ${e}`); return 'chat'; })
|
||||
: Promise.resolve(rawPiece),
|
||||
]);
|
||||
// Title is NOT generated by an LLM at creation time anymore — that fired a
|
||||
// second concurrent LLM request per task and churned gateway backend
|
||||
// slots. Instead we set a cheap synchronous fallback now, and the agent
|
||||
// upgrades it during the run by deriving from the Mission Brief goal
|
||||
// (see Repository.updateMissionBriefSync). On-demand AI regeneration is
|
||||
// available via POST /api/local/tasks/:id/regenerate-title.
|
||||
const autoSelectedPiece = (rawPiece === 'auto' && opts.selectPiece)
|
||||
? await opts.selectPiece(body.body.trim(), attachmentNames, (req.user as Express.User | undefined)?.id)
|
||||
.catch((e: unknown) => { logger.warn(`Piece classification failed: ${e}`); return 'chat'; })
|
||||
: rawPiece;
|
||||
|
||||
if (!taskTitle) {
|
||||
taskTitle = generatedTitle || body.body.trim().slice(0, 40).replace(/\n/g, ' ');
|
||||
}
|
||||
const taskTitle = userTitle || buildTitleFallback(body.body.trim());
|
||||
const titleSource: 'auto' | 'user' = userTitle ? 'user' : 'auto';
|
||||
const piece = autoSelectedPiece;
|
||||
const profile = body.profile ?? 'auto';
|
||||
const outputFormat = body.outputFormat ?? 'markdown';
|
||||
@@ -168,6 +164,7 @@ export function mountLocalTasksApi(app: Application, opts: LocalTasksApiOptions)
|
||||
|
||||
const task = await repo.createLocalTask({
|
||||
title: taskTitle,
|
||||
titleSource,
|
||||
body: body.body.trim(),
|
||||
pieceName: piece,
|
||||
profile,
|
||||
@@ -406,7 +403,18 @@ export function mountLocalTasksApi(app: Application, opts: LocalTasksApiOptions)
|
||||
const task = await repo.getLocalTask(taskId, { viewer: req.user as Express.User | undefined });
|
||||
if (!checkTaskOwnership(req, res, task)) return;
|
||||
|
||||
const updates: { visibility?: 'private' | 'org' | 'public'; visibilityScopeOrgId?: string | null } = {};
|
||||
const updates: { title?: string; titleSource?: 'user'; visibility?: 'private' | 'org' | 'public'; visibilityScopeOrgId?: string | null } = {};
|
||||
if (req.body.title !== undefined) {
|
||||
if (typeof req.body.title !== 'string') {
|
||||
res.status(400).json({ error: 'title must be a string' }); return;
|
||||
}
|
||||
const trimmed = req.body.title.trim();
|
||||
if (!trimmed) { res.status(400).json({ error: 'title must not be empty' }); return; }
|
||||
if (trimmed.length > 200) { res.status(400).json({ error: 'title must be 200 characters or less' }); return; }
|
||||
// Manual edit pins the title: the agent never auto-overwrites a user title.
|
||||
updates.title = trimmed;
|
||||
updates.titleSource = 'user';
|
||||
}
|
||||
if (req.body.visibility !== undefined) {
|
||||
const v = req.body.visibility;
|
||||
if (!['private', 'org', 'public'].includes(v)) {
|
||||
@@ -443,6 +451,41 @@ export function mountLocalTasksApi(app: Application, opts: LocalTasksApiOptions)
|
||||
}
|
||||
});
|
||||
|
||||
// On-demand AI title regeneration. Unlike the old creation-time path this
|
||||
// only fires when the user explicitly asks (a button), so it never adds a
|
||||
// concurrent LLM request to the task-creation hot path. Owner/admin only.
|
||||
app.post('/api/local/tasks/:taskId/regenerate-title', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const taskId = parseTaskId(req.params.taskId);
|
||||
if (taskId === null) { res.status(400).json({ error: 'Invalid task ID' }); return; }
|
||||
const task = await repo.getLocalTask(taskId, { viewer: req.user as Express.User | undefined });
|
||||
if (!checkTaskOwnership(req, res, task)) return;
|
||||
if (!opts.generateTitle) { res.status(503).json({ error: 'Title generation is not configured' }); return; }
|
||||
|
||||
let title = '';
|
||||
try {
|
||||
title = await Promise.race([
|
||||
// Ownerless (no-auth) tasks attribute to 'local', matching the
|
||||
// worker/piece-runner convention (ownerId ?? 'local').
|
||||
opts.generateTitle(task!.body, task!.ownerId ?? 'local'),
|
||||
new Promise<string>((_, reject) => setTimeout(() => reject(new Error('timeout')), 8000)),
|
||||
]);
|
||||
} catch (e) {
|
||||
logger.warn(`Title regeneration failed (task=${taskId}): ${e}`);
|
||||
res.status(502).json({ error: 'Title generation failed' }); return;
|
||||
}
|
||||
// Empty model output is not an error: fall back to the cheap synchronous
|
||||
// title so the button always yields something (matching the old creation
|
||||
// path's behaviour).
|
||||
title = (title ?? '').trim() || buildTitleFallback(task!.body);
|
||||
await repo.updateLocalTask(taskId, { title, titleSource: 'agent' });
|
||||
res.json({ title });
|
||||
} catch (err) {
|
||||
logger.error(`Regenerate title API error: ${err}`);
|
||||
res.status(500).json({ error: 'Failed to regenerate title' });
|
||||
}
|
||||
});
|
||||
|
||||
app.delete('/api/local/tasks/:taskId', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const taskId = parseTaskId(req.params.taskId);
|
||||
@@ -556,6 +599,15 @@ export function mountLocalTasksApi(app: Application, opts: LocalTasksApiOptions)
|
||||
|
||||
await repo.updateLocalTask(taskId, { pieceName: piece });
|
||||
|
||||
// Persist the switch-time instruction as a user request. Without this it
|
||||
// lives only in job.instruction, and buildLocalConversationContext picks
|
||||
// the *latest user comment* as the current instruction — so a stale older
|
||||
// comment would win and the switch text would be demoted to the "original
|
||||
// task (possibly already handled)" slot, making the agent re-follow prior
|
||||
// instructions instead of the new one. Mirrors the create path, which
|
||||
// also persists the body as a 'request' comment.
|
||||
await repo.addLocalTaskComment(taskId, 'user', instruction.trim(), 'request');
|
||||
|
||||
// Surface the handoff in the timeline so the user (and the LLM, when
|
||||
// it later inspects task comments) can see when piece switches happened.
|
||||
await repo.addLocalTaskComment(
|
||||
|
||||
@@ -6,6 +6,7 @@ import { fileURLToPath } from 'url';
|
||||
import { existsSync } from 'fs';
|
||||
import express from 'express';
|
||||
import type { Server } from 'http';
|
||||
import type { Server as HttpsServer } from 'https';
|
||||
import type { SessionManager, BrowserSession } from '../engine/browser-session.js';
|
||||
import type { UpgradeAuthChecker } from './auth.js';
|
||||
import { logger } from '../logger.js';
|
||||
@@ -78,7 +79,9 @@ export function createNovncRouter(): Router {
|
||||
* - authenticateUpgrade 未設定 (dev モード) は session 存在確認だけで通す
|
||||
*/
|
||||
export function setupNovncWebSocketProxy(
|
||||
server: Server,
|
||||
// Both http.Server and https.Server emit the 'upgrade' event used for WSS,
|
||||
// so either type works here as a WebSocket proxy host.
|
||||
server: Server | HttpsServer,
|
||||
getSessionManager: () => SessionManager | null,
|
||||
authenticateUpgrade?: UpgradeAuthChecker,
|
||||
authorizeSession?: NovncSessionAuthorizer,
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
import { describe, it, expect, afterEach } from 'vitest';
|
||||
import { createServer as createHttpsServer, type Server } from 'https';
|
||||
import { request as httpsRequest } from 'https';
|
||||
import { mkdtempSync, rmSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { resolveTlsOptions } from '../net/tls-options.js';
|
||||
import { SERVER_TLS_DEFAULTS } from '../server/config.js';
|
||||
|
||||
describe('native HTTPS listener (self-signed)', () => {
|
||||
let server: Server | undefined;
|
||||
let dir: string | undefined;
|
||||
afterEach(async () => {
|
||||
if (server) await new Promise<void>((r) => server!.close(() => r()));
|
||||
if (dir) rmSync(dir, { recursive: true, force: true });
|
||||
server = undefined;
|
||||
dir = undefined;
|
||||
});
|
||||
|
||||
it('completes a TLS>=1.2 handshake and serves the app over https', { timeout: 15000 }, async () => {
|
||||
dir = mkdtempSync(join(tmpdir(), 'tls-listener-'));
|
||||
const resolved = resolveTlsOptions({ ...SERVER_TLS_DEFAULTS, enabled: true, selfSignedDir: dir });
|
||||
server = createHttpsServer(
|
||||
{ cert: resolved.cert, key: resolved.key, minVersion: resolved.minVersion },
|
||||
(_req, res) => {
|
||||
res.writeHead(200);
|
||||
res.end('ok');
|
||||
},
|
||||
);
|
||||
await new Promise<void>((r) => server!.listen(0, '127.0.0.1', r));
|
||||
const addr = server.address();
|
||||
const port = typeof addr === 'object' && addr ? addr.port : 0;
|
||||
|
||||
const result = await new Promise<{ code: number; body: string; proto: string | null }>(
|
||||
(resolve, reject) => {
|
||||
const req = httpsRequest(
|
||||
{ host: '127.0.0.1', port, path: '/', rejectUnauthorized: false },
|
||||
(res) => {
|
||||
// Capture protocol before the socket is torn down (socket may be
|
||||
// null by the time 'end' fires so we snapshot it on 'response').
|
||||
const proto = (res.socket as import('tls').TLSSocket | null)?.getProtocol?.() ?? null;
|
||||
let d = '';
|
||||
res.on('data', (c) => (d += c));
|
||||
res.on('end', () =>
|
||||
resolve({ code: res.statusCode ?? 0, body: d, proto }),
|
||||
);
|
||||
},
|
||||
);
|
||||
req.on('error', reject);
|
||||
req.end();
|
||||
},
|
||||
);
|
||||
expect(result.code).toBe(200);
|
||||
expect(result.body).toBe('ok');
|
||||
expect(['TLSv1.2', 'TLSv1.3']).toContain(result.proto);
|
||||
});
|
||||
|
||||
it('a strict client rejects the self-signed cert', async () => {
|
||||
dir = mkdtempSync(join(tmpdir(), 'tls-listener-strict-'));
|
||||
const resolved = resolveTlsOptions({ ...SERVER_TLS_DEFAULTS, enabled: true, selfSignedDir: dir });
|
||||
server = createHttpsServer(
|
||||
{ cert: resolved.cert, key: resolved.key, minVersion: resolved.minVersion },
|
||||
(_req, res) => {
|
||||
res.writeHead(200);
|
||||
res.end('ok');
|
||||
},
|
||||
);
|
||||
await new Promise<void>((r) => server!.listen(0, '127.0.0.1', r));
|
||||
const addr = server.address();
|
||||
const port = typeof addr === 'object' && addr ? addr.port : 0;
|
||||
const outcome = await new Promise<string>((resolve) => {
|
||||
const req = httpsRequest(
|
||||
{ host: '127.0.0.1', port, path: '/', rejectUnauthorized: true },
|
||||
() => resolve('UNEXPECTED_OK'),
|
||||
);
|
||||
req.on('error', (e) => resolve('rejected:' + (e as NodeJS.ErrnoException).code));
|
||||
req.end();
|
||||
});
|
||||
expect(outcome).toMatch(/^rejected:/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,26 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { computeEffectiveSecureCookie, shouldWarnDoubleTls } from './server.js';
|
||||
|
||||
describe('computeEffectiveSecureCookie', () => {
|
||||
it('is true when secure_cookie is on (proxy mode)', () => {
|
||||
expect(computeEffectiveSecureCookie(true, false)).toBe(true);
|
||||
});
|
||||
it('is true when native TLS is on even if secure_cookie is off', () => {
|
||||
expect(computeEffectiveSecureCookie(false, true)).toBe(true);
|
||||
});
|
||||
it('is false when neither', () => {
|
||||
expect(computeEffectiveSecureCookie(false, false)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('shouldWarnDoubleTls', () => {
|
||||
it('warns when native TLS and secure_cookie (proxy signal) are both on', () => {
|
||||
expect(shouldWarnDoubleTls(true, true)).toBe(true);
|
||||
});
|
||||
it('does not warn for a plain native-TLS install (secure_cookie off)', () => {
|
||||
expect(shouldWarnDoubleTls(true, false)).toBe(false);
|
||||
});
|
||||
it('does not warn when TLS is disabled', () => {
|
||||
expect(shouldWarnDoubleTls(false, true)).toBe(false);
|
||||
});
|
||||
});
|
||||
+112
-14
@@ -16,6 +16,7 @@ import { mountBrandingApi, resolveBranding } from './branding-api.js';
|
||||
import { createBrowserApi } from './browser-api.js';
|
||||
import { createBrowserSessionApi } from './browser-session-api.js';
|
||||
import { createSubtaskActivityRouter } from './subtask-activity-api.js';
|
||||
import { createUsageRouter } from './usage-api.js';
|
||||
import { SessionManager } from '../engine/browser-session.js';
|
||||
import { createNovncRouter, setupNovncWebSocketProxy } from './novnc-proxy.js';
|
||||
import { setSessionManager } from '../engine/tools/browser.js';
|
||||
@@ -104,6 +105,11 @@ import { createNotesApi } from './notes-api.js';
|
||||
import { mountGateway, type GatewayMountHandle } from './gateway-mount.js';
|
||||
import { readGatewayConfig } from '../gateway/config.js';
|
||||
import { createAdminGatewayStatusRouter } from './admin-gateway-status-api.js';
|
||||
import { createServer as createHttpsServer } from 'https';
|
||||
import { X509Certificate } from 'crypto';
|
||||
import { mergeServerConfig } from '../server/config.js';
|
||||
import { resolveTlsOptions } from '../net/tls-options.js';
|
||||
import { createHttpRedirectServer } from '../net/http-redirect.js';
|
||||
|
||||
const __filenameServer = fileURLToPath(import.meta.url);
|
||||
const __dirnameServer = dirname(__filenameServer);
|
||||
@@ -112,7 +118,7 @@ export interface CoreServerOptions {
|
||||
repo: Repository;
|
||||
worktreeDir?: string;
|
||||
configuredRepos?: string[];
|
||||
generateTitle?: (body: string) => Promise<string>;
|
||||
generateTitle?: (body: string, ownerId?: string) => Promise<string>;
|
||||
selectPiece?: (body: string, fileNames: string[], userId?: string) => Promise<string>;
|
||||
configManager?: ConfigManager;
|
||||
piecesDir?: string;
|
||||
@@ -170,6 +176,14 @@ export function createCoreServer(opts: CoreServerOptions): {
|
||||
gatewayMount: GatewayMountHandle | null;
|
||||
/** True when an OAuth provider or local auth is active. False = no-auth mode. */
|
||||
authActive: boolean;
|
||||
/**
|
||||
* Resolved server config snapshot — computed once with the real listen port
|
||||
* so both the cookie-secure decision (inside createCoreServer) and the
|
||||
* listener branch (inside startCoreServer) share the SAME object.
|
||||
* A config hot-reload between the two would otherwise cause tls.enabled to
|
||||
* disagree between the cookie flag and the actual listener type.
|
||||
*/
|
||||
serverConfig: ReturnType<typeof mergeServerConfig>;
|
||||
} {
|
||||
const { repo, worktreeDir } = opts;
|
||||
const app = express();
|
||||
@@ -276,6 +290,16 @@ export function createCoreServer(opts: CoreServerOptions): {
|
||||
}
|
||||
let authenticateUpgrade: import('./auth.js').UpgradeAuthChecker | undefined;
|
||||
|
||||
// Resolve server config ONCE here with the real listen port (threaded in via
|
||||
// opts.listenPort by startCoreServer). Both the cookie-secure decision below
|
||||
// and the listener branch in startCoreServer consume this same snapshot so a
|
||||
// config hot-reload between the two calls cannot produce a mismatch between
|
||||
// tls.enabled and the cookie secure flag.
|
||||
const serverCfg = mergeServerConfig(loadConfig().server, {
|
||||
freshInstall: false,
|
||||
httpsPort: opts.listenPort ?? Number(process.env['PORT'] ?? 9876),
|
||||
});
|
||||
|
||||
if (authActive) {
|
||||
// Idempotently seed the shared `local` system admin (id='local', the same
|
||||
// owner the no-auth path uses) so an existing single-user / no-auth
|
||||
@@ -286,9 +310,21 @@ export function createCoreServer(opts: CoreServerOptions): {
|
||||
logger.info(`[auth] seeded local system admin id=local email=${bootstrap.email}`);
|
||||
}
|
||||
|
||||
// Compose effective secureCookie: native TLS termination also requires
|
||||
// the secure flag on session cookies, even when no upstream proxy is
|
||||
// present. IMPORTANT: trust-proxy (line ~191) stays keyed on the
|
||||
// ORIGINAL opts.authConfig.secureCookie — native TLS must NOT enable it.
|
||||
const effectiveSecureCookie = computeEffectiveSecureCookie(
|
||||
!!opts.authConfig?.secureCookie,
|
||||
serverCfg.tls.enabled,
|
||||
);
|
||||
const authConfigForSetup = opts.authConfig
|
||||
? { ...opts.authConfig, secureCookie: effectiveSecureCookie }
|
||||
: opts.authConfig;
|
||||
|
||||
const auth = setupAuth(
|
||||
repo,
|
||||
opts.authConfig!,
|
||||
authConfigForSetup!,
|
||||
() => {
|
||||
const b = resolveBranding(opts.configManager);
|
||||
return { appName: b.appName, loginPageTitle: b.loginPageTitle };
|
||||
@@ -330,6 +366,7 @@ export function createCoreServer(opts: CoreServerOptions): {
|
||||
// per-piece write authz (built-in/global-custom → admin, user-custom → owner)
|
||||
// is enforced inside pieces-api.ts handlers.
|
||||
app.use('/api/pieces', requireAuth);
|
||||
app.use('/api/usage', requireAuth);
|
||||
// Scheduled tasks: any authenticated user can create/list (visibility-filtered).
|
||||
// PATCH/DELETE owner-or-admin enforcement lives in the handlers (Task 14).
|
||||
app.use('/api/scheduled-tasks', requireAuth);
|
||||
@@ -893,10 +930,11 @@ export function createCoreServer(opts: CoreServerOptions): {
|
||||
});
|
||||
|
||||
// --- Local files API ---
|
||||
mountLocalFilesApi(app, repo);
|
||||
mountLocalFilesApi(app, repo, { authActive });
|
||||
|
||||
// --- Subtask activity API ---
|
||||
app.use('/api/local/tasks', createSubtaskActivityRouter(repo));
|
||||
app.use('/api/usage', createUsageRouter(repo, { authActive }));
|
||||
|
||||
// --- Subtask files API (listing MUST come before wildcard) ---
|
||||
mountSubtaskFilesApi(app, repo);
|
||||
@@ -1176,7 +1214,19 @@ export function createCoreServer(opts: CoreServerOptions): {
|
||||
return isOwner || user.role === 'admin';
|
||||
};
|
||||
|
||||
return { app, browserSessionManager, authenticateUpgrade, authorizeNovncSession, sshConsole, backendStatusRegistry, workerMetrics, gatewayMount, authActive };
|
||||
return { app, browserSessionManager, authenticateUpgrade, authorizeNovncSession, sshConsole, backendStatusRegistry, workerMetrics, gatewayMount, authActive, serverConfig: serverCfg };
|
||||
}
|
||||
|
||||
/** Cookie `secure` must be set whenever the user-facing scheme is https —
|
||||
* via an upstream TLS proxy (secureCookie) OR native TLS termination. */
|
||||
export function computeEffectiveSecureCookie(secureCookie: boolean, tlsEnabled: boolean): boolean {
|
||||
return secureCookie || tlsEnabled;
|
||||
}
|
||||
|
||||
/** Heuristic: native TLS + the proxy signal (secure_cookie) likely means a
|
||||
* reverse proxy is also terminating TLS → double-TLS misconfiguration. */
|
||||
export function shouldWarnDoubleTls(tlsEnabled: boolean, secureCookie: boolean): boolean {
|
||||
return tlsEnabled && secureCookie;
|
||||
}
|
||||
|
||||
export function finalizeServer(app: express.Application): express.Application {
|
||||
@@ -1231,6 +1281,7 @@ export function startCoreServer(opts: CoreServerOptions, port: number = 9876): v
|
||||
workerMetrics,
|
||||
gatewayMount,
|
||||
authActive,
|
||||
serverConfig,
|
||||
// Forward the actual port to createCoreServer so the admin gateway
|
||||
// status endpoint reports the real bind port (not the PORT env
|
||||
// guess). See `listenPort` doc on CoreServerOptions.
|
||||
@@ -1255,18 +1306,65 @@ export function startCoreServer(opts: CoreServerOptions, port: number = 9876): v
|
||||
// 127.0.0.1:9876 port mapping instead.
|
||||
const host = process.env['HOST'] ?? '127.0.0.1';
|
||||
const isLoopbackBind = host === '127.0.0.1' || host === '::1' || host === 'localhost';
|
||||
const server = finalApp.listen(port, host, () => {
|
||||
logger.info(`Core server listening on ${host}:${port}`);
|
||||
if (!isLoopbackBind && !authActive) {
|
||||
logger.warn(
|
||||
`[security] Listening on ${host} with authentication DISABLED. The agent API ` +
|
||||
`(including the Bash tool) is reachable by anyone who can reach this host — ` +
|
||||
`this is effectively unauthenticated remote code execution. Enable auth in ` +
|
||||
`config.yaml (auth.local or an OAuth provider) before exposing a non-loopback ` +
|
||||
`interface, or unset HOST to bind 127.0.0.1.`,
|
||||
// Use the config snapshot already resolved in createCoreServer (same port,
|
||||
// same loadConfig() call) — no second read so a hot-reload between the two
|
||||
// cannot make the cookie-secure flag disagree with the listener type.
|
||||
const tls = serverConfig.tls;
|
||||
let server: import('http').Server | import('https').Server;
|
||||
if (tls.enabled) {
|
||||
// Augment the self-signed SAN list with the redirect target host and the
|
||||
// (non-wildcard) bind host so that browsers following the HTTP→HTTPS redirect
|
||||
// always land on a hostname that is covered by the certificate. Provided-cert
|
||||
// deployments are unaffected because resolveTlsOptions ignores selfSignedHosts
|
||||
// when cert_file/key_file are set.
|
||||
const extraSan = [
|
||||
tls.redirectHost,
|
||||
host && host !== '0.0.0.0' && host !== '::' ? host : null,
|
||||
].filter((h): h is string => !!h);
|
||||
const tlsForResolve = extraSan.length
|
||||
? { ...tls, selfSignedHosts: [...tls.selfSignedHosts, ...extraSan] }
|
||||
: tls;
|
||||
const resolved = resolveTlsOptions(tlsForResolve); // fatal throw on bad operator cert
|
||||
server = createHttpsServer({ cert: resolved.cert, key: resolved.key, minVersion: resolved.minVersion }, finalApp);
|
||||
server.listen(port, host, () => {
|
||||
const source = tls.certFile ? `provided(${tls.certFile})` : 'self-signed';
|
||||
const fp = new X509Certificate(resolved.cert).fingerprint256;
|
||||
logger.info(`Core server listening on https://${host}:${port} cert=${source} sha256=${fp}`);
|
||||
if (shouldWarnDoubleTls(true, !!opts.authConfig?.secureCookie)) {
|
||||
logger.warn(
|
||||
`[security] server.tls.enabled is ON while auth.secure_cookie is also ON (reverse-proxy signal). ` +
|
||||
`If a TLS-terminating proxy is in front of this app, set server.tls.enabled: false to avoid double TLS.`,
|
||||
);
|
||||
}
|
||||
});
|
||||
if (tls.httpRedirect) {
|
||||
if (tls.redirectHost == null && (host === '0.0.0.0' || host === '::')) {
|
||||
logger.warn(
|
||||
`[server] HTTP->HTTPS redirect host falls back to the wildcard bind address (${host}); browsers cannot follow it. ` +
|
||||
`Set server.tls.redirect_host to the externally reachable hostname.`,
|
||||
);
|
||||
}
|
||||
const pinnedHost = tls.redirectHost ?? (isLoopbackBind ? 'localhost' : host);
|
||||
const redirector = createHttpRedirectServer({ httpsPort: port, pinnedHost });
|
||||
redirector.on('error', (e) => logger.warn(`[server] HTTP redirect listener error: ${(e as Error).message}`));
|
||||
redirector.listen(tls.httpRedirectPort, host, () =>
|
||||
logger.info(`HTTP->HTTPS redirect listening on http://${host}:${tls.httpRedirectPort}`),
|
||||
);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
server = finalApp.listen(port, host, () => {
|
||||
logger.info(`Core server listening on ${host}:${port}`);
|
||||
if (!isLoopbackBind && !authActive) {
|
||||
logger.warn(
|
||||
`[security] Listening on ${host} with authentication DISABLED. The agent API ` +
|
||||
`(including the Bash tool) is reachable by anyone who can reach this host — ` +
|
||||
`this is effectively unauthenticated remote code execution. Enable auth in ` +
|
||||
`config.yaml (auth.local or an OAuth provider) before exposing a non-loopback ` +
|
||||
`interface, or unset HOST to bind 127.0.0.1.`,
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// 起動と同時に CAPTCHA Pool の idle GC を回す (task session を 5 分アイドルで GC)
|
||||
if (browserSessionManager) browserSessionManager.startIdleGc();
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
/**
|
||||
* Usage dashboard API (GET /api/usage/daily) tests.
|
||||
*
|
||||
* Coverage:
|
||||
* - admin sees all users + byUser breakdown; non-admin scoped to own rows
|
||||
* - no-auth (authActive=false) sees everyone (scope 'all')
|
||||
* - day / week / month bucketing collapses model/route correctly
|
||||
* - inclusive range, default range, from>to → 400, range-too-large → 400
|
||||
* - invalid dates fall back to defaults (not 500)
|
||||
*
|
||||
* Spec: docs/superpowers/specs/2026-06-11-llm-usage-aggregation-design.md
|
||||
*/
|
||||
import { describe, it, expect, beforeEach } from 'vitest';
|
||||
import express from 'express';
|
||||
import request from 'supertest';
|
||||
import { Repository } from '../db/repository.js';
|
||||
import { createUsageRouter } from './usage-api.js';
|
||||
|
||||
function makeApp(repo: Repository, opts: { authActive: boolean; user?: { id: string; role?: string } }) {
|
||||
const app = express();
|
||||
app.use((req, _res, next) => {
|
||||
if (opts.user) (req as unknown as { user: unknown }).user = opts.user;
|
||||
next();
|
||||
});
|
||||
app.use('/api/usage', createUsageRouter(repo, { authActive: opts.authActive }));
|
||||
return app;
|
||||
}
|
||||
|
||||
function seed(repo: Repository, rows: Array<{ day: string; userId: string; source: 'gateway' | 'direct'; model?: string; route?: string; tin: number; tout: number; req?: number }>) {
|
||||
for (const r of rows) {
|
||||
repo.incrementLlmUsage({
|
||||
day: r.day, userId: r.userId, source: r.source,
|
||||
model: r.model ?? 'm', route: r.route ?? 'r',
|
||||
tokensIn: r.tin, tokensOut: r.tout, requests: r.req ?? 1,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
describe('GET /api/usage/daily', () => {
|
||||
let repo: Repository;
|
||||
beforeEach(() => {
|
||||
repo = new Repository(':memory:');
|
||||
seed(repo, [
|
||||
{ day: '2026-06-10', userId: 'u1', source: 'gateway', tin: 100, tout: 40 },
|
||||
{ day: '2026-06-10', userId: 'u1', source: 'direct', model: 'x', route: 'h', tin: 10, tout: 5 },
|
||||
{ day: '2026-06-11', userId: 'u1', source: 'gateway', tin: 7, tout: 3 },
|
||||
{ day: '2026-06-11', userId: 'u2', source: 'direct', tin: 1000, tout: 500 },
|
||||
]);
|
||||
});
|
||||
|
||||
it('non-admin sees only their own rows (scope=self, no byUser)', async () => {
|
||||
const app = makeApp(repo, { authActive: true, user: { id: 'u1', role: 'user' } });
|
||||
const res = await request(app).get('/api/usage/daily?from=2026-06-01&to=2026-06-30');
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.scope).toBe('self');
|
||||
expect(res.body.byUser).toBeUndefined();
|
||||
// u1 only: gateway 100+40+7+3=150, direct 10+5=15
|
||||
expect(res.body.totals.gateway).toMatchObject({ tokensIn: 107, tokensOut: 43, requests: 2 });
|
||||
expect(res.body.totals.direct).toMatchObject({ tokensIn: 10, tokensOut: 5, requests: 1 });
|
||||
});
|
||||
|
||||
it('admin sees all users with a byUser breakdown', async () => {
|
||||
const app = makeApp(repo, { authActive: true, user: { id: 'admin1', role: 'admin' } });
|
||||
const res = await request(app).get('/api/usage/daily?from=2026-06-01&to=2026-06-30');
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.scope).toBe('all');
|
||||
expect(res.body.totals.direct.tokensIn).toBe(1010); // u1 10 + u2 1000
|
||||
const users = (res.body.byUser as Array<{ userId: string }>).map((u) => u.userId).sort();
|
||||
expect(users).toEqual(['u1', 'u2']);
|
||||
// sorted by total tokens desc → u2 (1500) first
|
||||
expect(res.body.byUser[0].userId).toBe('u2');
|
||||
});
|
||||
|
||||
it('resolves byUser display names (real users → name, sentinels verbatim)', async () => {
|
||||
const u = repo.createUser({ email: '[email protected]', name: 'Alice', role: 'user', status: 'active' });
|
||||
seed(repo, [
|
||||
{ day: '2026-06-11', userId: u.id, source: 'direct', tin: 5, tout: 5 },
|
||||
{ day: '2026-06-11', userId: 'local', source: 'direct', tin: 1, tout: 1 },
|
||||
]);
|
||||
const app = makeApp(repo, { authActive: true, user: { id: 'admin1', role: 'admin' } });
|
||||
const res = await request(app).get('/api/usage/daily?from=2026-06-01&to=2026-06-30');
|
||||
const byId = Object.fromEntries((res.body.byUser as Array<{ userId: string; displayName: string }>).map((r) => [r.userId, r.displayName]));
|
||||
expect(byId[u.id]).toBe('Alice');
|
||||
expect(byId['local']).toBe('local'); // sentinel returned verbatim for UI localization
|
||||
});
|
||||
|
||||
it('no-auth mode (authActive=false) sees everyone', async () => {
|
||||
const app = makeApp(repo, { authActive: false }); // no req.user
|
||||
const res = await request(app).get('/api/usage/daily?from=2026-06-01&to=2026-06-30');
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.scope).toBe('all');
|
||||
expect(res.body.byUser.length).toBe(2);
|
||||
});
|
||||
|
||||
it('day granularity yields one bucket per active day', async () => {
|
||||
const app = makeApp(repo, { authActive: true, user: { id: 'admin1', role: 'admin' } });
|
||||
const res = await request(app).get('/api/usage/daily?from=2026-06-10&to=2026-06-11&granularity=day');
|
||||
expect(res.body.series.map((b: { bucket: string }) => b.bucket)).toEqual(['2026-06-10', '2026-06-11']);
|
||||
});
|
||||
|
||||
it('month granularity collapses days into a YYYY-MM bucket', async () => {
|
||||
const app = makeApp(repo, { authActive: true, user: { id: 'admin1', role: 'admin' } });
|
||||
const res = await request(app).get('/api/usage/daily?from=2026-06-01&to=2026-06-30&granularity=month');
|
||||
expect(res.body.series).toHaveLength(1);
|
||||
expect(res.body.series[0].bucket).toBe('2026-06');
|
||||
});
|
||||
|
||||
it('week granularity uses an ISO YYYY-Www bucket', async () => {
|
||||
const app = makeApp(repo, { authActive: true, user: { id: 'admin1', role: 'admin' } });
|
||||
const res = await request(app).get('/api/usage/daily?from=2026-06-08&to=2026-06-14&granularity=week');
|
||||
// 2026-06-10 / -11 fall in ISO week 24 of 2026
|
||||
expect(res.body.series).toHaveLength(1);
|
||||
expect(res.body.series[0].bucket).toBe('2026-W24');
|
||||
});
|
||||
|
||||
it('rejects from > to with 400', async () => {
|
||||
const app = makeApp(repo, { authActive: true, user: { id: 'u1', role: 'user' } });
|
||||
const res = await request(app).get('/api/usage/daily?from=2026-06-30&to=2026-06-01');
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
it('rejects an absurdly large range with 400', async () => {
|
||||
const app = makeApp(repo, { authActive: true, user: { id: 'u1', role: 'user' } });
|
||||
const res = await request(app).get('/api/usage/daily?from=2000-01-01&to=2026-06-30');
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
it('falls back to defaults for invalid dates (no 500)', async () => {
|
||||
const app = makeApp(repo, { authActive: true, user: { id: 'u1', role: 'user' } });
|
||||
const res = await request(app).get('/api/usage/daily?from=2026-99-99');
|
||||
expect(res.status).toBe(200);
|
||||
// default window is the last 30 days, ending today
|
||||
expect(res.body.to).toMatch(/^\d{4}-\d{2}-\d{2}$/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,171 @@
|
||||
import { Router, Request, Response } from 'express';
|
||||
import type { Repository, LlmUsageDailyAgg } from '../db/repository.js';
|
||||
import { logger } from '../logger.js';
|
||||
|
||||
/**
|
||||
* Per-user LLM usage dashboard API. Reads the llm_usage_daily ledger
|
||||
* (gateway + direct, recorded at the OpenAICompatClient completion
|
||||
* boundary) and shapes a time series for the Usage tab.
|
||||
*
|
||||
* Visibility: admin (and the no-auth single-user local mode) see every
|
||||
* user's usage; a non-admin authenticated user sees only their own rows.
|
||||
* This is a separate lens from the gateway per-key billing view — the two
|
||||
* are never summed.
|
||||
*
|
||||
* Spec: docs/superpowers/specs/2026-06-11-llm-usage-aggregation-design.md
|
||||
*/
|
||||
|
||||
const DAY_RE = /^\d{4}-\d{2}-\d{2}$/;
|
||||
|
||||
/** True only for a real calendar day in 'YYYY-MM-DD' (rejects 2026-99-99). */
|
||||
function isValidDay(s: unknown): s is string {
|
||||
if (typeof s !== 'string' || !DAY_RE.test(s)) return false;
|
||||
const d = new Date(`${s}T00:00:00.000Z`);
|
||||
return !Number.isNaN(d.getTime()) && d.toISOString().slice(0, 10) === s;
|
||||
}
|
||||
const MAX_RANGE_DAYS = 800; // ~2y guard so a hand-crafted range can't scan unbounded
|
||||
type Granularity = 'day' | 'week' | 'month';
|
||||
|
||||
interface Counters {
|
||||
tokensIn: number;
|
||||
tokensOut: number;
|
||||
requests: number;
|
||||
}
|
||||
|
||||
function emptyCounters(): Counters {
|
||||
return { tokensIn: 0, tokensOut: 0, requests: 0 };
|
||||
}
|
||||
|
||||
function addInto(target: Counters, row: LlmUsageDailyAgg): void {
|
||||
target.tokensIn += row.tokensIn;
|
||||
target.tokensOut += row.tokensOut;
|
||||
target.requests += row.requests;
|
||||
}
|
||||
|
||||
function utcToday(): string {
|
||||
return new Date().toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
/** day - n days, as 'YYYY-MM-DD' (UTC). */
|
||||
function shiftDay(day: string, deltaDays: number): string {
|
||||
const d = new Date(`${day}T00:00:00.000Z`);
|
||||
d.setUTCDate(d.getUTCDate() + deltaDays);
|
||||
return d.toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
/** Inclusive day count between two 'YYYY-MM-DD' (UTC). */
|
||||
function dayDiff(from: string, to: string): number {
|
||||
const a = Date.parse(`${from}T00:00:00.000Z`);
|
||||
const b = Date.parse(`${to}T00:00:00.000Z`);
|
||||
return Math.round((b - a) / 86_400_000);
|
||||
}
|
||||
|
||||
/** ISO-8601 week key 'YYYY-Www' for a 'YYYY-MM-DD' day (UTC). */
|
||||
function isoWeekKey(day: string): string {
|
||||
const d = new Date(`${day}T00:00:00.000Z`);
|
||||
// ISO week: Thursday of the current week decides the year.
|
||||
const dayNum = (d.getUTCDay() + 6) % 7; // Mon=0 .. Sun=6
|
||||
d.setUTCDate(d.getUTCDate() - dayNum + 3);
|
||||
const firstThursday = new Date(Date.UTC(d.getUTCFullYear(), 0, 4));
|
||||
const firstDayNum = (firstThursday.getUTCDay() + 6) % 7;
|
||||
firstThursday.setUTCDate(firstThursday.getUTCDate() - firstDayNum + 3);
|
||||
const week = 1 + Math.round((d.getTime() - firstThursday.getTime()) / (7 * 86_400_000));
|
||||
return `${d.getUTCFullYear()}-W${String(week).padStart(2, '0')}`;
|
||||
}
|
||||
|
||||
function bucketKey(day: string, granularity: Granularity): string {
|
||||
if (granularity === 'month') return day.slice(0, 7);
|
||||
if (granularity === 'week') return isoWeekKey(day);
|
||||
return day;
|
||||
}
|
||||
|
||||
/**
|
||||
* Human-friendly label for a usage owner id. Real users resolve to their
|
||||
* name (or email) so the admin breakdown isn't a wall of opaque ids; the
|
||||
* 'local' / 'system' sentinels are returned verbatim so the UI can localize
|
||||
* them. Falls back to the raw id when no user row exists.
|
||||
*/
|
||||
function resolveDisplayName(repo: Repository, userId: string): string {
|
||||
if (userId === 'local' || userId === 'system') return userId;
|
||||
const u = repo.getUserById(userId);
|
||||
return u?.name || u?.email || userId;
|
||||
}
|
||||
|
||||
export function createUsageRouter(repo: Repository, opts: { authActive: boolean }): Router {
|
||||
const router = Router();
|
||||
|
||||
// GET /daily?from=YYYY-MM-DD&to=YYYY-MM-DD&granularity=day|week|month
|
||||
router.get('/daily', (req: Request, res: Response) => {
|
||||
try {
|
||||
const to = isValidDay(req.query['to']) ? req.query['to'] : utcToday();
|
||||
const from = isValidDay(req.query['from']) ? req.query['from'] : shiftDay(to, -29);
|
||||
if (from > to) {
|
||||
res.status(400).json({ error: 'from must be on or before to' });
|
||||
return;
|
||||
}
|
||||
if (dayDiff(from, to) > MAX_RANGE_DAYS) {
|
||||
res.status(400).json({ error: `range too large (max ${MAX_RANGE_DAYS} days)` });
|
||||
return;
|
||||
}
|
||||
const gq = req.query['granularity'];
|
||||
const granularity: Granularity =
|
||||
gq === 'week' || gq === 'month' ? gq : 'day';
|
||||
|
||||
// Visibility: a non-admin authenticated user is scoped to their own
|
||||
// rows. Admin and the no-auth local mode see everyone.
|
||||
const user = req.user as Express.User | undefined;
|
||||
const isAdmin = !opts.authActive || user?.role === 'admin';
|
||||
const scopeUserId = isAdmin ? undefined : (user?.id ?? 'local');
|
||||
|
||||
const rows = repo.queryLlmUsageDaily({ from, to, userId: scopeUserId });
|
||||
|
||||
// Bucket by (bucketKey, source). Buckets are sparse — only days with
|
||||
// usage appear; the client fills gaps for the chart.
|
||||
const buckets = new Map<string, { gateway: Counters; direct: Counters }>();
|
||||
const totals = { gateway: emptyCounters(), direct: emptyCounters() };
|
||||
const byUser = new Map<string, Counters>();
|
||||
|
||||
for (const row of rows) {
|
||||
const key = bucketKey(row.day, granularity);
|
||||
let b = buckets.get(key);
|
||||
if (!b) {
|
||||
b = { gateway: emptyCounters(), direct: emptyCounters() };
|
||||
buckets.set(key, b);
|
||||
}
|
||||
const sourceKey = row.source === 'gateway' ? 'gateway' : 'direct';
|
||||
addInto(b[sourceKey], row);
|
||||
addInto(totals[sourceKey], row);
|
||||
if (isAdmin) {
|
||||
let u = byUser.get(row.userId);
|
||||
if (!u) { u = emptyCounters(); byUser.set(row.userId, u); }
|
||||
addInto(u, row);
|
||||
}
|
||||
}
|
||||
|
||||
const series = Array.from(buckets.entries())
|
||||
.sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0))
|
||||
.map(([bucket, c]) => ({ bucket, gateway: c.gateway, direct: c.direct }));
|
||||
|
||||
res.json({
|
||||
from,
|
||||
to,
|
||||
granularity,
|
||||
scope: isAdmin ? 'all' : 'self',
|
||||
series,
|
||||
totals,
|
||||
...(isAdmin
|
||||
? {
|
||||
byUser: Array.from(byUser.entries())
|
||||
.map(([userId, c]) => ({ userId, displayName: resolveDisplayName(repo, userId), ...c }))
|
||||
.sort((a, b) => (b.tokensIn + b.tokensOut) - (a.tokensIn + a.tokensOut)),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
} catch (e) {
|
||||
logger.error(`[usage-api] /daily failed: ${String(e)}`);
|
||||
res.status(500).json({ error: 'Failed to load usage' });
|
||||
}
|
||||
});
|
||||
|
||||
return router;
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import { logger } from './logger.js';
|
||||
import { normalizeConfig } from './config-normalize.js';
|
||||
import type { McpRuntimeConfig } from './mcp/config.js';
|
||||
import type { SshRuntimeConfig } from './ssh/config.js';
|
||||
import type { ServerConfig } from './server/config.js';
|
||||
|
||||
export interface AskConfig {
|
||||
maxPerJob: number; // default: 2
|
||||
@@ -60,6 +61,7 @@ export interface ToolsConfig {
|
||||
officePdfMaxSizeMb?: number; // ReadPdf の最大ファイルサイズ (default: 10)
|
||||
officePptxMaxSizeMb?: number; // ReadPPTX の最大ファイルサイズ (default: 50)
|
||||
officePptxMaxUncompressedMb?: number; // ReadPPTX の ZIP 展開後サイズ上限 (default: 200)
|
||||
officeMsgMaxSizeMb?: number; // ReadMsg の最大ファイルサイズ (default: 25)
|
||||
/**
|
||||
* Max request body size (MB) for the POST /api/local/tasks and
|
||||
* /api/local/tasks/:id/comments endpoints (includes base64-encoded
|
||||
@@ -502,6 +504,7 @@ export interface AppConfig {
|
||||
ssh?: Partial<SshRuntimeConfig>;
|
||||
notes?: NotesConfig;
|
||||
notifications?: NotificationsConfig;
|
||||
server?: Partial<ServerConfig>;
|
||||
}
|
||||
|
||||
const DEFAULT_REFLECTION: ReflectionConfig = {
|
||||
|
||||
@@ -338,27 +338,38 @@ describe('runReflectionJob integration', () => {
|
||||
}),
|
||||
} as any);
|
||||
|
||||
// Mock fetch so callReflectionLlm returns a valid abstain result
|
||||
const mockFetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
choices: [{
|
||||
message: {
|
||||
tool_calls: [{
|
||||
function: {
|
||||
arguments: JSON.stringify({
|
||||
memory_changes: [],
|
||||
piece_changes: { should_edit: false },
|
||||
reasoning: 'nothing to learn',
|
||||
abstain_reason: 'task completed successfully without issues',
|
||||
}),
|
||||
},
|
||||
}],
|
||||
},
|
||||
}],
|
||||
usage: { prompt_tokens: 123, completion_tokens: 45 },
|
||||
}),
|
||||
} as any);
|
||||
// Mock fetch so callReflectionLlm (now streaming via OpenAICompatClient)
|
||||
// returns a valid abstain result as an SSE submit_reflection tool_call.
|
||||
const reflectionArgs = JSON.stringify({
|
||||
memory_changes: [],
|
||||
piece_changes: { should_edit: false },
|
||||
reasoning: 'nothing to learn',
|
||||
abstain_reason: 'task completed successfully without issues',
|
||||
});
|
||||
const sseLines = [
|
||||
`data: ${JSON.stringify({ model: 'reflect-model', choices: [{ delta: { tool_calls: [{ index: 0, id: 'r1', function: { name: 'submit_reflection', arguments: reflectionArgs } }] }, finish_reason: null }] })}\n\n`,
|
||||
`data: ${JSON.stringify({ choices: [{ delta: {}, finish_reason: 'tool_calls' }] })}\n\n`,
|
||||
`data: ${JSON.stringify({ choices: [], usage: { prompt_tokens: 123, completion_tokens: 45 } })}\n\n`,
|
||||
'data: [DONE]\n\n',
|
||||
];
|
||||
const mockFetch = vi.fn().mockImplementation(async () => {
|
||||
const encoder = new TextEncoder();
|
||||
let i = 0;
|
||||
return {
|
||||
ok: true,
|
||||
status: 200,
|
||||
headers: { get: () => null },
|
||||
body: {
|
||||
getReader: () => ({
|
||||
read: async () =>
|
||||
i < sseLines.length
|
||||
? { done: false, value: encoder.encode(sseLines[i++]) }
|
||||
: { done: true, value: undefined },
|
||||
releaseLock: () => {},
|
||||
}),
|
||||
},
|
||||
} as any;
|
||||
});
|
||||
vi.stubGlobal('fetch', mockFetch);
|
||||
|
||||
const { runReflectionJob } = await import('../engine/reflection/reflection-runner.js');
|
||||
|
||||
@@ -90,12 +90,44 @@ export function runMigrations(db: Database.Database): void {
|
||||
db.exec("ALTER TABLE local_tasks ADD COLUMN options TEXT DEFAULT '{}'");
|
||||
});
|
||||
|
||||
// Title provenance: 'auto' (creation fallback) / 'agent' (derived from
|
||||
// Mission Brief goal) / 'user' (manual edit, never auto-overwritten).
|
||||
addColumnIfMissing(db, 'local_tasks', 'title_source', () => {
|
||||
db.exec("ALTER TABLE local_tasks ADD COLUMN title_source TEXT NOT NULL DEFAULT 'auto'");
|
||||
});
|
||||
|
||||
migrateMcpTables(db);
|
||||
migrateSshTables(db);
|
||||
migrateNotesTables(db);
|
||||
migrateDashboardWidgets(db);
|
||||
migrateGatewayVirtualKeys(db);
|
||||
migratePushNotificationsTables(db);
|
||||
migrateLlmUsageDaily(db);
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-user daily LLM usage aggregation (gateway + direct). Idempotent.
|
||||
* Mirrors schema.sql + Repository.initSchema (dual-path rule:
|
||||
* project_db_migration_dual_path). Additive table, no mixed-version risk.
|
||||
* Spec: docs/superpowers/specs/2026-06-11-llm-usage-aggregation-design.md.
|
||||
*/
|
||||
function migrateLlmUsageDaily(db: Database.Database): void {
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS llm_usage_daily (
|
||||
day TEXT NOT NULL,
|
||||
user_id TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
route TEXT NOT NULL,
|
||||
tokens_in INTEGER NOT NULL DEFAULT 0,
|
||||
tokens_out INTEGER NOT NULL DEFAULT 0,
|
||||
requests INTEGER NOT NULL DEFAULT 0,
|
||||
last_updated_at TEXT NOT NULL,
|
||||
PRIMARY KEY (day, user_id, source, model, route)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_llm_usage_daily_user_day
|
||||
ON llm_usage_daily (user_id, day);
|
||||
`);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
/**
|
||||
* Per-user daily LLM usage ledger (llm_usage_daily) repository tests.
|
||||
*
|
||||
* Coverage:
|
||||
* - incrementLlmUsage UPSERTs on first call, accumulates on second
|
||||
* - requests defaults to +1; a usage-less call still bumps requests
|
||||
* - negative deltas clamp to zero
|
||||
* - distinct (model) and (route) produce distinct rows on the same day
|
||||
* - 'system' / 'local' sentinels aggregate as single rows (NULL trap avoided)
|
||||
* - queryLlmUsageDaily collapses model/route, groups by (day, user, source)
|
||||
* - day filter is an inclusive range; userId filter scopes a single user
|
||||
* - UTC day boundary splits into separate buckets
|
||||
*
|
||||
* Spec: docs/superpowers/specs/2026-06-11-llm-usage-aggregation-design.md
|
||||
*/
|
||||
import { describe, expect, it, beforeEach } from 'vitest';
|
||||
import { Repository } from './repository.js';
|
||||
|
||||
function makeRepo(): Repository {
|
||||
return new Repository(':memory:');
|
||||
}
|
||||
|
||||
describe('llm_usage_daily repository', () => {
|
||||
let repo: Repository;
|
||||
beforeEach(() => {
|
||||
repo = makeRepo();
|
||||
});
|
||||
|
||||
it('UPSERTs first call and accumulates on the same grain', () => {
|
||||
const grain = { day: '2026-06-11', userId: 'u1', source: 'direct' as const, model: 'm', route: 'r' };
|
||||
repo.incrementLlmUsage({ ...grain, tokensIn: 100, tokensOut: 40 });
|
||||
repo.incrementLlmUsage({ ...grain, tokensIn: 10, tokensOut: 5 });
|
||||
const rows = repo.queryLlmUsageDaily({ from: '2026-06-11', to: '2026-06-11' });
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0]).toMatchObject({
|
||||
day: '2026-06-11', userId: 'u1', source: 'direct',
|
||||
tokensIn: 110, tokensOut: 45, requests: 2,
|
||||
});
|
||||
});
|
||||
|
||||
it('a usage-less call still bumps requests (0-token request != failure)', () => {
|
||||
repo.incrementLlmUsage({ day: '2026-06-11', userId: 'u1', source: 'gateway', model: 'm', route: 'r' });
|
||||
const rows = repo.queryLlmUsageDaily({ from: '2026-06-11', to: '2026-06-11' });
|
||||
expect(rows[0]).toMatchObject({ tokensIn: 0, tokensOut: 0, requests: 1 });
|
||||
});
|
||||
|
||||
it('clamps negative deltas to zero', () => {
|
||||
repo.incrementLlmUsage({ day: '2026-06-11', userId: 'u1', source: 'direct', model: 'm', route: 'r', tokensIn: -5, tokensOut: -9, requests: -3 });
|
||||
const rows = repo.queryLlmUsageDaily({ from: '2026-06-11', to: '2026-06-11' });
|
||||
expect(rows[0]).toMatchObject({ tokensIn: 0, tokensOut: 0, requests: 0 });
|
||||
});
|
||||
|
||||
it('distinct model and route are separate rows but collapse in the query', () => {
|
||||
const base = { day: '2026-06-11', userId: 'u1', source: 'direct' as const, tokensIn: 10, tokensOut: 5 };
|
||||
repo.incrementLlmUsage({ ...base, model: 'big', route: 'host-a' });
|
||||
repo.incrementLlmUsage({ ...base, model: 'small', route: 'host-a' });
|
||||
repo.incrementLlmUsage({ ...base, model: 'big', route: 'host-b' });
|
||||
// 3 distinct (model,route) rows underneath, collapsed to one (day,user,source).
|
||||
const rows = repo.queryLlmUsageDaily({ from: '2026-06-11', to: '2026-06-11' });
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0]).toMatchObject({ tokensIn: 30, tokensOut: 15, requests: 3 });
|
||||
});
|
||||
|
||||
it("'system' and 'local' sentinels each aggregate as a single row", () => {
|
||||
const g = { day: '2026-06-11', source: 'direct' as const, model: 'm', route: 'r', tokensIn: 1, tokensOut: 1 };
|
||||
repo.incrementLlmUsage({ ...g, userId: 'system' });
|
||||
repo.incrementLlmUsage({ ...g, userId: 'system' });
|
||||
repo.incrementLlmUsage({ ...g, userId: 'local' });
|
||||
const rows = repo.queryLlmUsageDaily({ from: '2026-06-11', to: '2026-06-11' });
|
||||
const byUser = Object.fromEntries(rows.map((r) => [r.userId, r.requests]));
|
||||
expect(byUser).toEqual({ system: 2, local: 1 });
|
||||
});
|
||||
|
||||
it('gateway and direct are distinct rows for the same user/day', () => {
|
||||
repo.incrementLlmUsage({ day: '2026-06-11', userId: 'u1', source: 'gateway', model: 'm', route: 'r', tokensIn: 7, tokensOut: 3 });
|
||||
repo.incrementLlmUsage({ day: '2026-06-11', userId: 'u1', source: 'direct', model: 'm', route: 'r', tokensIn: 2, tokensOut: 1 });
|
||||
const rows = repo.queryLlmUsageDaily({ from: '2026-06-11', to: '2026-06-11' });
|
||||
expect(rows).toHaveLength(2);
|
||||
expect(rows.map((r) => r.source).sort()).toEqual(['direct', 'gateway']);
|
||||
});
|
||||
|
||||
it('queryLlmUsageDaily honours an inclusive day range', () => {
|
||||
for (const day of ['2026-06-09', '2026-06-10', '2026-06-11', '2026-06-12']) {
|
||||
repo.incrementLlmUsage({ day, userId: 'u1', source: 'direct', model: 'm', route: 'r', tokensIn: 1, tokensOut: 0 });
|
||||
}
|
||||
const rows = repo.queryLlmUsageDaily({ from: '2026-06-10', to: '2026-06-11' });
|
||||
expect(rows.map((r) => r.day)).toEqual(['2026-06-10', '2026-06-11']);
|
||||
});
|
||||
|
||||
it('userId filter scopes the query to one user', () => {
|
||||
repo.incrementLlmUsage({ day: '2026-06-11', userId: 'u1', source: 'direct', model: 'm', route: 'r', tokensIn: 1, tokensOut: 0 });
|
||||
repo.incrementLlmUsage({ day: '2026-06-11', userId: 'u2', source: 'direct', model: 'm', route: 'r', tokensIn: 9, tokensOut: 0 });
|
||||
const mine = repo.queryLlmUsageDaily({ from: '2026-06-11', to: '2026-06-11', userId: 'u1' });
|
||||
expect(mine).toHaveLength(1);
|
||||
expect(mine[0]).toMatchObject({ userId: 'u1', tokensIn: 1 });
|
||||
});
|
||||
|
||||
it('derives the UTC day from `at` when day is omitted; boundary splits buckets', () => {
|
||||
const grain = { userId: 'u1', source: 'direct' as const, model: 'm', route: 'r', tokensIn: 1, tokensOut: 0 };
|
||||
repo.incrementLlmUsage({ ...grain, at: '2026-06-11T23:59:59.000Z' });
|
||||
repo.incrementLlmUsage({ ...grain, at: '2026-06-12T00:00:01.000Z' });
|
||||
const rows = repo.queryLlmUsageDaily({ from: '2026-06-01', to: '2026-06-30' });
|
||||
expect(rows.map((r) => r.day)).toEqual(['2026-06-11', '2026-06-12']);
|
||||
});
|
||||
});
|
||||
@@ -1653,3 +1653,74 @@ describe('Repository browser notifications V2', () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('Repository title derivation from Mission Brief goal', () => {
|
||||
let tempDir = '';
|
||||
|
||||
afterEach(() => {
|
||||
if (tempDir) {
|
||||
rmSync(tempDir, { recursive: true, force: true });
|
||||
tempDir = '';
|
||||
}
|
||||
});
|
||||
|
||||
function makeRepo(): Repository {
|
||||
tempDir = mkdtempSync(join(tmpdir(), 'maestro-title-'));
|
||||
return new Repository(join(tempDir, 'orchestrator.db'));
|
||||
}
|
||||
|
||||
it('derives the title from the goal when title is auto, and marks it agent', async () => {
|
||||
const repo = makeRepo();
|
||||
try {
|
||||
const task = await repo.createLocalTask({ title: '仮タイトル', titleSource: 'auto', body: 'b' });
|
||||
repo.makeMissionBriefIO(task.id).update({ goal: '議事録を作成する\n背景...' });
|
||||
const after = await repo.getLocalTask(task.id);
|
||||
expect(after?.title).toBe('議事録を作成する');
|
||||
expect(after?.titleSource).toBe('agent');
|
||||
} finally {
|
||||
repo.close();
|
||||
}
|
||||
});
|
||||
|
||||
it('re-derives on a later goal update while still agent-owned', async () => {
|
||||
const repo = makeRepo();
|
||||
try {
|
||||
const task = await repo.createLocalTask({ title: 'x', titleSource: 'auto', body: 'b' });
|
||||
const io = repo.makeMissionBriefIO(task.id);
|
||||
io.update({ goal: '最初の目標' });
|
||||
io.update({ goal: '更新された目標' });
|
||||
const after = await repo.getLocalTask(task.id);
|
||||
expect(after?.title).toBe('更新された目標');
|
||||
expect(after?.titleSource).toBe('agent');
|
||||
} finally {
|
||||
repo.close();
|
||||
}
|
||||
});
|
||||
|
||||
it('never overwrites a user-edited title', async () => {
|
||||
const repo = makeRepo();
|
||||
try {
|
||||
const task = await repo.createLocalTask({ title: 'x', titleSource: 'auto', body: 'b' });
|
||||
await repo.updateLocalTask(task.id, { title: '手動タイトル', titleSource: 'user' });
|
||||
repo.makeMissionBriefIO(task.id).update({ goal: 'エージェントの目標' });
|
||||
const after = await repo.getLocalTask(task.id);
|
||||
expect(after?.title).toBe('手動タイトル');
|
||||
expect(after?.titleSource).toBe('user');
|
||||
} finally {
|
||||
repo.close();
|
||||
}
|
||||
});
|
||||
|
||||
it('does not touch the title when the patch has no goal', async () => {
|
||||
const repo = makeRepo();
|
||||
try {
|
||||
const task = await repo.createLocalTask({ title: '仮', titleSource: 'auto', body: 'b' });
|
||||
repo.makeMissionBriefIO(task.id).update({ done: '- step 1' });
|
||||
const after = await repo.getLocalTask(task.id);
|
||||
expect(after?.title).toBe('仮');
|
||||
expect(after?.titleSource).toBe('auto');
|
||||
} finally {
|
||||
repo.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
+164
-8
@@ -6,6 +6,7 @@ import { randomUUID, scryptSync, randomBytes, timingSafeEqual } from 'crypto';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import { logger } from '../logger.js';
|
||||
import { buildVisibilityWhere } from '../bridge/visibility.js';
|
||||
import { buildTitleFromGoal } from '../title-generation.js';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
@@ -146,9 +147,13 @@ export interface SubtaskInfo {
|
||||
childCompleted?: number;
|
||||
}
|
||||
|
||||
export type TitleSource = 'auto' | 'agent' | 'user';
|
||||
|
||||
export interface LocalTask {
|
||||
id: number;
|
||||
title: string;
|
||||
/** Provenance of `title`. 'user' is never auto-overwritten by the agent. */
|
||||
titleSource: TitleSource;
|
||||
body: string;
|
||||
pieceName: string;
|
||||
profile: 'auto' | 'fast' | 'quality' | string;
|
||||
@@ -335,6 +340,33 @@ function rowToGatewayKeyUsage(row: GatewayKeyUsageRow): GatewayKeyUsage {
|
||||
};
|
||||
}
|
||||
|
||||
/** Per-call delta for the per-user daily LLM usage ledger. */
|
||||
export interface LlmUsageIncrement {
|
||||
/** UTC day bucket 'YYYY-MM-DD'. Defaults to today (UTC) when omitted. */
|
||||
day?: string;
|
||||
/** Owner id, or 'local' (no-auth) / 'system' (ownerless) sentinel. */
|
||||
userId: string;
|
||||
source: 'gateway' | 'direct';
|
||||
/** Real model name (chunk.model), routing-key fallback, or 'unknown'. */
|
||||
model: string;
|
||||
/** Backend server name (gateway backendId / direct host), or 'unknown'. */
|
||||
route: string;
|
||||
tokensIn?: number;
|
||||
tokensOut?: number;
|
||||
requests?: number;
|
||||
at?: string;
|
||||
}
|
||||
|
||||
/** Daily-grouped aggregate row (model/route collapsed) for the usage API. */
|
||||
export interface LlmUsageDailyAgg {
|
||||
day: string;
|
||||
userId: string;
|
||||
source: string;
|
||||
tokensIn: number;
|
||||
tokensOut: number;
|
||||
requests: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Coerce an optional limit (tokens_budget / rate_limit_rpm) to either
|
||||
* a positive integer or null. Anything else (undefined, null, 0,
|
||||
@@ -432,6 +464,8 @@ export interface UpsertWorkerNodeParams {
|
||||
|
||||
export interface CreateLocalTaskParams {
|
||||
title: string;
|
||||
/** Defaults to 'auto'. Pass 'user' when the caller supplied an explicit title. */
|
||||
titleSource?: TitleSource;
|
||||
body: string;
|
||||
pieceName?: string;
|
||||
profile?: 'auto' | 'fast' | 'quality';
|
||||
@@ -650,6 +684,7 @@ interface JobRow {
|
||||
interface LocalTaskRow {
|
||||
id: number;
|
||||
title: string;
|
||||
title_source: string | null;
|
||||
body: string;
|
||||
piece_name: string;
|
||||
profile: string;
|
||||
@@ -802,6 +837,7 @@ function rowToLocalTask(row: LocalTaskRow): LocalTask {
|
||||
return {
|
||||
id: row.id,
|
||||
title: row.title,
|
||||
titleSource: (row.title_source as TitleSource | null) ?? 'auto',
|
||||
body: row.body,
|
||||
pieceName: row.piece_name,
|
||||
profile: row.profile,
|
||||
@@ -1133,6 +1169,26 @@ export class Repository {
|
||||
CREATE INDEX IF NOT EXISTS idx_gateway_usage_key
|
||||
ON gateway_key_usage (key_id);
|
||||
`);
|
||||
|
||||
// Per-user daily LLM usage (gateway + direct). Mirrors schema.sql +
|
||||
// migrate.ts (dual-path rule). Separate lens from gateway_key_usage.
|
||||
// Spec: docs/superpowers/specs/2026-06-11-llm-usage-aggregation-design.md
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS llm_usage_daily (
|
||||
day TEXT NOT NULL,
|
||||
user_id TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
route TEXT NOT NULL,
|
||||
tokens_in INTEGER NOT NULL DEFAULT 0,
|
||||
tokens_out INTEGER NOT NULL DEFAULT 0,
|
||||
requests INTEGER NOT NULL DEFAULT 0,
|
||||
last_updated_at TEXT NOT NULL,
|
||||
PRIMARY KEY (day, user_id, source, model, route)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_llm_usage_daily_user_day
|
||||
ON llm_usage_daily (user_id, day);
|
||||
`);
|
||||
}
|
||||
|
||||
private ensureColumn(tableName: string, columnName: string, definition: string): void {
|
||||
@@ -1247,11 +1303,12 @@ export class Repository {
|
||||
async createLocalTask(params: CreateLocalTaskParams): Promise<LocalTask> {
|
||||
const result = this.db
|
||||
.prepare(
|
||||
`INSERT INTO local_tasks (title, body, piece_name, profile, output_format, ask_policy, priority, workspace_path, owner_id, visibility, visibility_scope_org_id, browser_session_profile_id, options)
|
||||
VALUES (@title, @body, @pieceName, @profile, @outputFormat, @askPolicy, @priority, @workspacePath, @ownerId, @visibility, @visibilityScopeOrgId, @browserSessionProfileId, @options)`
|
||||
`INSERT INTO local_tasks (title, title_source, body, piece_name, profile, output_format, ask_policy, priority, workspace_path, owner_id, visibility, visibility_scope_org_id, browser_session_profile_id, options)
|
||||
VALUES (@title, @titleSource, @body, @pieceName, @profile, @outputFormat, @askPolicy, @priority, @workspacePath, @ownerId, @visibility, @visibilityScopeOrgId, @browserSessionProfileId, @options)`
|
||||
)
|
||||
.run({
|
||||
title: params.title,
|
||||
titleSource: params.titleSource ?? 'auto',
|
||||
body: params.body,
|
||||
pieceName: params.pieceName ?? 'chat',
|
||||
profile: params.profile ?? 'auto',
|
||||
@@ -1365,9 +1422,10 @@ export class Repository {
|
||||
* from sync paths (e.g. buildSystemPrompt). better-sqlite3 is sync
|
||||
* underneath anyway. */
|
||||
updateMissionBriefSync(taskId: number, patch: Partial<MissionBrief>): MissionBrief | null {
|
||||
const existing = parseMissionBrief(
|
||||
(this.db.prepare(`SELECT mission_brief FROM local_tasks WHERE id = ?`).get(taskId) as { mission_brief: string | null } | undefined)?.mission_brief ?? null,
|
||||
);
|
||||
const row = this.db
|
||||
.prepare(`SELECT mission_brief, title_source FROM local_tasks WHERE id = ?`)
|
||||
.get(taskId) as { mission_brief: string | null; title_source: string | null } | undefined;
|
||||
const existing = parseMissionBrief(row?.mission_brief ?? null);
|
||||
const next: MissionBrief = {
|
||||
goal: patch.goal !== undefined ? patch.goal : existing?.goal ?? '',
|
||||
done: patch.done !== undefined ? patch.done : existing?.done ?? '',
|
||||
@@ -1376,9 +1434,29 @@ export class Repository {
|
||||
};
|
||||
const allEmpty = !next.goal && !next.done && !next.open && !next.clarifications;
|
||||
const stored = allEmpty ? null : JSON.stringify(next);
|
||||
this.db.prepare(
|
||||
`UPDATE local_tasks SET mission_brief = ?, updated_at = datetime('now') WHERE id = ?`
|
||||
).run(stored, taskId);
|
||||
|
||||
// Derive the task title from the agent's goal (no LLM call). Only when the
|
||||
// goal value actually changed (agents re-send an unchanged brief across
|
||||
// iterations — re-deriving every time would churn updated_at and flicker
|
||||
// the title) and the user hasn't taken manual control (a user edit pins
|
||||
// title_source='user' and is never overwritten).
|
||||
const goalChanged = patch.goal !== undefined && patch.goal !== (existing?.goal ?? '');
|
||||
const derivedTitle = (goalChanged && (row?.title_source ?? 'auto') !== 'user')
|
||||
? buildTitleFromGoal(next.goal)
|
||||
: '';
|
||||
|
||||
// Atomic: persist the brief and the derived title as one unit so a crash
|
||||
// between them can't leave the title out of sync with the goal.
|
||||
this.db.transaction(() => {
|
||||
this.db.prepare(
|
||||
`UPDATE local_tasks SET mission_brief = ?, updated_at = datetime('now') WHERE id = ?`
|
||||
).run(stored, taskId);
|
||||
if (derivedTitle) {
|
||||
this.db.prepare(
|
||||
`UPDATE local_tasks SET title = ?, title_source = 'agent' WHERE id = ?`
|
||||
).run(derivedTitle, taskId);
|
||||
}
|
||||
})();
|
||||
return allEmpty ? null : next;
|
||||
}
|
||||
|
||||
@@ -1636,6 +1714,7 @@ export class Repository {
|
||||
const params: Record<string, unknown> = { taskId };
|
||||
const fieldMap: Record<string, string> = {
|
||||
title: 'title',
|
||||
titleSource: 'title_source',
|
||||
body: 'body',
|
||||
pieceName: 'piece_name',
|
||||
profile: 'profile',
|
||||
@@ -3490,6 +3569,83 @@ export class Repository {
|
||||
return rows.map(rowToGatewayKeyUsage);
|
||||
}
|
||||
|
||||
// ── Per-user daily LLM usage (gateway + direct) ──────────────────────
|
||||
//
|
||||
// Recorded at the OpenAICompatClient completion boundary for every
|
||||
// successful chat completion. UPSERT on the (day, user_id, source,
|
||||
// model, route) grain. Separate lens from gateway_key_usage — never
|
||||
// summed across the two tables. Spec:
|
||||
// docs/superpowers/specs/2026-06-11-llm-usage-aggregation-design.md
|
||||
|
||||
/**
|
||||
* UPSERT: bump per-(day, user, source, model, route) counters. Deltas
|
||||
* are clamped at zero. `day` defaults to the UTC day of `at` (or now).
|
||||
* Called once per successful stream completion; `usage`-less completions
|
||||
* still bump `requests` (tokens 0) so a 0-token request is distinct from
|
||||
* a failed/aborted one (which is never recorded).
|
||||
*/
|
||||
incrementLlmUsage(params: LlmUsageIncrement): void {
|
||||
const tIn = Math.max(0, Math.floor(params.tokensIn ?? 0));
|
||||
const tOut = Math.max(0, Math.floor(params.tokensOut ?? 0));
|
||||
const reqs = Math.max(0, Math.floor(params.requests ?? 1));
|
||||
const ts = params.at ?? new Date().toISOString();
|
||||
const day = params.day ?? ts.slice(0, 10);
|
||||
this.db
|
||||
.prepare(
|
||||
`INSERT INTO llm_usage_daily
|
||||
(day, user_id, source, model, route, tokens_in, tokens_out, requests, last_updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (day, user_id, source, model, route) DO UPDATE SET
|
||||
tokens_in = tokens_in + excluded.tokens_in,
|
||||
tokens_out = tokens_out + excluded.tokens_out,
|
||||
requests = requests + excluded.requests,
|
||||
last_updated_at = excluded.last_updated_at`,
|
||||
)
|
||||
.run(day, params.userId, params.source, params.model, params.route, tIn, tOut, reqs, ts);
|
||||
}
|
||||
|
||||
/**
|
||||
* Daily time series for the usage dashboard, grouped by (day, user_id,
|
||||
* source) with model/route collapsed. `userId` filter scopes a non-admin
|
||||
* to their own rows; omit it for the admin all-users view (callers can
|
||||
* collapse user_id afterwards). Inclusive `from`/`to` are 'YYYY-MM-DD'.
|
||||
*/
|
||||
queryLlmUsageDaily(opts: { from: string; to: string; userId?: string }): LlmUsageDailyAgg[] {
|
||||
const where = ['day >= ?', 'day <= ?'];
|
||||
const args: unknown[] = [opts.from, opts.to];
|
||||
if (opts.userId !== undefined) {
|
||||
where.push('user_id = ?');
|
||||
args.push(opts.userId);
|
||||
}
|
||||
const rows = this.db
|
||||
.prepare(
|
||||
`SELECT day, user_id, source,
|
||||
SUM(tokens_in) AS tokens_in,
|
||||
SUM(tokens_out) AS tokens_out,
|
||||
SUM(requests) AS requests
|
||||
FROM llm_usage_daily
|
||||
WHERE ${where.join(' AND ')}
|
||||
GROUP BY day, user_id, source
|
||||
ORDER BY day ASC`,
|
||||
)
|
||||
.all(...args) as Array<{
|
||||
day: string;
|
||||
user_id: string;
|
||||
source: string;
|
||||
tokens_in: number;
|
||||
tokens_out: number;
|
||||
requests: number;
|
||||
}>;
|
||||
return rows.map((r) => ({
|
||||
day: r.day,
|
||||
userId: r.user_id,
|
||||
source: r.source,
|
||||
tokensIn: r.tokens_in,
|
||||
tokensOut: r.tokens_out,
|
||||
requests: r.requests,
|
||||
}));
|
||||
}
|
||||
|
||||
/** Return the underlying Database instance (needed by migrate.ts and session store) */
|
||||
getDb(): Database.Database {
|
||||
return this.db;
|
||||
|
||||
+28
-1
@@ -65,7 +65,11 @@ CREATE TABLE IF NOT EXISTS local_tasks (
|
||||
mission_brief TEXT,
|
||||
-- Per-task options (JSON blob). Controls runtime behaviour toggles such as
|
||||
-- { mcpDisabled: true, skillsDisabled: true }. Default '{}' = all enabled.
|
||||
options TEXT DEFAULT '{}'
|
||||
options TEXT DEFAULT '{}',
|
||||
-- Provenance of `title`: 'auto' = cheap fallback set at creation,
|
||||
-- 'agent' = derived from the Mission Brief goal during the run,
|
||||
-- 'user' = manually edited (never overwritten by agent/regeneration).
|
||||
title_source TEXT NOT NULL DEFAULT 'auto'
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_local_tasks_updated_at ON local_tasks (updated_at DESC);
|
||||
@@ -586,6 +590,29 @@ CREATE TABLE IF NOT EXISTS gateway_key_usage (
|
||||
CREATE INDEX IF NOT EXISTS idx_gateway_usage_key
|
||||
ON gateway_key_usage (key_id);
|
||||
|
||||
-- ── LLM usage: per-user daily aggregation (gateway + direct) ────────────
|
||||
-- Daily UPSERT buckets recorded at the OpenAICompatClient completion
|
||||
-- boundary, covering BOTH gateway-routed and direct LLM calls. This is a
|
||||
-- separate lens from gateway_key_usage (which is per-virtual-key / billing).
|
||||
-- grain = (day, user_id, source, model, route); user_id is NOT NULL with a
|
||||
-- 'system' / 'local' sentinel so ON CONFLICT keys never hit the SQLite
|
||||
-- NULL != NULL trap. Spec:
|
||||
-- docs/superpowers/specs/2026-06-11-llm-usage-aggregation-design.md
|
||||
CREATE TABLE IF NOT EXISTS llm_usage_daily (
|
||||
day TEXT NOT NULL, -- 'YYYY-MM-DD' (UTC)
|
||||
user_id TEXT NOT NULL, -- owner id / 'local' / 'system'
|
||||
source TEXT NOT NULL, -- 'gateway' | 'direct'
|
||||
model TEXT NOT NULL, -- real model name (chunk.model), routing key fallback
|
||||
route TEXT NOT NULL, -- backend server name (gateway backendId / direct host)
|
||||
tokens_in INTEGER NOT NULL DEFAULT 0,
|
||||
tokens_out INTEGER NOT NULL DEFAULT 0,
|
||||
requests INTEGER NOT NULL DEFAULT 0,
|
||||
last_updated_at TEXT NOT NULL,
|
||||
PRIMARY KEY (day, user_id, source, model, route)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_llm_usage_daily_user_day
|
||||
ON llm_usage_daily (user_id, day);
|
||||
|
||||
-- ── Browser Notifications V2: Web Push subscriptions + per-user prefs ───
|
||||
-- Spec: docs/superpowers/specs/2026-05-28-browser-notifications-v2-webpush.md
|
||||
-- endpoint is globally UNIQUE so logging into a different user in the same
|
||||
|
||||
@@ -1963,7 +1963,7 @@ export async function executeMovement(
|
||||
{ role: 'user', content: taskInstruction },
|
||||
];
|
||||
const runIsolatedLlm = (isolatedMessages: Message[]): Promise<string> =>
|
||||
runIsolatedLlmHelper(client, isolatedMessages, cancelSignal);
|
||||
runIsolatedLlmHelper(client, isolatedMessages, cancelSignal, { userId: ctx.userId });
|
||||
|
||||
// Traceability T-1: ensure eventLogger is non-undefined for the
|
||||
// duration of the movement. Production callers (piece-runner) always
|
||||
@@ -2166,6 +2166,7 @@ export async function executeMovement(
|
||||
},
|
||||
},
|
||||
`movement=${movement.name} `,
|
||||
{ userId: ctx.userId },
|
||||
);
|
||||
const llmDurationMs = Date.now() - llmStartedAt;
|
||||
let { accumulatedText } = consumed;
|
||||
|
||||
@@ -4,6 +4,7 @@ import type {
|
||||
ToolCall,
|
||||
OpenAICompatClient,
|
||||
LLMEvent,
|
||||
LlmCallContext,
|
||||
} from '../llm/openai-compat.js';
|
||||
import { logger } from '../logger.js';
|
||||
import { stripThinkingTokens } from './strip-thinking.js';
|
||||
@@ -22,9 +23,10 @@ export async function runIsolatedLlm(
|
||||
client: OpenAICompatClient,
|
||||
messages: Message[],
|
||||
cancelSignal?: AbortSignal,
|
||||
context?: LlmCallContext,
|
||||
): Promise<string> {
|
||||
let output = '';
|
||||
for await (const event of client.chat(messages, undefined, cancelSignal)) {
|
||||
for await (const event of client.chat(messages, undefined, cancelSignal, context)) {
|
||||
if (event.type === 'text') {
|
||||
output += event.text;
|
||||
continue;
|
||||
@@ -107,8 +109,9 @@ export async function consumeLlmStream(
|
||||
idleTimeoutMs: number,
|
||||
callbacks: ConsumeStreamCallbacks = {},
|
||||
contextLabel: string = '',
|
||||
context?: LlmCallContext,
|
||||
): Promise<ConsumedLLMResponse> {
|
||||
const stream = client.chat(messages, tools, cancelSignal);
|
||||
const stream = client.chat(messages, tools, cancelSignal, context);
|
||||
const accumulator: ConsumedLLMResponse = {
|
||||
accumulatedText: '',
|
||||
pendingToolCalls: [],
|
||||
|
||||
@@ -79,6 +79,7 @@ export async function classifyPiece(
|
||||
pieces: PieceDescription[],
|
||||
fileNames: string[],
|
||||
timeoutMs: number = 8000,
|
||||
userId?: string,
|
||||
): Promise<string | null> {
|
||||
const prompt = buildClassificationPrompt(taskText, pieces, fileNames);
|
||||
logger.debug(`[piece-classifier] candidates=[${pieces.map(p => p.name).join(', ')}] textLen=${taskText.length}`);
|
||||
@@ -87,7 +88,7 @@ export async function classifyPiece(
|
||||
const llmCall = async (): Promise<string | null> => {
|
||||
let result = '';
|
||||
try {
|
||||
for await (const event of client.chat(messages)) {
|
||||
for await (const event of client.chat(messages, undefined, undefined, { userId })) {
|
||||
if (event.type === 'text') result += event.text;
|
||||
else if (event.type === 'error') return null;
|
||||
else if (event.type === 'done') break;
|
||||
|
||||
@@ -13,26 +13,57 @@ const validResult = {
|
||||
reasoning: 'x',
|
||||
};
|
||||
|
||||
const okResponse = {
|
||||
ok: true,
|
||||
json: () => Promise.resolve({
|
||||
choices: [
|
||||
{
|
||||
message: {
|
||||
tool_calls: [
|
||||
{
|
||||
function: {
|
||||
name: 'submit_reflection',
|
||||
arguments: JSON.stringify(validResult),
|
||||
},
|
||||
},
|
||||
],
|
||||
/**
|
||||
* The reflection client now routes through OpenAICompatClient, which speaks
|
||||
* streaming SSE. Build a fake streaming `Response` that emits the given SSE
|
||||
* `data:` payloads, then `[DONE]`.
|
||||
*/
|
||||
function sseResponse(chunks: unknown[]): Response {
|
||||
const lines = chunks.map((c) => `data: ${JSON.stringify(c)}\n\n`);
|
||||
lines.push('data: [DONE]\n\n');
|
||||
const encoder = new TextEncoder();
|
||||
let i = 0;
|
||||
return {
|
||||
ok: true,
|
||||
status: 200,
|
||||
headers: { get: () => null },
|
||||
body: {
|
||||
getReader: () => ({
|
||||
read: async () =>
|
||||
i < lines.length
|
||||
? { done: false, value: encoder.encode(lines[i++]) }
|
||||
: { done: true, value: undefined },
|
||||
releaseLock: () => {},
|
||||
}),
|
||||
},
|
||||
} as unknown as Response;
|
||||
}
|
||||
|
||||
/** A complete, valid submit_reflection tool-call stream with usage. */
|
||||
function okStream(args: unknown = validResult): Response {
|
||||
return sseResponse([
|
||||
{
|
||||
model: 'test-model',
|
||||
choices: [
|
||||
{
|
||||
delta: { tool_calls: [{ index: 0, id: 'c1', function: { name: 'submit_reflection', arguments: JSON.stringify(args) } }] },
|
||||
finish_reason: null,
|
||||
},
|
||||
},
|
||||
],
|
||||
usage: { prompt_tokens: 42, completion_tokens: 17 },
|
||||
}),
|
||||
};
|
||||
],
|
||||
},
|
||||
{ choices: [{ delta: {}, finish_reason: 'tool_calls' }] },
|
||||
{ choices: [], usage: { prompt_tokens: 42, completion_tokens: 17 } },
|
||||
]);
|
||||
}
|
||||
|
||||
function httpError(status: number, bodyText: string): Response {
|
||||
return {
|
||||
ok: false,
|
||||
status,
|
||||
headers: { get: () => null },
|
||||
text: () => Promise.resolve(bodyText),
|
||||
} as unknown as Response;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
// No real backoff sleeps in tests.
|
||||
@@ -45,7 +76,7 @@ afterEach(() => {
|
||||
|
||||
describe('callReflectionLlm', () => {
|
||||
it('happy path: parses tool_call arguments and extracts token usage', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue(okResponse));
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue(okStream()));
|
||||
|
||||
const result = await callReflectionLlm(cfg, 'system prompt', 'user prompt');
|
||||
|
||||
@@ -59,12 +90,8 @@ describe('callReflectionLlm', () => {
|
||||
|
||||
it('retries a 5xx (backend tool-call parse failure) and succeeds on resample', async () => {
|
||||
const fetchMock = vi.fn()
|
||||
.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
status: 500,
|
||||
text: () => Promise.resolve('{"error":{"message":"Failed to parse input at pos 41: <tool_call>..."}}'),
|
||||
})
|
||||
.mockResolvedValueOnce(okResponse);
|
||||
.mockResolvedValueOnce(httpError(500, '{"error":{"message":"Failed to parse input at pos 41"}}'))
|
||||
.mockResolvedValueOnce(okStream());
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const result = await callReflectionLlm(cfg, 's', 'u');
|
||||
@@ -73,11 +100,7 @@ describe('callReflectionLlm', () => {
|
||||
});
|
||||
|
||||
it('gives up after 3 attempts of persistent 5xx', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue({
|
||||
ok: false,
|
||||
status: 500,
|
||||
text: () => Promise.resolve('parse error'),
|
||||
});
|
||||
const fetchMock = vi.fn().mockResolvedValue(httpError(500, 'parse error'));
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
await expect(callReflectionLlm(cfg, 's', 'u')).rejects.toThrow('HTTP 500');
|
||||
@@ -85,38 +108,62 @@ describe('callReflectionLlm', () => {
|
||||
});
|
||||
|
||||
it('does NOT retry a 4xx (deterministic config error, e.g. invalid api key)', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue({
|
||||
ok: false,
|
||||
status: 401,
|
||||
text: () => Promise.resolve('invalid api key'),
|
||||
});
|
||||
const fetchMock = vi.fn().mockResolvedValue(httpError(401, 'invalid api key'));
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
await expect(callReflectionLlm(cfg, 's', 'u')).rejects.toThrow('HTTP 401');
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('retries when no tool_calls present, then throws after exhaustion', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: () => Promise.resolve({ choices: [{ message: {} }] }),
|
||||
});
|
||||
it('retries when no tool_call present, then throws after exhaustion', async () => {
|
||||
// A stream that yields only text and finishes — no submit_reflection call.
|
||||
const noToolStream = () => sseResponse([
|
||||
{ choices: [{ delta: { content: 'just text' }, finish_reason: 'stop' }] },
|
||||
]);
|
||||
const fetchMock = vi.fn().mockImplementation(async () => noToolStream());
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
await expect(callReflectionLlm(cfg, 'system prompt', 'user prompt'))
|
||||
.rejects.toThrow('no tool_call');
|
||||
.rejects.toThrow('no submit_reflection tool_call');
|
||||
expect(fetchMock).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it('retries malformed tool_call arguments JSON', async () => {
|
||||
it('does NOT retry a budget_exhausted gateway sentinel (fail fast)', async () => {
|
||||
// SSE sentinel error → client yields gatewayErrorType=budget_exhausted.
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
sseResponse([{ error: { type: 'budget_exhausted', message: 'over quota' } }]),
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
await expect(callReflectionLlm(cfg, 's', 'u')).rejects.toThrow('budget_exhausted');
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does NOT retry a prompt-size preflight block (fail fast)', async () => {
|
||||
// Tiny context window forces the client preflight guard to block before
|
||||
// any fetch; resampling the identical prompt cannot help.
|
||||
const fetchMock = vi.fn().mockResolvedValue(okStream());
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
await expect(callReflectionLlm({ ...cfg, contextLimitTokens: 1 }, 'system', 'user'))
|
||||
.rejects.toThrow('blocked before send');
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('retries malformed tool_call arguments (client yields empty input)', async () => {
|
||||
// First stream carries broken JSON args → client parses to {} → structural
|
||||
// guard treats it as malformed → resample. Second stream is valid.
|
||||
const brokenStream = () => sseResponse([
|
||||
{
|
||||
choices: [
|
||||
{ delta: { tool_calls: [{ index: 0, id: 'c1', function: { name: 'submit_reflection', arguments: '{broken' } }] }, finish_reason: null },
|
||||
],
|
||||
},
|
||||
{ choices: [{ delta: {}, finish_reason: 'tool_calls' }] },
|
||||
]);
|
||||
const fetchMock = vi.fn()
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: () => Promise.resolve({
|
||||
choices: [{ message: { tool_calls: [{ function: { name: 'submit_reflection', arguments: '{broken' } }] } }],
|
||||
}),
|
||||
})
|
||||
.mockResolvedValueOnce(okResponse);
|
||||
.mockImplementationOnce(async () => brokenStream())
|
||||
.mockImplementationOnce(async () => okStream());
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const result = await callReflectionLlm(cfg, 's', 'u');
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { logger } from '../../logger.js';
|
||||
import { getDefaultProviderRetryConfig } from '../../config.js';
|
||||
import { OpenAICompatClient, type LLMEvent, type Message, type ToolDef } from '../../llm/openai-compat.js';
|
||||
import type { ReflectionResult } from './types.js';
|
||||
import { REFLECTION_TOOL_SCHEMA } from './reflection-schema.js';
|
||||
|
||||
@@ -6,6 +8,17 @@ export interface ReflectionLlmConfig {
|
||||
endpoint: string;
|
||||
model: string | undefined;
|
||||
apiKey?: string;
|
||||
/** True when the reflection worker routes through the AAO Gateway (proxy). */
|
||||
proxy?: boolean;
|
||||
/** Reflection target user — recorded as the usage owner. */
|
||||
userId?: string;
|
||||
/**
|
||||
* Model context window in tokens. Passed to the shared client's
|
||||
* prompt-size preflight guard. Reflection prompts can be large (uncapped
|
||||
* memory snapshot), so use the worker's real limit rather than the
|
||||
* client's conservative 32k default, which would block valid prompts.
|
||||
*/
|
||||
contextLimitTokens?: number;
|
||||
}
|
||||
|
||||
export interface ReflectionLlmResult {
|
||||
@@ -62,54 +75,104 @@ export async function callReflectionLlm(
|
||||
throw lastErr ?? new Error('reflection LLM failed');
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify an OpenAICompatClient error for the reflection resample loop.
|
||||
* - HTTP 5xx (incl. tool-call parse errors on malformed model output) and
|
||||
* gateway_shutdown / gateway_timeout: transient → resample.
|
||||
* - HTTP 4xx (bad key / request shape), budget_exhausted / rate_limited
|
||||
* (won't pass until the period resets), and the client-side
|
||||
* "blocked before send" prompt-size guard: deterministic → fail fast.
|
||||
* - Everything else (transport / parse / idle timeout): stochastic → resample.
|
||||
*/
|
||||
function classifyClientError(message: string, gatewayErrorType?: string): Error {
|
||||
if (gatewayErrorType === 'budget_exhausted' || gatewayErrorType === 'rate_limited') {
|
||||
return new Error(message);
|
||||
}
|
||||
if (gatewayErrorType === 'gateway_shutdown' || gatewayErrorType === 'gateway_timeout') {
|
||||
return new RetryableLlmError(message);
|
||||
}
|
||||
// Client-side preflight rejection — the prompt is too large; resampling the
|
||||
// identical prompt cannot help.
|
||||
if (message.includes('blocked before send')) {
|
||||
return new Error(message);
|
||||
}
|
||||
const m = /HTTP (\d{3})/.exec(message);
|
||||
if (m) {
|
||||
const status = Number(m[1]);
|
||||
if (status >= 500) return new RetryableLlmError(message);
|
||||
return new Error(message);
|
||||
}
|
||||
return new RetryableLlmError(message);
|
||||
}
|
||||
|
||||
async function callOnce(
|
||||
cfg: ReflectionLlmConfig,
|
||||
systemPrompt: string,
|
||||
userPrompt: string,
|
||||
start: number,
|
||||
): Promise<ReflectionLlmResult> {
|
||||
const body: Record<string, unknown> = {
|
||||
messages: [
|
||||
{ role: 'system', content: systemPrompt },
|
||||
{ role: 'user', content: userPrompt },
|
||||
],
|
||||
tools: [REFLECTION_TOOL_SCHEMA],
|
||||
tool_choice: { type: 'function', function: { name: 'submit_reflection' } },
|
||||
temperature: 0.2,
|
||||
};
|
||||
if (cfg.model) {
|
||||
body['model'] = cfg.model;
|
||||
// Route through the shared client so usage lands in the single
|
||||
// per-user ledger (gateway + direct) like every other LLM call.
|
||||
// maxAttempts=1: the outer callReflectionLlm loop owns resampling.
|
||||
const client = new OpenAICompatClient(
|
||||
cfg.endpoint,
|
||||
cfg.model,
|
||||
cfg.apiKey,
|
||||
{ ...getDefaultProviderRetryConfig(), maxAttempts: 1 },
|
||||
undefined,
|
||||
cfg.contextLimitTokens, // real model window; avoid the 32k default blocking large reflection prompts
|
||||
undefined,
|
||||
undefined,
|
||||
{ proxy: cfg.proxy === true },
|
||||
);
|
||||
const messages: Message[] = [
|
||||
{ role: 'system', content: systemPrompt },
|
||||
{ role: 'user', content: userPrompt },
|
||||
];
|
||||
|
||||
let parsed: ReflectionResult | null = null;
|
||||
let usage: { prompt_tokens: number; completion_tokens: number } | undefined;
|
||||
let errorMsg: string | null = null;
|
||||
let errorGatewayType: string | undefined;
|
||||
|
||||
for await (const event of client.chat(
|
||||
messages,
|
||||
[REFLECTION_TOOL_SCHEMA as unknown as ToolDef],
|
||||
undefined,
|
||||
{ userId: cfg.userId },
|
||||
{ temperature: 0.2, toolChoice: { type: 'function', function: { name: 'submit_reflection' } } },
|
||||
) as AsyncGenerator<LLMEvent>) {
|
||||
if (event.type === 'tool_use') {
|
||||
if (event.name === 'submit_reflection' && parsed === null) {
|
||||
parsed = event.input as unknown as ReflectionResult;
|
||||
}
|
||||
} else if (event.type === 'done') {
|
||||
usage = event.usage;
|
||||
} else if (event.type === 'error') {
|
||||
errorMsg = event.error;
|
||||
errorGatewayType = event.gatewayErrorType;
|
||||
}
|
||||
}
|
||||
const resp = await fetch(`${cfg.endpoint}/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
...(cfg.apiKey ? { authorization: `Bearer ${cfg.apiKey}` } : {}),
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
if (!resp.ok) {
|
||||
const text = await resp.text();
|
||||
const msg = `reflection LLM HTTP ${resp.status}: ${text}`;
|
||||
// 5xx: backend-side failure (incl. tool-call parse errors on malformed
|
||||
// model output) — resample. 4xx: deterministic config error — fail fast.
|
||||
if (resp.status >= 500) throw new RetryableLlmError(msg);
|
||||
throw new Error(msg);
|
||||
|
||||
if (errorMsg !== null) {
|
||||
throw classifyClientError(`reflection LLM ${errorMsg}`, errorGatewayType);
|
||||
}
|
||||
const data = await resp.json() as any;
|
||||
const toolCall = data.choices?.[0]?.message?.tool_calls?.[0];
|
||||
if (!toolCall) throw new RetryableLlmError('reflection LLM returned no tool_call');
|
||||
let parsed: ReflectionResult;
|
||||
try {
|
||||
parsed = JSON.parse(toolCall.function.arguments) as ReflectionResult;
|
||||
} catch {
|
||||
throw new RetryableLlmError('reflection LLM tool_call arguments were not valid JSON');
|
||||
if (parsed === null) {
|
||||
throw new RetryableLlmError('reflection LLM returned no submit_reflection tool_call');
|
||||
}
|
||||
// The shared client swallows tool-argument JSON parse errors and yields an
|
||||
// empty `{}` input. Preserve the old resample-on-malformed behaviour with a
|
||||
// shallow structural check against the tool schema's required fields — a
|
||||
// genuinely-empty object means the model emitted broken tool markup.
|
||||
const p = parsed as unknown as Record<string, unknown>;
|
||||
if (p['piece_changes'] === undefined || p['reasoning'] === undefined) {
|
||||
throw new RetryableLlmError('reflection LLM tool_call arguments were malformed or incomplete');
|
||||
}
|
||||
return {
|
||||
parsed,
|
||||
tokensIn: data.usage?.prompt_tokens ?? 0,
|
||||
tokensOut: data.usage?.completion_tokens ?? 0,
|
||||
tokensIn: usage?.prompt_tokens ?? 0,
|
||||
tokensOut: usage?.completion_tokens ?? 0,
|
||||
durationMs: Date.now() - start,
|
||||
raw: data,
|
||||
raw: { usage },
|
||||
};
|
||||
}
|
||||
|
||||
@@ -24,6 +24,10 @@ export interface RunReflectionDeps {
|
||||
* (normal task calls always send the worker's key — reflection must too).
|
||||
*/
|
||||
llmApiKey?: string;
|
||||
/** True when the reflection worker routes through the AAO Gateway (proxy). */
|
||||
llmProxy?: boolean;
|
||||
/** Reflection worker's model context window (tokens) for the prompt guard. */
|
||||
llmContextLimitTokens?: number;
|
||||
}
|
||||
|
||||
export async function runReflectionJob(
|
||||
@@ -86,6 +90,9 @@ export async function runReflectionJob(
|
||||
endpoint: deps.llmEndpoint,
|
||||
model: deps.llmModel,
|
||||
apiKey: deps.llmApiKey,
|
||||
proxy: deps.llmProxy === true,
|
||||
userId: meta.userId,
|
||||
contextLimitTokens: deps.llmContextLimitTokens,
|
||||
};
|
||||
|
||||
let llmResult;
|
||||
|
||||
Binary file not shown.
@@ -51,6 +51,7 @@ export interface ToolsConfig {
|
||||
officePdfMaxSizeMb?: number; // ReadPdf の最大ファイルサイズ (default: 10)
|
||||
officePptxMaxSizeMb?: number; // ReadPPTX の最大ファイルサイズ (default: 50)
|
||||
officePptxMaxUncompressedMb?: number; // ReadPPTX の ZIP 展開後サイズ上限 (default: 200)
|
||||
officeMsgMaxSizeMb?: number; // ReadMsg の最大ファイルサイズ (default: 25)
|
||||
webfetchScreenshot?: boolean; // WebFetch で vlmEnabled 時にスクショを添付するか (default: true)
|
||||
webfetchScreenshotTimeoutMs?: number; // スクショ取得のタイムアウト (default: 15000)
|
||||
}
|
||||
|
||||
@@ -52,6 +52,7 @@ const TOOL_DOC_ALIASES: Record<string, string> = {
|
||||
readexcel: 'office',
|
||||
readdocx: 'office',
|
||||
readpptx: 'office',
|
||||
readmsg: 'office',
|
||||
pdftoimages: 'office',
|
||||
splitexcelsheets: 'office',
|
||||
splitdocxsections: 'office',
|
||||
|
||||
@@ -0,0 +1,324 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import {
|
||||
formatAddress,
|
||||
stripHtml,
|
||||
selectMsgBody,
|
||||
sanitizeAttachmentName,
|
||||
formatMsgOutput,
|
||||
assembleMsgOutput,
|
||||
pickEmail,
|
||||
isParsedMsgValid,
|
||||
executeReadMsg,
|
||||
type MsgView,
|
||||
} from './msg.js';
|
||||
import type { ToolContext } from './core.js';
|
||||
import { executeTool as officeExecuteTool, TOOL_DEFS as OFFICE_TOOL_DEFS } from './office.js';
|
||||
|
||||
const FIXTURE = path.join(
|
||||
path.dirname(fileURLToPath(import.meta.url)),
|
||||
'__fixtures__',
|
||||
'attachmentFiles.msg',
|
||||
);
|
||||
|
||||
describe('formatAddress', () => {
|
||||
it('renders name and email together', () => {
|
||||
expect(formatAddress({ name: 'Alice', email: '[email protected]' })).toBe(
|
||||
'Alice <[email protected]>',
|
||||
);
|
||||
});
|
||||
|
||||
it('renders name only when email is missing', () => {
|
||||
expect(formatAddress({ name: 'Alice' })).toBe('Alice');
|
||||
});
|
||||
|
||||
it('renders email only when name is missing', () => {
|
||||
expect(formatAddress({ email: '[email protected]' })).toBe('[email protected]');
|
||||
});
|
||||
|
||||
it('falls back to a placeholder when both are missing', () => {
|
||||
expect(formatAddress({})).toBe('(unknown)');
|
||||
});
|
||||
});
|
||||
|
||||
describe('stripHtml', () => {
|
||||
it('removes tags and decodes entities', () => {
|
||||
expect(stripHtml('<p>Hello <b>world</b> & co</p>')).toBe('Hello world & co');
|
||||
});
|
||||
|
||||
it('drops script and style content', () => {
|
||||
const html = '<style>.x{color:red}</style><p>Keep</p><script>alert(1)</script>';
|
||||
expect(stripHtml(html)).toBe('Keep');
|
||||
});
|
||||
|
||||
it('turns block boundaries into newlines', () => {
|
||||
expect(stripHtml('<div>line1</div><div>line2</div>')).toBe('line1\nline2');
|
||||
});
|
||||
|
||||
it('decodes valid numeric entities', () => {
|
||||
expect(stripHtml('<p>AB</p>')).toBe('AB');
|
||||
});
|
||||
|
||||
it('does not throw on out-of-range numeric entities', () => {
|
||||
expect(() => stripHtml('<p>�</p>')).not.toThrow();
|
||||
expect(stripHtml('A�B')).toBe('A�B');
|
||||
});
|
||||
});
|
||||
|
||||
describe('selectMsgBody', () => {
|
||||
it('prefers the plain-text body', () => {
|
||||
expect(selectMsgBody({ body: 'plain text', bodyHtml: '<p>html</p>' })).toEqual({
|
||||
text: 'plain text',
|
||||
format: 'plain',
|
||||
});
|
||||
});
|
||||
|
||||
it('falls back to stripped HTML when no plain body exists', () => {
|
||||
expect(selectMsgBody({ bodyHtml: '<p>html body</p>' })).toEqual({
|
||||
text: 'html body',
|
||||
format: 'html',
|
||||
});
|
||||
});
|
||||
|
||||
it('reports none when no body is present', () => {
|
||||
expect(selectMsgBody({})).toEqual({ text: '', format: 'none' });
|
||||
});
|
||||
|
||||
it('decodes PidTagHtml (html) when body and bodyHtml are absent', () => {
|
||||
const html = new TextEncoder().encode('<p>from pidtag</p>');
|
||||
expect(selectMsgBody({ html })).toEqual({ text: 'from pidtag', format: 'html' });
|
||||
});
|
||||
|
||||
it('prefers plain body over the PidTagHtml field', () => {
|
||||
const html = new TextEncoder().encode('<p>html</p>');
|
||||
expect(selectMsgBody({ body: 'plain', html })).toEqual({ text: 'plain', format: 'plain' });
|
||||
});
|
||||
|
||||
it('falls back to PidTagHtml when bodyHtml is empty/whitespace', () => {
|
||||
const html = new TextEncoder().encode('<p>pidtag body</p>');
|
||||
expect(selectMsgBody({ bodyHtml: ' ', html })).toEqual({
|
||||
text: 'pidtag body',
|
||||
format: 'html',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('pickEmail', () => {
|
||||
it('prefers a real SMTP address over a legacy EX DN', () => {
|
||||
expect(pickEmail('/O=EX/OU=x/CN=alice', '[email protected]')).toBe('[email protected]');
|
||||
expect(pickEmail('[email protected]', '/O=EX/OU=x/CN=alice')).toBe('[email protected]');
|
||||
});
|
||||
|
||||
it('falls back to the EX DN when no SMTP-looking address exists', () => {
|
||||
expect(pickEmail(undefined, '/O=EX/OU=x/CN=alice')).toBe('/O=EX/OU=x/CN=alice');
|
||||
});
|
||||
|
||||
it('returns undefined when nothing usable is provided', () => {
|
||||
expect(pickEmail(undefined, undefined)).toBeUndefined();
|
||||
expect(pickEmail('', ' ')).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('isParsedMsgValid', () => {
|
||||
it('accepts a parsed Outlook message', () => {
|
||||
expect(isParsedMsgValid({ dataType: 'msg' })).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects an unsupported CFBF result (old .doc/.xls, corrupted compound file)', () => {
|
||||
expect(isParsedMsgValid({ error: 'Unsupported file type!', dataType: null })).toBe(false);
|
||||
expect(isParsedMsgValid({ dataType: null })).toBe(false);
|
||||
expect(isParsedMsgValid({ dataType: 'attachment' })).toBe(false);
|
||||
});
|
||||
|
||||
it('treats a whitespace-only plain body as empty and uses HTML', () => {
|
||||
expect(selectMsgBody({ body: ' \n ', bodyHtml: '<p>real</p>' })).toEqual({
|
||||
text: 'real',
|
||||
format: 'html',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('sanitizeAttachmentName', () => {
|
||||
it('keeps a normal filename unchanged', () => {
|
||||
expect(sanitizeAttachmentName('report.pdf', 0)).toBe('report.pdf');
|
||||
});
|
||||
|
||||
it('strips directory components to prevent path traversal', () => {
|
||||
expect(sanitizeAttachmentName('../../etc/passwd', 0)).toBe('passwd');
|
||||
expect(sanitizeAttachmentName('foo/bar/baz.txt', 0)).toBe('baz.txt');
|
||||
expect(sanitizeAttachmentName('a\\b\\c.doc', 0)).toBe('c.doc');
|
||||
});
|
||||
|
||||
it('removes control characters and null bytes', () => {
|
||||
expect(sanitizeAttachmentName('na\x00me.txt', 0)).toBe('name.txt');
|
||||
expect(sanitizeAttachmentName('tab\tname.txt', 0)).toBe('tabname.txt');
|
||||
});
|
||||
|
||||
it('preserves spaces inside the filename', () => {
|
||||
expect(sanitizeAttachmentName('my report.pdf', 0)).toBe('my report.pdf');
|
||||
});
|
||||
|
||||
it('falls back to an indexed name when the result is empty', () => {
|
||||
expect(sanitizeAttachmentName('', 2)).toBe('attachment-3');
|
||||
expect(sanitizeAttachmentName('...', 0)).toBe('attachment-1');
|
||||
});
|
||||
});
|
||||
|
||||
describe('formatMsgOutput', () => {
|
||||
const baseView: MsgView = {
|
||||
subject: 'Quarterly report',
|
||||
from: { name: 'Alice', email: '[email protected]' },
|
||||
to: [{ name: 'Bob', email: '[email protected]' }],
|
||||
cc: [],
|
||||
date: 'Mon, 1 Jun 2026 10:00:00 +0900',
|
||||
body: { text: 'See attached.', format: 'plain' },
|
||||
attachments: [],
|
||||
};
|
||||
|
||||
it('renders the header block and body', () => {
|
||||
const out = formatMsgOutput(baseView);
|
||||
expect(out).toContain('Subject: Quarterly report');
|
||||
expect(out).toContain('From: Alice <[email protected]>');
|
||||
expect(out).toContain('To: Bob <[email protected]>');
|
||||
expect(out).toContain('Date: Mon, 1 Jun 2026 10:00:00 +0900');
|
||||
expect(out).toContain('See attached.');
|
||||
});
|
||||
|
||||
it('lists saved attachments with their paths and sizes', () => {
|
||||
const out = formatMsgOutput({
|
||||
...baseView,
|
||||
attachments: [{ fileName: 'report.pdf', contentLength: 2048, savedPath: 'input/report.pdf' }],
|
||||
});
|
||||
expect(out).toContain('Attachments (1)');
|
||||
expect(out).toContain('report.pdf');
|
||||
expect(out).toContain('input/report.pdf');
|
||||
expect(out).toContain('2048');
|
||||
});
|
||||
|
||||
it('shows a skip reason for attachments that were not saved', () => {
|
||||
const out = formatMsgOutput({
|
||||
...baseView,
|
||||
attachments: [{ fileName: 'huge.bin', skipped: 'exceeds size limit' }],
|
||||
});
|
||||
expect(out).toContain('huge.bin');
|
||||
expect(out).toContain('exceeds size limit');
|
||||
});
|
||||
|
||||
it('notes when the body could not be extracted', () => {
|
||||
const out = formatMsgOutput({ ...baseView, body: { text: '', format: 'none' } });
|
||||
expect(out).toContain('(no text body)');
|
||||
});
|
||||
|
||||
it('omits the CC line when there are no CC recipients', () => {
|
||||
expect(formatMsgOutput(baseView)).not.toContain('Cc:');
|
||||
});
|
||||
|
||||
it('keeps the attachment list when the body is truncated to budget', () => {
|
||||
const longBody = 'word '.repeat(20000);
|
||||
const out = assembleMsgOutput(
|
||||
{
|
||||
...baseView,
|
||||
body: { text: longBody, format: 'plain' },
|
||||
attachments: [{ fileName: 'a.pdf', contentLength: 10, savedPath: 'input/a.pdf' }],
|
||||
},
|
||||
100,
|
||||
'mail.msg',
|
||||
);
|
||||
expect(out).toContain('input/a.pdf');
|
||||
expect(out).toContain('Subject: Quarterly report');
|
||||
expect(out.length).toBeLessThan(longBody.length);
|
||||
});
|
||||
|
||||
it('includes the CC line when CC recipients exist', () => {
|
||||
const out = formatMsgOutput({ ...baseView, cc: [{ email: '[email protected]' }] });
|
||||
expect(out).toContain('Cc: [email protected]');
|
||||
});
|
||||
});
|
||||
|
||||
describe('executeReadMsg (integration)', () => {
|
||||
let workspace: string;
|
||||
const ctx = (): ToolContext => ({ workspacePath: workspace, editAllowed: true });
|
||||
|
||||
beforeEach(() => {
|
||||
workspace = fs.mkdtempSync(path.join(os.tmpdir(), 'readmsg-'));
|
||||
fs.copyFileSync(FIXTURE, path.join(workspace, 'mail.msg'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(workspace, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('extracts headers and body from a real .msg file', async () => {
|
||||
const result = await executeReadMsg({ file_path: 'mail.msg' }, ctx());
|
||||
expect(result.isError).toBeFalsy();
|
||||
expect(result.output).toContain('Subject: attachmentFiles');
|
||||
expect(result.output).toContain('From: hmailuser <[email protected]>');
|
||||
expect(result.output).toContain('To: [email protected]');
|
||||
expect(result.output).toContain('attachmentFiles');
|
||||
});
|
||||
|
||||
it('saves attachments to input/ and lists them', async () => {
|
||||
const result = await executeReadMsg({ file_path: 'mail.msg' }, ctx());
|
||||
expect(result.output).toContain('Attachments (3)');
|
||||
for (const [name, size] of [
|
||||
['jpg.jpg', 726],
|
||||
['png.png', 134],
|
||||
['tif.tif', 664],
|
||||
] as const) {
|
||||
const saved = path.join(workspace, 'input', name);
|
||||
expect(fs.existsSync(saved)).toBe(true);
|
||||
expect(fs.statSync(saved).size).toBe(size);
|
||||
expect(result.output).toContain(path.join('input', name));
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects paths outside the workspace', async () => {
|
||||
const result = await executeReadMsg({ file_path: '../../etc/passwd' }, ctx());
|
||||
expect(result.isError).toBe(true);
|
||||
});
|
||||
|
||||
it('reports a clear error for a non-.msg file', async () => {
|
||||
fs.writeFileSync(path.join(workspace, 'junk.msg'), 'not a real msg file');
|
||||
const result = await executeReadMsg({ file_path: 'junk.msg' }, ctx());
|
||||
expect(result.isError).toBe(true);
|
||||
expect(result.output).toContain('ReadMsg');
|
||||
});
|
||||
|
||||
it('does not write attachments in a read-only phase', async () => {
|
||||
const result = await executeReadMsg(
|
||||
{ file_path: 'mail.msg' },
|
||||
{ workspacePath: workspace, editAllowed: false },
|
||||
);
|
||||
expect(result.isError).toBeFalsy();
|
||||
expect(fs.existsSync(path.join(workspace, 'input', 'jpg.jpg'))).toBe(false);
|
||||
expect(result.output).toContain('read-only');
|
||||
});
|
||||
|
||||
it('does not overwrite an existing file in input/', async () => {
|
||||
fs.mkdirSync(path.join(workspace, 'input'), { recursive: true });
|
||||
fs.writeFileSync(path.join(workspace, 'input', 'jpg.jpg'), 'pre-existing');
|
||||
const result = await executeReadMsg({ file_path: 'mail.msg' }, ctx());
|
||||
expect(fs.readFileSync(path.join(workspace, 'input', 'jpg.jpg'), 'utf8')).toBe('pre-existing');
|
||||
expect(fs.existsSync(path.join(workspace, 'input', 'jpg-1.jpg'))).toBe(true);
|
||||
expect(result.output).toContain('jpg-1.jpg');
|
||||
});
|
||||
|
||||
it('rejects files exceeding the configured size limit', async () => {
|
||||
const result = await executeReadMsg(
|
||||
{ file_path: 'mail.msg' },
|
||||
{ workspacePath: workspace, editAllowed: true, toolsConfig: { officeMsgMaxSizeMb: 0.001 } },
|
||||
);
|
||||
expect(result.isError).toBe(true);
|
||||
expect(result.output).toMatch(/size|limit|too large/i);
|
||||
});
|
||||
|
||||
it('is registered and routed through the office module dispatch', async () => {
|
||||
expect(OFFICE_TOOL_DEFS.ReadMsg).toBeDefined();
|
||||
const result = await officeExecuteTool('ReadMsg', { file_path: 'mail.msg' }, ctx());
|
||||
expect(result?.isError).toBeFalsy();
|
||||
expect(result?.output).toContain('Subject: attachmentFiles');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,416 @@
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import MsgReaderImport from '@kenjiuno/msgreader';
|
||||
import { ToolDef } from '../../llm/openai-compat.js';
|
||||
import type { ToolContext, ToolResult } from './core.js';
|
||||
import { resolveAndGuard, truncateToBudget, getToolOutputBudgetTokens } from './core.js';
|
||||
import { logger } from '../../logger.js';
|
||||
|
||||
// CJS/ESM interop: under native Node ESM (the built dist), a default import of
|
||||
// this CommonJS package resolves to the module.exports namespace object, not the
|
||||
// class — so `new MsgReaderImport()` throws "is not a constructor". Vitest/tsx
|
||||
// hide this via __esModule interop. Pick the real constructor for both worlds.
|
||||
const MsgReader = (
|
||||
typeof MsgReaderImport === 'function'
|
||||
? MsgReaderImport
|
||||
: (MsgReaderImport as unknown as { default: typeof MsgReaderImport }).default
|
||||
) as typeof MsgReaderImport;
|
||||
type MsgReaderInstance = InstanceType<typeof MsgReader>;
|
||||
|
||||
const DEFAULT_MSG_MAX_SIZE_MB = 25;
|
||||
|
||||
export interface MsgAddress {
|
||||
name?: string;
|
||||
email?: string;
|
||||
}
|
||||
|
||||
export interface MsgAttachmentMeta {
|
||||
fileName: string;
|
||||
contentLength?: number;
|
||||
/** Relative path the attachment was written to (when saved). */
|
||||
savedPath?: string;
|
||||
/** Reason the attachment was not saved (mutually exclusive with savedPath). */
|
||||
skipped?: string;
|
||||
}
|
||||
|
||||
export interface MsgView {
|
||||
subject?: string;
|
||||
from?: MsgAddress;
|
||||
to: MsgAddress[];
|
||||
cc: MsgAddress[];
|
||||
date?: string;
|
||||
body: { text: string; format: 'plain' | 'html' | 'none' };
|
||||
attachments: MsgAttachmentMeta[];
|
||||
}
|
||||
|
||||
/** Render a single address as `Name <email>`, falling back gracefully. */
|
||||
export function formatAddress(a: MsgAddress): string {
|
||||
const name = a.name?.trim();
|
||||
const email = a.email?.trim();
|
||||
if (name && email) return `${name} <${email}>`;
|
||||
if (name) return name;
|
||||
if (email) return email;
|
||||
return '(unknown)';
|
||||
}
|
||||
|
||||
const NAMED_ENTITIES: Record<string, string> = {
|
||||
' ': ' ',
|
||||
'&': '&',
|
||||
'<': '<',
|
||||
'>': '>',
|
||||
'"': '"',
|
||||
''': "'",
|
||||
''': "'",
|
||||
};
|
||||
|
||||
// Decode a numeric character reference, preserving the original entity if the
|
||||
// code point is out of range (broken email HTML must not crash the whole read).
|
||||
function safeFromCodePoint(code: number, original: string): string {
|
||||
if (!Number.isFinite(code) || code < 0 || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff)) {
|
||||
return original;
|
||||
}
|
||||
try {
|
||||
return String.fromCodePoint(code);
|
||||
} catch {
|
||||
return original;
|
||||
}
|
||||
}
|
||||
|
||||
function decodeEntities(s: string): string {
|
||||
let out = s.replace(/ |&|<|>|"|'|'/g, (m) => NAMED_ENTITIES[m]);
|
||||
out = out.replace(/&#(\d+);/g, (m, code) => safeFromCodePoint(Number(code), m));
|
||||
out = out.replace(/&#x([0-9a-fA-F]+);/g, (m, code) => safeFromCodePoint(parseInt(code, 16), m));
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Convert an HTML fragment into readable plain text. */
|
||||
export function stripHtml(html: string): string {
|
||||
let s = html;
|
||||
// Drop script/style blocks including their contents.
|
||||
s = s.replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1>/gi, '');
|
||||
// Treat <br> and block-level boundaries as newlines.
|
||||
s = s.replace(/<br\s*\/?>/gi, '\n');
|
||||
s = s.replace(/<\/(p|div|li|tr|h[1-6]|ul|ol|table|blockquote|section|article)\s*>/gi, '\n');
|
||||
// Remove all remaining tags.
|
||||
s = s.replace(/<[^>]+>/g, '');
|
||||
s = decodeEntities(s);
|
||||
// Normalize whitespace: collapse intra-line runs, trim each line, collapse blank runs.
|
||||
s = s
|
||||
.split('\n')
|
||||
.map((line) => line.replace(/[^\S\n]+/g, ' ').trim())
|
||||
.join('\n')
|
||||
.replace(/\n{3,}/g, '\n\n')
|
||||
.trim();
|
||||
return s;
|
||||
}
|
||||
|
||||
/** Choose the best available body text, preferring plain over HTML. */
|
||||
export function selectMsgBody(fields: {
|
||||
body?: string;
|
||||
bodyHtml?: string;
|
||||
// PidTagHtml: some HTML-only messages carry the body here as raw bytes.
|
||||
html?: Uint8Array | string;
|
||||
}): {
|
||||
text: string;
|
||||
format: 'plain' | 'html' | 'none';
|
||||
} {
|
||||
const plain = fields.body?.trim();
|
||||
if (plain) return { text: plain, format: 'plain' };
|
||||
// Try each HTML source in order; an empty/whitespace bodyHtml must not block
|
||||
// the PidTagHtml fallback, so we check the stripped result of each.
|
||||
const htmlSources = [
|
||||
fields.bodyHtml,
|
||||
fields.html != null
|
||||
? typeof fields.html === 'string'
|
||||
? fields.html
|
||||
: Buffer.from(fields.html).toString('utf8')
|
||||
: undefined,
|
||||
];
|
||||
for (const source of htmlSources) {
|
||||
if (!source) continue;
|
||||
const stripped = stripHtml(source);
|
||||
if (stripped) return { text: stripped, format: 'html' };
|
||||
}
|
||||
return { text: '', format: 'none' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick the most usable email address from candidates, preferring a real SMTP
|
||||
* address (contains '@') over an Exchange legacy EX DN (`/O=.../CN=...`).
|
||||
*/
|
||||
export function pickEmail(...candidates: (string | undefined)[]): string | undefined {
|
||||
const valid = candidates.map((c) => c?.trim()).filter((c): c is string => !!c);
|
||||
return valid.find((c) => c.includes('@')) ?? valid[0];
|
||||
}
|
||||
|
||||
/** Reduce an attachment name to a safe basename, never escaping the target dir. */
|
||||
export function sanitizeAttachmentName(name: string, index: number): string {
|
||||
// Take the last path segment across both separators (defends path traversal).
|
||||
const base = name.split(/[/\\]/).pop() ?? '';
|
||||
// Strip control characters and null bytes.
|
||||
// eslint-disable-next-line no-control-regex
|
||||
const cleaned = base.replace(/[\x00-\x1f\x7f]/g, '').trim();
|
||||
// Reject names that are empty or consist only of dots/spaces.
|
||||
if (!cleaned || /^[.\s]*$/.test(cleaned)) {
|
||||
return `attachment-${index + 1}`;
|
||||
}
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
/** Find a filename that collides with neither this run nor existing files on disk. */
|
||||
function resolveFreeName(dir: string, name: string, used: Set<string>): string {
|
||||
const ext = path.extname(name);
|
||||
const stem = name.slice(0, name.length - ext.length);
|
||||
let candidate = name;
|
||||
let n = 1;
|
||||
while (used.has(candidate) || fs.existsSync(path.join(dir, candidate))) {
|
||||
candidate = `${stem}-${n}${ext}`;
|
||||
n += 1;
|
||||
}
|
||||
return candidate;
|
||||
}
|
||||
|
||||
/** Build the human-readable text output for a parsed message. */
|
||||
export function formatMsgOutput(view: MsgView): string {
|
||||
const lines: string[] = [];
|
||||
if (view.subject) lines.push(`Subject: ${view.subject}`);
|
||||
if (view.from) lines.push(`From: ${formatAddress(view.from)}`);
|
||||
if (view.to.length) lines.push(`To: ${view.to.map(formatAddress).join(', ')}`);
|
||||
if (view.cc.length) lines.push(`Cc: ${view.cc.map(formatAddress).join(', ')}`);
|
||||
if (view.date) lines.push(`Date: ${view.date}`);
|
||||
|
||||
const parts: string[] = [lines.join('\n')];
|
||||
|
||||
parts.push(view.body.format === 'none' ? '(no text body)' : view.body.text);
|
||||
|
||||
if (view.attachments.length) {
|
||||
const attLines = [`Attachments (${view.attachments.length}):`];
|
||||
for (const att of view.attachments) {
|
||||
if (att.savedPath) {
|
||||
const size = att.contentLength != null ? ` (${att.contentLength} bytes)` : '';
|
||||
attLines.push(`- ${att.fileName}${size} -> ${att.savedPath}`);
|
||||
} else {
|
||||
attLines.push(`- ${att.fileName} - skipped: ${att.skipped ?? 'not saved'}`);
|
||||
}
|
||||
}
|
||||
parts.push(attLines.join('\n'));
|
||||
}
|
||||
|
||||
return parts.join('\n\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Assemble the final output, truncating ONLY the body to the token budget.
|
||||
* Headers and the attachment list (with saved input/ paths) always survive —
|
||||
* attachments are already written to disk and the caller needs their paths.
|
||||
*/
|
||||
export function assembleMsgOutput(view: MsgView, budgetTokens: number, sourceLabel: string): string {
|
||||
const shell = formatMsgOutput({ ...view, body: { text: '', format: 'plain' } });
|
||||
const reserveTokens = Math.ceil(shell.length / 4) + 64;
|
||||
const bodyBudget = Math.max(500, budgetTokens - reserveTokens);
|
||||
const bodyText = view.body.format === 'none' ? '' : view.body.text;
|
||||
const truncatedBody = truncateToBudget(bodyText, bodyBudget, { sourceLabel }).text;
|
||||
return formatMsgOutput({ ...view, body: { text: truncatedBody, format: view.body.format } });
|
||||
}
|
||||
|
||||
/**
|
||||
* msgreader's getFileData() returns `{ error: 'Unsupported file type!' }` (not a
|
||||
* throw) for CFBF files that aren't Outlook messages (legacy .doc/.xls, broken
|
||||
* compound files). Treat anything whose root isn't a 'msg' as a read failure.
|
||||
*/
|
||||
export function isParsedMsgValid(fields: { error?: string; dataType?: string | null }): boolean {
|
||||
return !fields.error && fields.dataType === 'msg';
|
||||
}
|
||||
|
||||
export const READ_MSG_DEF: ToolDef = {
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'ReadMsg',
|
||||
description:
|
||||
'Read an Outlook .msg email file, extracting subject/sender/recipients/body and saving attachments to input/. 詳細は ReadToolDoc({ name: "ReadMsg" }) で取得可能。',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
file_path: { type: 'string', description: 'Path to the .msg file' },
|
||||
},
|
||||
required: ['file_path'],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
interface FieldsLike {
|
||||
error?: string;
|
||||
dataType?: string | null;
|
||||
subject?: string;
|
||||
senderName?: string;
|
||||
senderEmail?: string;
|
||||
senderSmtpAddress?: string;
|
||||
body?: string;
|
||||
bodyHtml?: string;
|
||||
html?: Uint8Array;
|
||||
messageDeliveryTime?: string;
|
||||
clientSubmitTime?: string;
|
||||
recipients?: { name?: string; email?: string; smtpAddress?: string; recipType?: string }[];
|
||||
attachments?: {
|
||||
fileName?: string;
|
||||
fileNameShort?: string;
|
||||
contentLength?: number;
|
||||
innerMsgContent?: boolean;
|
||||
dataType?: string | null;
|
||||
}[];
|
||||
}
|
||||
|
||||
export async function executeReadMsg(
|
||||
input: Record<string, unknown>,
|
||||
ctx: ToolContext,
|
||||
): Promise<ToolResult> {
|
||||
const filePath = String(input.file_path ?? '');
|
||||
if (!filePath) {
|
||||
return { output: 'ReadMsg: file_path is required', isError: true };
|
||||
}
|
||||
|
||||
let resolved: string;
|
||||
try {
|
||||
resolved = resolveAndGuard(ctx.workspacePath, filePath);
|
||||
} catch (e) {
|
||||
return { output: `ReadMsg: ${(e as Error).message}`, isError: true };
|
||||
}
|
||||
|
||||
// Enforce a size cap before loading the whole file into memory (matches the
|
||||
// other office tools, which each guard against oversized inputs).
|
||||
const maxMb =
|
||||
typeof ctx.toolsConfig?.officeMsgMaxSizeMb === 'number' &&
|
||||
Number.isFinite(ctx.toolsConfig.officeMsgMaxSizeMb) &&
|
||||
ctx.toolsConfig.officeMsgMaxSizeMb > 0
|
||||
? ctx.toolsConfig.officeMsgMaxSizeMb
|
||||
: DEFAULT_MSG_MAX_SIZE_MB;
|
||||
try {
|
||||
const sizeMb = fs.statSync(resolved).size / 1024 / 1024;
|
||||
if (sizeMb > maxMb) {
|
||||
return {
|
||||
output: `ReadMsg: file size ${sizeMb.toFixed(1)}MB exceeds limit of ${maxMb}MB`,
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
} catch (e) {
|
||||
return { output: `ReadMsg: cannot stat file: ${(e as Error).message}`, isError: true };
|
||||
}
|
||||
|
||||
let buffer: Buffer;
|
||||
try {
|
||||
buffer = fs.readFileSync(resolved);
|
||||
} catch (e) {
|
||||
return { output: `ReadMsg: cannot read file: ${(e as Error).message}`, isError: true };
|
||||
}
|
||||
|
||||
// .msg is an OLE2 / CFBF compound file. Validate the magic header up front:
|
||||
// MsgReader silently returns an empty result for non-CFBF data instead of throwing.
|
||||
const CFBF_MAGIC = Buffer.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]);
|
||||
if (buffer.length < 8 || !buffer.subarray(0, 8).equals(CFBF_MAGIC)) {
|
||||
return {
|
||||
output: `ReadMsg: not a valid Outlook .msg file (bad signature): ${path.basename(resolved)}`,
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
|
||||
let reader: MsgReaderInstance;
|
||||
let fields: FieldsLike;
|
||||
try {
|
||||
// Copy into a standalone ArrayBuffer (MsgReader rejects Node Buffers).
|
||||
const arrayBuffer = new Uint8Array(buffer).buffer;
|
||||
reader = new MsgReader(arrayBuffer);
|
||||
fields = reader.getFileData() as unknown as FieldsLike;
|
||||
} catch (e) {
|
||||
return {
|
||||
output: `ReadMsg: failed to parse .msg (is this a valid Outlook message?): ${(e as Error).message}`,
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
|
||||
if (!isParsedMsgValid(fields)) {
|
||||
return {
|
||||
output: `ReadMsg: not a parseable Outlook message${fields.error ? ` (${fields.error})` : ''}: ${path.basename(resolved)}`,
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
|
||||
const recipients = fields.recipients ?? [];
|
||||
const to = recipients
|
||||
.filter((r) => (r.recipType ?? 'to') === 'to')
|
||||
.map((r) => ({ name: r.name, email: pickEmail(r.smtpAddress, r.email) }));
|
||||
const cc = recipients
|
||||
.filter((r) => r.recipType === 'cc')
|
||||
.map((r) => ({ name: r.name, email: pickEmail(r.smtpAddress, r.email) }));
|
||||
|
||||
const inputDir = path.join(ctx.workspacePath, 'input');
|
||||
const attachments: MsgAttachmentMeta[] = [];
|
||||
const rawAttachments = fields.attachments ?? [];
|
||||
const usedNames = new Set<string>();
|
||||
|
||||
rawAttachments.forEach((att, i) => {
|
||||
const rawName = att.fileName || att.fileNameShort || '';
|
||||
const baseName = sanitizeAttachmentName(rawName, i);
|
||||
|
||||
// Read-only movements (verify etc.) must not mutate the workspace.
|
||||
if (!ctx.editAllowed) {
|
||||
attachments.push({
|
||||
fileName: baseName,
|
||||
contentLength: att.contentLength,
|
||||
skipped: 'read-only phase (attachment not saved)',
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (att.innerMsgContent) {
|
||||
attachments.push({
|
||||
fileName: baseName,
|
||||
contentLength: att.contentLength,
|
||||
skipped: 'embedded message (open separately)',
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Resolve a name that collides with neither an earlier attachment this run
|
||||
// nor a file already present in input/ (user uploads, prior extractions).
|
||||
const name = resolveFreeName(inputDir, baseName, usedNames);
|
||||
usedNames.add(name);
|
||||
|
||||
try {
|
||||
const data = reader.getAttachment(att as never);
|
||||
fs.mkdirSync(inputDir, { recursive: true });
|
||||
const dest = path.join(inputDir, name);
|
||||
fs.writeFileSync(dest, Buffer.from(data.content));
|
||||
attachments.push({
|
||||
fileName: name,
|
||||
contentLength: data.content.length,
|
||||
savedPath: path.join('input', name),
|
||||
});
|
||||
} catch (e) {
|
||||
logger.warn(`[ReadMsg] failed to save attachment ${name}: ${(e as Error).message}`);
|
||||
attachments.push({
|
||||
fileName: name,
|
||||
contentLength: att.contentLength,
|
||||
skipped: `extraction failed: ${(e as Error).message}`,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
const view: MsgView = {
|
||||
subject: fields.subject,
|
||||
from: (() => {
|
||||
const email = pickEmail(fields.senderSmtpAddress, fields.senderEmail);
|
||||
return fields.senderName || email ? { name: fields.senderName, email } : undefined;
|
||||
})(),
|
||||
to,
|
||||
cc,
|
||||
date: fields.messageDeliveryTime || fields.clientSubmitTime,
|
||||
body: selectMsgBody(fields),
|
||||
attachments,
|
||||
};
|
||||
|
||||
const output = assembleMsgOutput(view, getToolOutputBudgetTokens(ctx), path.basename(resolved));
|
||||
logger.info(
|
||||
`[ReadMsg] ${path.basename(resolved)}: attachments=${attachments.length} bodyFormat=${view.body.format}`,
|
||||
);
|
||||
return { output, isError: false };
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import { ToolDef } from '../../llm/openai-compat.js';
|
||||
import type { ToolContext, ToolResult } from './core.js';
|
||||
import { resolveAndGuard, resolveOutputPathWithin, truncateToBudget, getToolOutputBudgetTokens } from './core.js';
|
||||
import { resolveThemePalette, extractSheetStyles } from './excel-styles.js';
|
||||
import { READ_MSG_DEF, executeReadMsg } from './msg.js';
|
||||
import { logger } from '../../logger.js';
|
||||
import { callVisionModel, resolveImagePath } from './image.js';
|
||||
import type {
|
||||
@@ -351,6 +352,7 @@ export const TOOL_DEFS: Record<string, ToolDef> = {
|
||||
ReadDocx: READ_DOCX_DEF,
|
||||
ReadPdf: READ_PDF_DEF,
|
||||
ReadPPTX: READ_PPTX_DEF,
|
||||
ReadMsg: READ_MSG_DEF,
|
||||
SplitExcelSheets: SPLIT_EXCEL_SHEETS_DEF,
|
||||
SplitDocxSections: SPLIT_DOCX_SECTIONS_DEF,
|
||||
PdfToImages: PDF_TO_IMAGES_DEF,
|
||||
@@ -2284,6 +2286,8 @@ export async function executeTool(
|
||||
return executeReadPdf(input, ctx);
|
||||
case 'ReadPPTX':
|
||||
return executeReadPptx(input, ctx);
|
||||
case 'ReadMsg':
|
||||
return executeReadMsg(input, ctx);
|
||||
case 'SplitExcelSheets':
|
||||
return executeSplitExcelSheets(input, ctx);
|
||||
case 'SplitDocxSections':
|
||||
|
||||
@@ -758,3 +758,43 @@ describe('OpenAICompatClient gateway sentinel error events', () => {
|
||||
expect(events.at(-1)?.type).toBe('done');
|
||||
});
|
||||
});
|
||||
|
||||
describe('OpenAICompatClient tool-call flush at stream end', () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
// Some OpenAI-compat backends finish a forced/named tool call with
|
||||
// finish_reason 'stop' (or omit it) instead of 'tool_calls'. The client must
|
||||
// still surface the accumulated tool_use at the done boundary, not drop it.
|
||||
it('emits tool_use when the stream ends on finish_reason "stop" (via [DONE])', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue(makeSseResponse([
|
||||
{ choices: [{ delta: { tool_calls: [{ index: 0, id: 'c1', function: { name: 'do_it', arguments: '{"a":1}' } }] }, finish_reason: null }] },
|
||||
{ choices: [{ delta: {}, finish_reason: 'stop' }] },
|
||||
'[DONE]',
|
||||
])));
|
||||
const events = await collectEvents(new OpenAICompatClient('http://h:1/v1', 'm'), [{ role: 'user', content: 'q' }]);
|
||||
const toolUse = events.find((e) => e.type === 'tool_use');
|
||||
expect(toolUse).toMatchObject({ type: 'tool_use', name: 'do_it', input: { a: 1 } });
|
||||
// done still follows the tool_use
|
||||
expect(events[events.length - 1].type).toBe('done');
|
||||
});
|
||||
|
||||
it('emits tool_use when the stream ends at EOF without [DONE]', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue(makeSseResponse([
|
||||
{ choices: [{ delta: { tool_calls: [{ index: 0, id: 'c2', function: { name: 'fn', arguments: '{}' } }] }, finish_reason: null }] },
|
||||
])));
|
||||
const events = await collectEvents(new OpenAICompatClient('http://h:1/v1', 'm'), [{ role: 'user', content: 'q' }]);
|
||||
expect(events.filter((e) => e.type === 'tool_use')).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('does not double-emit when finish_reason "tool_calls" already flushed', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue(makeSseResponse([
|
||||
{ choices: [{ delta: { tool_calls: [{ index: 0, id: 'c3', function: { name: 'fn', arguments: '{}' } }] }, finish_reason: 'tool_calls' }] },
|
||||
'[DONE]',
|
||||
])));
|
||||
const events = await collectEvents(new OpenAICompatClient('http://h:1/v1', 'm'), [{ role: 'user', content: 'q' }]);
|
||||
expect(events.filter((e) => e.type === 'tool_use')).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
+114
-18
@@ -1,5 +1,6 @@
|
||||
import { getDefaultProviderRetryConfig, type ProviderRetryConfig } from '../config.js';
|
||||
import { logger } from '../logger.js';
|
||||
import { recordLlmUsage } from './usage-recorder.js';
|
||||
import {
|
||||
IMAGE_CONTENT_TOKENS,
|
||||
estimateMessageTokens,
|
||||
@@ -238,6 +239,33 @@ interface ToolCallAccumulator {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Emit accumulated tool calls as `tool_use` events (sorted by index) and
|
||||
* clear the accumulator. Called both on `finish_reason === 'tool_calls'` and
|
||||
* at stream end — some OpenAI-compat backends finish a forced/named tool call
|
||||
* with finish_reason 'stop', so draining at the done boundary keeps the call
|
||||
* from being silently dropped. Returns an empty array when nothing is pending,
|
||||
* so the done-site flush is a no-op for the normal 'tool_calls' path (the map
|
||||
* is already cleared).
|
||||
*/
|
||||
function drainToolCalls(accumulators: Map<number, ToolCallAccumulator>): LLMEvent[] {
|
||||
if (accumulators.size === 0) return [];
|
||||
const events: LLMEvent[] = [];
|
||||
const sortedIndices = Array.from(accumulators.keys()).sort((a, b) => a - b);
|
||||
for (const idx of sortedIndices) {
|
||||
const acc = accumulators.get(idx)!;
|
||||
let input: Record<string, unknown> = {};
|
||||
try {
|
||||
input = JSON.parse(acc.function.arguments) as Record<string, unknown>;
|
||||
} catch {
|
||||
logger.warn(`OpenAICompatClient: failed to parse tool arguments: ${acc.function.arguments}`);
|
||||
}
|
||||
events.push({ type: 'tool_use', id: acc.id, name: acc.function.name, input });
|
||||
}
|
||||
accumulators.clear();
|
||||
return events;
|
||||
}
|
||||
|
||||
export interface OpenAICompatClientOptions {
|
||||
/**
|
||||
* When true, this client treats its endpoint as an LLM gateway / proxy
|
||||
@@ -252,6 +280,26 @@ export interface OpenAICompatClientOptions {
|
||||
proxy?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-call attribution context. Threaded from each call site so the
|
||||
* usage recorder can attribute the completion to a MAESTRO user. Absent
|
||||
* userId falls back to the 'system' sentinel (never NULL).
|
||||
*/
|
||||
export interface LlmCallContext {
|
||||
userId?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-call request-shaping overrides. Used by callers that need to force a
|
||||
* tool (reflection's forced submit_reflection) or pin sampling temperature.
|
||||
* Kept off the hot agent path (which leaves these unset).
|
||||
*/
|
||||
export interface LlmRequestOptions {
|
||||
temperature?: number;
|
||||
/** OpenAI tool_choice (e.g. `{ type: 'function', function: { name } }`). */
|
||||
toolChoice?: unknown;
|
||||
}
|
||||
|
||||
export class OpenAICompatClient {
|
||||
private retryConfig: ProviderRetryConfig;
|
||||
readonly timeoutMs: number;
|
||||
@@ -292,7 +340,43 @@ export class OpenAICompatClient {
|
||||
this.preferredBackendId = backendId;
|
||||
}
|
||||
|
||||
async *chat(messages: Message[], tools?: ToolDef[], externalSignal?: AbortSignal): AsyncGenerator<LLMEvent> {
|
||||
/**
|
||||
* Record one successful completion to the per-user daily usage ledger.
|
||||
* Called from the single done funnel (both the `[DONE]` and EOF exits)
|
||||
* so the two terminal paths can never double-count. `source` is the
|
||||
* client's proxy flag, `model` is the first observed chunk.model (routing
|
||||
* key fallback), `route` is the gateway backendId (proxy) or endpoint host
|
||||
* (direct). Never records on abort / timeout / error (those don't `done`).
|
||||
*/
|
||||
private finalizeDone(
|
||||
usage: { prompt_tokens: number; completion_tokens: number } | undefined,
|
||||
observedModel: string,
|
||||
observedBackendId: string,
|
||||
context?: LlmCallContext,
|
||||
): void {
|
||||
const source: 'gateway' | 'direct' = this.proxy ? 'gateway' : 'direct';
|
||||
const model = observedModel || this.model || 'unknown';
|
||||
let route = 'unknown';
|
||||
if (this.proxy) {
|
||||
route = observedBackendId || 'unknown';
|
||||
} else {
|
||||
try {
|
||||
route = new URL(this.baseUrl).host || 'unknown';
|
||||
} catch {
|
||||
route = 'unknown';
|
||||
}
|
||||
}
|
||||
recordLlmUsage({
|
||||
userId: context?.userId || 'system',
|
||||
source,
|
||||
model,
|
||||
route,
|
||||
tokensIn: usage?.prompt_tokens ?? 0,
|
||||
tokensOut: usage?.completion_tokens ?? 0,
|
||||
});
|
||||
}
|
||||
|
||||
async *chat(messages: Message[], tools?: ToolDef[], externalSignal?: AbortSignal, context?: LlmCallContext, requestOptions?: LlmRequestOptions): AsyncGenerator<LLMEvent> {
|
||||
const controller = new AbortController();
|
||||
// アイドルタイムアウト: チャンク受信のたびにリセットされる
|
||||
let timeoutId = setTimeout(() => controller.abort(), this.timeoutMs);
|
||||
@@ -338,6 +422,12 @@ export class OpenAICompatClient {
|
||||
if (tools && tools.length > 0) {
|
||||
body['tools'] = tools;
|
||||
}
|
||||
if (requestOptions?.temperature != null) {
|
||||
body['temperature'] = requestOptions.temperature;
|
||||
}
|
||||
if (requestOptions?.toolChoice != null) {
|
||||
body['tool_choice'] = requestOptions.toolChoice;
|
||||
}
|
||||
// Block oversized prompts before the HTTP request so callers see a
|
||||
// structured error instead of an opaque HTTP 400. The runtime context
|
||||
// limit is fetched per-model (see fetchOllamaContextLimit) and passed
|
||||
@@ -358,6 +448,10 @@ export class OpenAICompatClient {
|
||||
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
let response: Response | null = null;
|
||||
// Usage attribution captured during this attempt's stream. Reset
|
||||
// per attempt; only the attempt that reaches `done` records.
|
||||
let observedModel = '';
|
||||
let observedBackendId = '';
|
||||
|
||||
try {
|
||||
response = await fetch(`${this.baseUrl}/chat/completions`, {
|
||||
@@ -440,6 +534,7 @@ export class OpenAICompatClient {
|
||||
const rawBackendId = response.headers.get('x-litellm-model-id');
|
||||
const backendId = rawBackendId ? rawBackendId.trim() : '';
|
||||
if (backendId.length > 0) {
|
||||
observedBackendId = backendId;
|
||||
const rawCacheKey = response.headers.get('x-litellm-cache-key');
|
||||
const cacheKey = rawCacheKey ? rawCacheKey.trim() : '';
|
||||
yield { type: 'backend', backendId, cacheKey: cacheKey.length > 0 ? cacheKey : null };
|
||||
@@ -475,7 +570,13 @@ export class OpenAICompatClient {
|
||||
const data = trimmed.slice('data: '.length);
|
||||
|
||||
if (data === '[DONE]') {
|
||||
// Flush any tool calls the backend left un-finished (some
|
||||
// OpenAI-compat servers end a forced/named tool call with
|
||||
// finish_reason 'stop' instead of 'tool_calls'). Without this
|
||||
// the accumulated call would be silently dropped.
|
||||
yield* drainToolCalls(toolCallAccumulators);
|
||||
// usage 付きで done を emit
|
||||
this.finalizeDone(usage, observedModel, observedBackendId, context);
|
||||
yield { type: 'done', usage };
|
||||
return;
|
||||
}
|
||||
@@ -488,6 +589,15 @@ export class OpenAICompatClient {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Real model name for usage attribution. The gateway passes
|
||||
// chunks through byte-for-byte, so chunk.model is the actual
|
||||
// backend model for both direct and gateway paths. First
|
||||
// non-empty value wins.
|
||||
if (!observedModel) {
|
||||
const m = chunk['model'];
|
||||
if (typeof m === 'string' && m.length > 0) observedModel = m;
|
||||
}
|
||||
|
||||
// AAO Gateway / LiteLLM sentinel error event:
|
||||
// data: {"error":{"type":"gateway_shutdown","message":"..."}}
|
||||
// gateway_shutdown / gateway_timeout は他 worker に retry すれば
|
||||
@@ -596,23 +706,7 @@ export class OpenAICompatClient {
|
||||
|
||||
// tool_calls が完了したら emit
|
||||
if (finishReason === 'tool_calls') {
|
||||
const sortedIndices = Array.from(toolCallAccumulators.keys()).sort((a, b) => a - b);
|
||||
for (const idx of sortedIndices) {
|
||||
const acc = toolCallAccumulators.get(idx)!;
|
||||
let input: Record<string, unknown> = {};
|
||||
try {
|
||||
input = JSON.parse(acc.function.arguments) as Record<string, unknown>;
|
||||
} catch {
|
||||
logger.warn(`OpenAICompatClient: failed to parse tool arguments: ${acc.function.arguments}`);
|
||||
}
|
||||
yield {
|
||||
type: 'tool_use',
|
||||
id: acc.id,
|
||||
name: acc.function.name,
|
||||
input,
|
||||
};
|
||||
}
|
||||
toolCallAccumulators.clear();
|
||||
yield* drainToolCalls(toolCallAccumulators);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -644,6 +738,8 @@ export class OpenAICompatClient {
|
||||
}
|
||||
|
||||
// [DONE] なしにストリームが終了した場合
|
||||
yield* drainToolCalls(toolCallAccumulators);
|
||||
this.finalizeDone(usage, observedModel, observedBackendId, context);
|
||||
yield { type: 'done', usage };
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import { logger } from '../logger.js';
|
||||
|
||||
/**
|
||||
* Per-user LLM usage event, emitted by OpenAICompatClient once per
|
||||
* successful chat completion (gateway-routed and direct alike). The
|
||||
* recorder is a process-global sink so every call site — agent-loop,
|
||||
* title generation, piece classification, reflection — is covered
|
||||
* without threading a Repository handle through every client
|
||||
* construction. This is the single chokepoint the design relies on to
|
||||
* avoid the propagation leaks this codebase has repeatedly hit.
|
||||
*
|
||||
* Spec: docs/superpowers/specs/2026-06-11-llm-usage-aggregation-design.md
|
||||
*/
|
||||
export interface LlmUsageEvent {
|
||||
/** Owner id, or 'local' (no-auth) / 'system' (ownerless) sentinel. NOT NULL. */
|
||||
userId: string;
|
||||
source: 'gateway' | 'direct';
|
||||
/** Real model name (chunk.model) with routing-key / 'unknown' fallback. */
|
||||
model: string;
|
||||
/** Backend server name (gateway backendId / direct host) or 'unknown'. */
|
||||
route: string;
|
||||
tokensIn: number;
|
||||
tokensOut: number;
|
||||
}
|
||||
|
||||
export type LlmUsageRecorder = (event: LlmUsageEvent) => void;
|
||||
|
||||
let recorder: LlmUsageRecorder | null = null;
|
||||
|
||||
/**
|
||||
* Install (or clear with null) the process-global usage recorder. Called
|
||||
* once during bootstrap with a thin wrapper over Repository.incrementLlmUsage.
|
||||
*/
|
||||
export function setLlmUsageRecorder(fn: LlmUsageRecorder | null): void {
|
||||
recorder = fn;
|
||||
}
|
||||
|
||||
/**
|
||||
* Record one successful completion. No-op when no recorder is installed
|
||||
* (e.g. unit tests, gateway server process). Writes are best-effort: a DB
|
||||
* hiccup must never kill the agent's stream, so failures are swallowed
|
||||
* with a warn.
|
||||
*/
|
||||
export function recordLlmUsage(event: LlmUsageEvent): void {
|
||||
if (!recorder) return;
|
||||
try {
|
||||
recorder(event);
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
logger.warn(`[usage-recorder] record failed (non-fatal): ${msg}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
/**
|
||||
* Usage recording at the OpenAICompatClient completion boundary.
|
||||
*
|
||||
* Verifies the single finalizeDone funnel:
|
||||
* - a successful stream records exactly one event with model (chunk.model),
|
||||
* tokens from usage, source=direct, route=endpoint host
|
||||
* - a usage-less but successful stream still records (tokens 0)
|
||||
* - an error stream records nothing (abort/timeout/error never `done`)
|
||||
* - proxy mode records source=gateway, route=backendId (x-litellm-model-id)
|
||||
* - no recorder installed = no-op (never throws)
|
||||
*
|
||||
* Spec: docs/superpowers/specs/2026-06-11-llm-usage-aggregation-design.md
|
||||
*/
|
||||
import { describe, it, expect, vi, afterEach } from 'vitest';
|
||||
import { OpenAICompatClient } from './openai-compat.js';
|
||||
import { getDefaultProviderRetryConfig } from '../config.js';
|
||||
import { setLlmUsageRecorder, type LlmUsageEvent } from './usage-recorder.js';
|
||||
|
||||
const NO_RETRY = { ...getDefaultProviderRetryConfig(), maxAttempts: 1 };
|
||||
|
||||
function sseResponse(chunks: unknown[], headers: Record<string, string> = {}): Response {
|
||||
const lines = chunks.map((c) => `data: ${JSON.stringify(c)}\n\n`);
|
||||
lines.push('data: [DONE]\n\n');
|
||||
const encoder = new TextEncoder();
|
||||
let i = 0;
|
||||
return {
|
||||
ok: true,
|
||||
status: 200,
|
||||
headers: { get: (k: string) => headers[k.toLowerCase()] ?? null },
|
||||
body: {
|
||||
getReader: () => ({
|
||||
read: async () =>
|
||||
i < lines.length
|
||||
? { done: false, value: encoder.encode(lines[i++]) }
|
||||
: { done: true, value: undefined },
|
||||
releaseLock: () => {},
|
||||
}),
|
||||
},
|
||||
} as unknown as Response;
|
||||
}
|
||||
|
||||
const textChunk = (model: string) => ({ model, choices: [{ delta: { content: 'hi' }, finish_reason: null }] });
|
||||
const usageChunk = (pin: number, pout: number) => ({ choices: [], usage: { prompt_tokens: pin, completion_tokens: pout } });
|
||||
|
||||
async function drain(client: OpenAICompatClient, ctx?: { userId?: string }): Promise<void> {
|
||||
for await (const _ of client.chat([{ role: 'user', content: 'q' }], undefined, undefined, ctx)) {
|
||||
// discard
|
||||
}
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
setLlmUsageRecorder(null);
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
describe('LLM usage recording', () => {
|
||||
it('records one direct event with model, tokens, and endpoint-host route', async () => {
|
||||
const events: LlmUsageEvent[] = [];
|
||||
setLlmUsageRecorder((e) => events.push(e));
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue(sseResponse([textChunk('llama-3.1-8b'), usageChunk(120, 30)])));
|
||||
|
||||
const client = new OpenAICompatClient('http://gpu-1.local:11434/v1', 'role-auto');
|
||||
await drain(client, { userId: 'u1' });
|
||||
|
||||
expect(events).toHaveLength(1);
|
||||
expect(events[0]).toEqual({
|
||||
userId: 'u1', source: 'direct', model: 'llama-3.1-8b', route: 'gpu-1.local:11434',
|
||||
tokensIn: 120, tokensOut: 30,
|
||||
});
|
||||
});
|
||||
|
||||
it('records a usage-less success with zero tokens', async () => {
|
||||
const events: LlmUsageEvent[] = [];
|
||||
setLlmUsageRecorder((e) => events.push(e));
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue(sseResponse([textChunk('m')])));
|
||||
|
||||
await drain(new OpenAICompatClient('http://h:1/v1', 'm'), { userId: 'u1' });
|
||||
expect(events).toHaveLength(1);
|
||||
expect(events[0]).toMatchObject({ tokensIn: 0, tokensOut: 0 });
|
||||
});
|
||||
|
||||
it('falls back userId to system and model to routing key', async () => {
|
||||
const events: LlmUsageEvent[] = [];
|
||||
setLlmUsageRecorder((e) => events.push(e));
|
||||
// No `model` field in the chunk → fall back to the client's routing key.
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue(sseResponse([{ choices: [{ delta: { content: 'x' }, finish_reason: null }] }, usageChunk(1, 1)])));
|
||||
|
||||
await drain(new OpenAICompatClient('http://h:1/v1', 'routing-key')); // no context
|
||||
expect(events[0]).toMatchObject({ userId: 'system', model: 'routing-key' });
|
||||
});
|
||||
|
||||
it('records nothing on an error stream', async () => {
|
||||
const events: LlmUsageEvent[] = [];
|
||||
setLlmUsageRecorder((e) => events.push(e));
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
|
||||
ok: false, status: 500, headers: { get: () => null }, text: () => Promise.resolve('boom'),
|
||||
} as unknown as Response));
|
||||
|
||||
await drain(new OpenAICompatClient('http://h:1/v1', 'm', undefined, NO_RETRY), { userId: 'u1' });
|
||||
expect(events).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('records source=gateway with backendId route in proxy mode', async () => {
|
||||
const events: LlmUsageEvent[] = [];
|
||||
setLlmUsageRecorder((e) => events.push(e));
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue(
|
||||
sseResponse([textChunk('qwen-72b'), usageChunk(5, 2)], { 'x-litellm-model-id': 'pool-a-node-3' }),
|
||||
));
|
||||
|
||||
const client = new OpenAICompatClient(
|
||||
'http://gateway:4000/v1', 'role-quality', undefined, undefined, undefined, undefined, undefined, undefined,
|
||||
{ proxy: true },
|
||||
);
|
||||
await drain(client, { userId: 'u9' });
|
||||
|
||||
expect(events).toHaveLength(1);
|
||||
expect(events[0]).toMatchObject({ source: 'gateway', model: 'qwen-72b', route: 'pool-a-node-3' });
|
||||
});
|
||||
|
||||
it('is a no-op when no recorder is installed', async () => {
|
||||
setLlmUsageRecorder(null);
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue(sseResponse([textChunk('m'), usageChunk(1, 1)])));
|
||||
await expect(drain(new OpenAICompatClient('http://h:1/v1', 'm'), { userId: 'u1' })).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -7,6 +7,7 @@ import { normalizeToolNameForMetric, BUILTIN_TOOL_NAMES } from './tool-name-allo
|
||||
import { createWorkerMetrics } from './worker-metrics.js';
|
||||
import { TOOL_DEFS as SLIDE_DEFS } from '../engine/tools/slide.js';
|
||||
import { TOOL_DEFS as MSLEARN_DEFS } from '../engine/tools/ms-learn.js';
|
||||
import { TOOL_DEFS as OFFICE_DEFS } from '../engine/tools/office.js';
|
||||
|
||||
describe('normalizeToolNameForMetric', () => {
|
||||
it('passes built-in tool names through verbatim', () => {
|
||||
@@ -69,15 +70,19 @@ describe('normalizeToolNameForMetric', () => {
|
||||
// so every real call collapsed to 'unknown' in metrics. Pin the allowlist to
|
||||
// the actual TOOL_DEFS so it can't drift again.
|
||||
describe('metrics allowlist ↔ real tool definitions (audit regression)', () => {
|
||||
const realNames = [...Object.keys(SLIDE_DEFS), ...Object.keys(MSLEARN_DEFS)];
|
||||
const realNames = [
|
||||
...Object.keys(SLIDE_DEFS),
|
||||
...Object.keys(MSLEARN_DEFS),
|
||||
...Object.keys(OFFICE_DEFS),
|
||||
];
|
||||
|
||||
it('every real slide/ms-learn tool normalizes to itself (not "unknown")', () => {
|
||||
it('every real slide/ms-learn/office tool normalizes to itself (not "unknown")', () => {
|
||||
for (const name of realNames) {
|
||||
expect(normalizeToolNameForMetric(name)).toBe(name);
|
||||
}
|
||||
});
|
||||
|
||||
it('every real slide/ms-learn tool is present in BUILTIN_TOOL_NAMES', () => {
|
||||
it('every real slide/ms-learn/office tool is present in BUILTIN_TOOL_NAMES', () => {
|
||||
for (const name of realNames) {
|
||||
expect(BUILTIN_TOOL_NAMES.has(name)).toBe(true);
|
||||
}
|
||||
|
||||
@@ -43,7 +43,7 @@ const BUILTIN_TOOL_NAMES_LIST: ReadonlyArray<string> = [
|
||||
// image.ts
|
||||
'AnnotateImage', 'ReadImage',
|
||||
// office.ts
|
||||
'PdfToImages', 'ReadDocx', 'ReadExcel', 'ReadPdf', 'ReadPPTX',
|
||||
'PdfToImages', 'ReadDocx', 'ReadExcel', 'ReadMsg', 'ReadPdf', 'ReadPPTX',
|
||||
'SplitDocxSections', 'SplitExcelSheets',
|
||||
// data.ts
|
||||
'SQLite',
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
import { describe, it, expect, afterEach } from 'vitest';
|
||||
import type { Server } from 'http';
|
||||
import * as http from 'http';
|
||||
import { buildRedirectLocation, createHttpRedirectServer } from './http-redirect.js';
|
||||
|
||||
describe('buildRedirectLocation', () => {
|
||||
it('pins the host and preserves path + query', () => {
|
||||
expect(buildRedirectLocation('app.lan', 9876, '/tasks?id=5')).toBe('https://app.lan:9876/tasks?id=5');
|
||||
});
|
||||
|
||||
it('buildRedirectLocation never echoes a caller-supplied host string', () => {
|
||||
// The function takes pinnedHost as a parameter — it cannot echo an attacker host
|
||||
// because the host is always taken from the first argument, not from reqUrl.
|
||||
// Pass a reqUrl that looks like it contains a host to confirm it is ignored.
|
||||
const loc = buildRedirectLocation('app.lan', 9876, '/');
|
||||
expect(loc).toBe('https://app.lan:9876/');
|
||||
expect(loc).not.toContain('evil.attacker.com');
|
||||
});
|
||||
|
||||
it('strips CR/LF/space from the path to prevent header injection', () => {
|
||||
const loc = buildRedirectLocation('app.lan', 9876, '/a\r\nSet-Cookie: x=1 b');
|
||||
expect(loc).not.toMatch(/[\r\n ]/);
|
||||
});
|
||||
|
||||
it('omits the port for 443', () => {
|
||||
expect(buildRedirectLocation('app.lan', 443, '/')).toBe('https://app.lan/');
|
||||
});
|
||||
|
||||
it('prefixes a leading slash when the url lacks one', () => {
|
||||
expect(buildRedirectLocation('app.lan', 9876, 'x')).toBe('https://app.lan:9876/x');
|
||||
});
|
||||
});
|
||||
|
||||
describe('createHttpRedirectServer', () => {
|
||||
let server: Server;
|
||||
afterEach(() => new Promise<void>((r) => (server ? server.close(() => r()) : r())));
|
||||
|
||||
it('responds 301 with the pinned https Location', async () => {
|
||||
server = createHttpRedirectServer({ httpsPort: 9876, pinnedHost: 'app.lan' });
|
||||
await new Promise<void>((r) => server.listen(0, '127.0.0.1', r));
|
||||
const addr = server.address();
|
||||
const port = typeof addr === 'object' && addr ? addr.port : 0;
|
||||
const res = await fetch(`http://127.0.0.1:${port}/x?y=1`, { redirect: 'manual' });
|
||||
expect(res.status).toBe(301);
|
||||
expect(res.headers.get('location')).toBe('https://app.lan:9876/x?y=1');
|
||||
});
|
||||
|
||||
it('ignores an attacker-controlled Host header (open-redirect guard)', async () => {
|
||||
const srv = createHttpRedirectServer({ httpsPort: 9876, pinnedHost: 'app.lan' });
|
||||
await new Promise<void>((r) => srv.listen(0, '127.0.0.1', r));
|
||||
try {
|
||||
const addr = srv.address();
|
||||
const port = typeof addr === 'object' && addr ? addr.port : 0;
|
||||
// Use node's http.request so we can set the Host header explicitly.
|
||||
// (Some fetch implementations silently drop Host overrides.)
|
||||
const loc = await new Promise<string>((resolve, reject) => {
|
||||
const req = http.request(
|
||||
{ hostname: '127.0.0.1', port, path: '/x', method: 'GET', headers: { Host: 'evil.attacker.com' } },
|
||||
(res) => resolve(res.headers['location'] ?? ''),
|
||||
);
|
||||
req.on('error', reject);
|
||||
req.end();
|
||||
});
|
||||
expect(loc).toBe('https://app.lan:9876/x');
|
||||
expect(loc).not.toContain('evil.attacker.com');
|
||||
} finally {
|
||||
await new Promise<void>((r) => srv.close(() => r()));
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,21 @@
|
||||
import { createServer, type Server } from 'http';
|
||||
|
||||
/** Build the redirect target from a PINNED host — never the request Host header. */
|
||||
export function buildRedirectLocation(pinnedHost: string, httpsPort: number, reqUrl: string): string {
|
||||
const safePath = reqUrl.replace(/[\r\n ]/g, '');
|
||||
const hostPort = httpsPort === 443 ? pinnedHost : `${pinnedHost}:${httpsPort}`;
|
||||
return `https://${hostPort}${safePath.startsWith('/') ? safePath : `/${safePath}`}`;
|
||||
}
|
||||
|
||||
export interface RedirectServerOpts {
|
||||
httpsPort: number;
|
||||
pinnedHost: string;
|
||||
}
|
||||
|
||||
export function createHttpRedirectServer(opts: RedirectServerOpts): Server {
|
||||
return createServer((req, res) => {
|
||||
const location = buildRedirectLocation(opts.pinnedHost, opts.httpsPort, req.url ?? '/');
|
||||
res.writeHead(301, { Location: location });
|
||||
res.end();
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import { mkdtempSync, rmSync, existsSync, writeFileSync, statSync, readFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
import { X509Certificate } from 'crypto';
|
||||
import { ensureSelfSignedCert } from './self-signed.js';
|
||||
|
||||
let dir: string;
|
||||
beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'selfsigned-')); });
|
||||
afterEach(() => { rmSync(dir, { recursive: true, force: true }); });
|
||||
|
||||
describe('ensureSelfSignedCert', () => {
|
||||
it('generates a cert+key pair with localhost/127.0.0.1 in the SAN', () => {
|
||||
const { cert, key } = ensureSelfSignedCert(dir, []);
|
||||
expect(cert).toContain('BEGIN CERTIFICATE');
|
||||
expect(key).toContain('PRIVATE KEY');
|
||||
const san = new X509Certificate(cert).subjectAltName ?? '';
|
||||
expect(san).toContain('localhost');
|
||||
expect(san).toContain('127.0.0.1');
|
||||
});
|
||||
|
||||
it('includes extra SAN hosts', () => {
|
||||
const { cert } = ensureSelfSignedCert(dir, ['maestro.lan']);
|
||||
expect(new X509Certificate(cert).subjectAltName ?? '').toContain('maestro.lan');
|
||||
});
|
||||
|
||||
it('includes a redirect-style hostname in the SAN (guards server.ts augmentation)', () => {
|
||||
// When an operator sets redirect_host: app.lan the server folds that name
|
||||
// into selfSignedHosts before calling resolveTlsOptions — this test guards
|
||||
// that the augmented host is actually present in the emitted certificate.
|
||||
const { cert } = ensureSelfSignedCert(dir, ['redirect.example']);
|
||||
expect(new X509Certificate(cert).subjectAltName ?? '').toContain('redirect.example');
|
||||
});
|
||||
|
||||
it('reuses an existing valid pair (stable across calls)', () => {
|
||||
const a = ensureSelfSignedCert(dir, []);
|
||||
const b = ensureSelfSignedCert(dir, []);
|
||||
expect(b.cert).toBe(a.cert);
|
||||
expect(b.key).toBe(a.key);
|
||||
});
|
||||
|
||||
it('regenerates when the existing pair is mismatched/corrupt', () => {
|
||||
const a = ensureSelfSignedCert(dir, []);
|
||||
writeFileSync(join(dir, 'cert.pem'), 'not a cert');
|
||||
const b = ensureSelfSignedCert(dir, []);
|
||||
expect(b.cert).toContain('BEGIN CERTIFICATE');
|
||||
expect(b.cert).not.toBe(a.cert);
|
||||
});
|
||||
|
||||
it('writes the key file with 0600 permissions', () => {
|
||||
ensureSelfSignedCert(dir, []);
|
||||
expect(statSync(join(dir, 'key.pem')).mode & 0o777).toBe(0o600);
|
||||
});
|
||||
|
||||
it('writes the cert file with 0600 permissions', () => {
|
||||
ensureSelfSignedCert(dir, []);
|
||||
expect(statSync(join(dir, 'cert.pem')).mode & 0o777).toBe(0o600);
|
||||
});
|
||||
|
||||
it('treats host:port as a DNS name, not an IP address (regression for isIP fix)', () => {
|
||||
const { cert } = ensureSelfSignedCert(dir, ['myhost:8443']);
|
||||
const san = new X509Certificate(cert).subjectAltName ?? '';
|
||||
// Must appear as a DNS-typed SAN entry, not an IP Address entry
|
||||
expect(san).toContain('DNS:myhost:8443');
|
||||
expect(san).not.toContain('IP Address:myhost:8443');
|
||||
});
|
||||
|
||||
it('always includes localhost even when hostname detection is junk', () => {
|
||||
const { cert } = ensureSelfSignedCert(dir, [], { hostname: '' });
|
||||
expect(new X509Certificate(cert).subjectAltName ?? '').toContain('localhost');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
import { readFileSync, writeFileSync, existsSync, renameSync, mkdirSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { hostname as osHostname } from 'os';
|
||||
import { isIP } from 'net';
|
||||
import { X509Certificate, createPrivateKey } from 'crypto';
|
||||
import { logger } from '../logger.js';
|
||||
import selfsigned from 'selfsigned';
|
||||
|
||||
export interface EnsureCertOpts {
|
||||
/** Override host detection (tests). */
|
||||
hostname?: string;
|
||||
}
|
||||
|
||||
const CERT_FILE = 'cert.pem';
|
||||
const KEY_FILE = 'key.pem';
|
||||
|
||||
function sanHosts(extra: string[], hostname: string): string[] {
|
||||
const base = ['localhost', '127.0.0.1', '::1'];
|
||||
const host = hostname.trim();
|
||||
if (host && host !== 'localhost') base.push(host);
|
||||
return [...new Set([...base, ...extra.filter((h) => h.trim().length > 0)])];
|
||||
}
|
||||
|
||||
function isIp(value: string): boolean {
|
||||
return isIP(value) !== 0;
|
||||
}
|
||||
|
||||
// Validates that the cert and key match each other; does NOT check expiry by design
|
||||
// (the caller regenerates on restart; the Settings UI surfaces expiry).
|
||||
function readValidPair(dir: string): { cert: string; key: string } | null {
|
||||
const certPath = join(dir, CERT_FILE);
|
||||
const keyPath = join(dir, KEY_FILE);
|
||||
if (!existsSync(certPath) || !existsSync(keyPath)) return null;
|
||||
try {
|
||||
const cert = readFileSync(certPath, 'utf-8');
|
||||
const key = readFileSync(keyPath, 'utf-8');
|
||||
const x509 = new X509Certificate(cert);
|
||||
const priv = createPrivateKey(key);
|
||||
if (!x509.checkPrivateKey(priv)) return null;
|
||||
return { cert, key };
|
||||
} catch (err) {
|
||||
logger.warn(`[self-signed] existing cert/key unreadable, will regenerate: ${(err as Error).message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function ensureSelfSignedCert(
|
||||
dir: string,
|
||||
extraHosts: string[],
|
||||
opts: EnsureCertOpts = {},
|
||||
): { cert: string; key: string } {
|
||||
const existing = readValidPair(dir);
|
||||
if (existing) return existing;
|
||||
|
||||
mkdirSync(dir, { recursive: true });
|
||||
const hosts = sanHosts(extraHosts, opts.hostname ?? osHostname());
|
||||
const altNames = hosts.map((h) => (isIp(h) ? { type: 7, ip: h } : { type: 2, value: h }));
|
||||
const pems = selfsigned.generate([{ name: 'commonName', value: hosts[0] }], {
|
||||
keySize: 2048,
|
||||
days: 825,
|
||||
algorithm: 'sha256',
|
||||
extensions: [{ name: 'subjectAltName', altNames }],
|
||||
});
|
||||
|
||||
const certPath = join(dir, CERT_FILE);
|
||||
const keyPath = join(dir, KEY_FILE);
|
||||
writeFileSync(`${certPath}.tmp`, pems.cert, { mode: 0o600 });
|
||||
writeFileSync(`${keyPath}.tmp`, pems.private, { mode: 0o600 });
|
||||
renameSync(`${keyPath}.tmp`, keyPath);
|
||||
renameSync(`${certPath}.tmp`, certPath);
|
||||
return { cert: pems.cert, key: pems.private };
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
import { resolveTlsOptions } from './tls-options.js';
|
||||
import { SERVER_TLS_DEFAULTS } from '../server/config.js';
|
||||
import { ensureSelfSignedCert } from './self-signed.js';
|
||||
|
||||
let dir: string;
|
||||
beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'tlsopt-')); });
|
||||
afterEach(() => { rmSync(dir, { recursive: true, force: true }); });
|
||||
|
||||
describe('resolveTlsOptions', () => {
|
||||
it('returns a self-signed pair when no cert_file is configured', () => {
|
||||
const opts = resolveTlsOptions({ ...SERVER_TLS_DEFAULTS, enabled: true, selfSignedDir: dir });
|
||||
expect(opts.cert).toContain('BEGIN CERTIFICATE');
|
||||
expect(opts.key).toContain('PRIVATE KEY');
|
||||
expect(opts.minVersion).toBe('TLSv1.2');
|
||||
});
|
||||
|
||||
it('uses provided PEM files when cert_file and key_file are set', () => {
|
||||
const { cert, key } = ensureSelfSignedCert(dir, []);
|
||||
writeFileSync(join(dir, 'my.crt'), cert);
|
||||
writeFileSync(join(dir, 'my.key'), key);
|
||||
const opts = resolveTlsOptions({
|
||||
...SERVER_TLS_DEFAULTS, enabled: true,
|
||||
certFile: join(dir, 'my.crt'), keyFile: join(dir, 'my.key'),
|
||||
});
|
||||
expect(opts.cert).toBe(cert);
|
||||
expect(opts.key).toBe(key);
|
||||
});
|
||||
|
||||
it('fails closed (throws) when cert_file is unreadable — never self-signs', () => {
|
||||
expect(() =>
|
||||
resolveTlsOptions({
|
||||
...SERVER_TLS_DEFAULTS, enabled: true,
|
||||
certFile: join(dir, 'missing.crt'), keyFile: join(dir, 'missing.key'),
|
||||
}),
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
it('fails closed when the provided cert is malformed PEM', () => {
|
||||
writeFileSync(join(dir, 'bad.crt'), 'garbage');
|
||||
writeFileSync(join(dir, 'bad.key'), 'garbage');
|
||||
expect(() =>
|
||||
resolveTlsOptions({
|
||||
...SERVER_TLS_DEFAULTS, enabled: true,
|
||||
certFile: join(dir, 'bad.crt'), keyFile: join(dir, 'bad.key'),
|
||||
}),
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
it('fails closed when provided cert and key do not match', () => {
|
||||
const a = ensureSelfSignedCert(dir, []);
|
||||
const dir2 = mkdtempSync(join(tmpdir(), 'tlsopt2-'));
|
||||
const b = ensureSelfSignedCert(dir2, ['other']);
|
||||
writeFileSync(join(dir, 'a.crt'), a.cert);
|
||||
writeFileSync(join(dir, 'b.key'), b.key);
|
||||
expect(() =>
|
||||
resolveTlsOptions({
|
||||
...SERVER_TLS_DEFAULTS, enabled: true,
|
||||
certFile: join(dir, 'a.crt'), keyFile: join(dir, 'b.key'),
|
||||
}),
|
||||
).toThrow();
|
||||
rmSync(dir2, { recursive: true, force: true });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,30 @@
|
||||
import { readFileSync } from 'fs';
|
||||
import { X509Certificate, createPrivateKey } from 'crypto';
|
||||
import type { ServerTlsConfig } from '../server/config.js';
|
||||
import { ensureSelfSignedCert } from './self-signed.js';
|
||||
|
||||
export interface ResolvedTlsOptions {
|
||||
cert: string;
|
||||
key: string;
|
||||
minVersion: 'TLSv1.2' | 'TLSv1.3';
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve TLS material. Any failure on an operator-provided cert is FATAL
|
||||
* (throws) — never silently fall back to self-signed or HTTP, which would
|
||||
* downgrade an operator who deliberately configured a real certificate.
|
||||
*/
|
||||
export function resolveTlsOptions(tls: ServerTlsConfig): ResolvedTlsOptions {
|
||||
if (tls.certFile && tls.keyFile) {
|
||||
const cert = readFileSync(tls.certFile, 'utf-8');
|
||||
const key = readFileSync(tls.keyFile, 'utf-8');
|
||||
const x509 = new X509Certificate(cert);
|
||||
const priv = createPrivateKey(key);
|
||||
if (!x509.checkPrivateKey(priv)) {
|
||||
throw new Error('server.tls: cert_file and key_file do not match');
|
||||
}
|
||||
return { cert, key, minVersion: tls.minVersion };
|
||||
}
|
||||
const { cert, key } = ensureSelfSignedCert(tls.selfSignedDir, tls.selfSignedHosts);
|
||||
return { cert, key, minVersion: tls.minVersion };
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { mergeServerConfig, SERVER_TLS_DEFAULTS } from './config.js';
|
||||
|
||||
describe('mergeServerConfig', () => {
|
||||
it('upgrade-safe: an absent server block disables TLS', () => {
|
||||
const cfg = mergeServerConfig(undefined, { freshInstall: false });
|
||||
expect(cfg.tls.enabled).toBe(false);
|
||||
});
|
||||
|
||||
it('fresh install with absent block enables TLS', () => {
|
||||
const cfg = mergeServerConfig(undefined, { freshInstall: true });
|
||||
expect(cfg.tls.enabled).toBe(true);
|
||||
});
|
||||
|
||||
it('an explicit enabled value always wins over the freshInstall default', () => {
|
||||
expect(mergeServerConfig({ tls: { enabled: false } }, { freshInstall: true }).tls.enabled).toBe(false);
|
||||
expect(mergeServerConfig({ tls: { enabled: true } }, { freshInstall: false }).tls.enabled).toBe(true);
|
||||
});
|
||||
|
||||
it('fills defaults for unspecified tls fields', () => {
|
||||
const cfg = mergeServerConfig({ tls: { enabled: true } }, { freshInstall: false });
|
||||
expect(cfg.tls.minVersion).toBe(SERVER_TLS_DEFAULTS.minVersion);
|
||||
expect(cfg.tls.selfSignedDir).toBe(SERVER_TLS_DEFAULTS.selfSignedDir);
|
||||
expect(cfg.tls.httpRedirect).toBe(true);
|
||||
expect(cfg.tls.httpRedirectPort).toBe(9080);
|
||||
expect(cfg.tls.selfSignedHosts).toEqual([]);
|
||||
});
|
||||
|
||||
it('throws when http_redirect_port equals the https port', () => {
|
||||
expect(() =>
|
||||
mergeServerConfig({ tls: { enabled: true, httpRedirectPort: 9876 } }, { freshInstall: false, httpsPort: 9876 }),
|
||||
).toThrow(/redirect.*port/i);
|
||||
});
|
||||
|
||||
it('throws when only one of cert_file/key_file is set', () => {
|
||||
expect(() =>
|
||||
mergeServerConfig({ tls: { enabled: true, certFile: '/x/cert.pem' } }, { freshInstall: false }),
|
||||
).toThrow(/cert_file.*key_file|both/i);
|
||||
});
|
||||
|
||||
it('throws when key_file is set without cert_file', () => {
|
||||
expect(() =>
|
||||
mergeServerConfig({ tls: { enabled: true, keyFile: '/x/key.pem' } }, { freshInstall: false }),
|
||||
).toThrow(/cert_file.*key_file|both/i);
|
||||
});
|
||||
|
||||
it('does not throw on port collision when httpRedirect is false', () => {
|
||||
expect(() =>
|
||||
mergeServerConfig(
|
||||
{ tls: { enabled: true, httpRedirect: false, httpRedirectPort: 9876 } },
|
||||
{ freshInstall: false, httpsPort: 9876 },
|
||||
),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it('disabled TLS with a lone certFile does not throw', () => {
|
||||
expect(() =>
|
||||
mergeServerConfig({ tls: { enabled: false, certFile: '/x/c.pem' } }, { freshInstall: false }),
|
||||
).not.toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,63 @@
|
||||
export interface ServerTlsConfig {
|
||||
enabled: boolean;
|
||||
certFile: string | null;
|
||||
keyFile: string | null;
|
||||
minVersion: 'TLSv1.2' | 'TLSv1.3';
|
||||
selfSignedDir: string;
|
||||
selfSignedHosts: string[];
|
||||
httpRedirect: boolean;
|
||||
httpRedirectPort: number;
|
||||
/**
|
||||
* Host used to build the HTTP→HTTPS redirect `Location` header.
|
||||
* When null the bind host is used instead.
|
||||
* Intentionally NOT taken from the request Host header to prevent open-redirect attacks.
|
||||
*/
|
||||
redirectHost: string | null;
|
||||
}
|
||||
|
||||
export interface ServerConfig {
|
||||
tls: ServerTlsConfig;
|
||||
}
|
||||
|
||||
export const SERVER_TLS_DEFAULTS: ServerTlsConfig = {
|
||||
enabled: false,
|
||||
certFile: null,
|
||||
keyFile: null,
|
||||
minVersion: 'TLSv1.2',
|
||||
selfSignedDir: './data/tls',
|
||||
selfSignedHosts: [],
|
||||
httpRedirect: true,
|
||||
httpRedirectPort: 9080,
|
||||
redirectHost: null,
|
||||
};
|
||||
|
||||
export interface MergeServerOpts {
|
||||
freshInstall: boolean;
|
||||
httpsPort?: number;
|
||||
}
|
||||
|
||||
export function mergeServerConfig(
|
||||
partial: Partial<ServerConfig> | undefined,
|
||||
opts: MergeServerOpts,
|
||||
): ServerConfig {
|
||||
const tlsPartial = (partial?.tls ?? {}) as Partial<ServerTlsConfig>;
|
||||
const enabledDefault = opts.freshInstall;
|
||||
const tls: ServerTlsConfig = {
|
||||
...SERVER_TLS_DEFAULTS,
|
||||
...tlsPartial,
|
||||
enabled: tlsPartial.enabled ?? enabledDefault,
|
||||
};
|
||||
tls.selfSignedHosts = [...(tlsPartial.selfSignedHosts ?? SERVER_TLS_DEFAULTS.selfSignedHosts)];
|
||||
|
||||
if (tls.enabled) {
|
||||
const hasCert = !!tls.certFile;
|
||||
const hasKey = !!tls.keyFile;
|
||||
if (hasCert !== hasKey) {
|
||||
throw new Error('server.tls: set both cert_file and key_file, or neither');
|
||||
}
|
||||
if (opts.httpsPort != null && tls.httpRedirect && tls.httpRedirectPort === opts.httpsPort) {
|
||||
throw new Error(`server.tls: http_redirect_port (${tls.httpRedirectPort}) must differ from the HTTPS port`);
|
||||
}
|
||||
}
|
||||
return { tls };
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
buildTitleFallback,
|
||||
buildTitleFromGoal,
|
||||
buildTitlePrompt,
|
||||
isUrlOnlyTitleInput,
|
||||
stripUrlsForTitle,
|
||||
@@ -27,3 +28,29 @@ describe('title generation helpers', () => {
|
||||
expect(buildTitlePrompt(input)).not.toContain('example.com');
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildTitleFromGoal', () => {
|
||||
it('uses the first non-empty line of the goal', () => {
|
||||
expect(buildTitleFromGoal('議事録を作成する\n\n詳細な背景説明...')).toBe('議事録を作成する');
|
||||
});
|
||||
|
||||
it('strips markdown heading and list markers', () => {
|
||||
expect(buildTitleFromGoal('## 売上レポートをまとめる')).toBe('売上レポートをまとめる');
|
||||
expect(buildTitleFromGoal('- 顧客リストを整理する')).toBe('顧客リストを整理する');
|
||||
});
|
||||
|
||||
it('caps length at 40 chars', () => {
|
||||
const long = 'あ'.repeat(80);
|
||||
expect(buildTitleFromGoal(long).length).toBe(40);
|
||||
});
|
||||
|
||||
it('masks URLs and returns empty for URL-only goals', () => {
|
||||
expect(buildTitleFromGoal('https://example.com/x')).toBe('');
|
||||
expect(buildTitleFromGoal('要約して https://example.com/x')).toBe('要約して [URL]');
|
||||
});
|
||||
|
||||
it('returns empty string for blank goal so callers skip the update', () => {
|
||||
expect(buildTitleFromGoal('')).toBe('');
|
||||
expect(buildTitleFromGoal(' \n ')).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -15,6 +15,23 @@ export function buildTitleFallback(input: string): string {
|
||||
return stripUrlsForTitle(input).slice(0, 40).trim() || '新しい依頼';
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive a concise task title from the Mission Brief `goal` set by the agent.
|
||||
* Used at runtime (no LLM call): the agent already restates the user's
|
||||
* requirement as `goal` via MissionUpdate, so we take its first non-empty
|
||||
* line, strip markdown list/heading markers and URLs, and cap the length.
|
||||
* Returns '' when nothing usable remains so callers can skip the update.
|
||||
*/
|
||||
export function buildTitleFromGoal(goal: string): string {
|
||||
const firstLine = goal
|
||||
.split('\n')
|
||||
.map(l => l.trim())
|
||||
.find(l => l.length > 0) ?? '';
|
||||
const cleaned = firstLine.replace(/^#{1,6}\s*/, '').replace(/^[-*+]\s+/, '');
|
||||
if (isUrlOnlyTitleInput(cleaned)) return '';
|
||||
return stripUrlsForTitle(cleaned).slice(0, 40).trim();
|
||||
}
|
||||
|
||||
export function buildTitlePrompt(input: string): string | null {
|
||||
if (isUrlOnlyTitleInput(input)) return null;
|
||||
const sanitized = stripUrlsForTitle(input).slice(0, 500);
|
||||
|
||||
+29
-5
@@ -19,6 +19,7 @@ import { logger } from './logger.js';
|
||||
import { accessSync, existsSync, mkdirSync, constants } from 'fs';
|
||||
import { dirname, resolve, join } from 'path';
|
||||
import { OpenAICompatClient } from './llm/openai-compat.js';
|
||||
import { setLlmUsageRecorder } from './llm/usage-recorder.js';
|
||||
import { llmRoutingKey } from './llm/routing-key.js';
|
||||
import { ConfigManager } from './config-manager.js';
|
||||
import { WorkerManager } from './worker-manager.js';
|
||||
@@ -121,6 +122,23 @@ export async function start(opts: StartWorkerOptions = {}): Promise<void> {
|
||||
const repo = new Repository(dbPath);
|
||||
runMigrations(repo.getDb());
|
||||
|
||||
// Install the process-global LLM usage recorder so every OpenAICompatClient
|
||||
// completion (agent loop, title, classify, reflection — gateway + direct)
|
||||
// lands in the per-user daily ledger. Best-effort: the recorder helper
|
||||
// swallows write errors so a DB hiccup never kills an agent stream.
|
||||
// Spec: docs/superpowers/specs/2026-06-11-llm-usage-aggregation-design.md
|
||||
setLlmUsageRecorder((event) => {
|
||||
repo.incrementLlmUsage({
|
||||
userId: event.userId,
|
||||
source: event.source,
|
||||
model: event.model,
|
||||
route: event.route,
|
||||
tokensIn: event.tokensIn,
|
||||
tokensOut: event.tokensOut,
|
||||
requests: 1,
|
||||
});
|
||||
});
|
||||
|
||||
// 起動時に孤立ジョブを回復
|
||||
await repo.recoverOrphanedJobs();
|
||||
|
||||
@@ -135,7 +153,7 @@ export async function start(opts: StartWorkerOptions = {}): Promise<void> {
|
||||
config.provider.workers.find(w => w.enabled !== false && w.roles?.includes('title')) ??
|
||||
config.provider.workers[0];
|
||||
let titleClient: OpenAICompatClient | null = null;
|
||||
let generateTitle: ((body: string) => Promise<string>) | undefined;
|
||||
let generateTitle: ((body: string, ownerId?: string) => Promise<string>) | undefined;
|
||||
|
||||
if (titleWorker) {
|
||||
const titleModel = titleWorker.model ?? config.provider.model;
|
||||
@@ -156,18 +174,24 @@ export async function start(opts: StartWorkerOptions = {}): Promise<void> {
|
||||
titleClient = new OpenAICompatClient(
|
||||
titleWorker.endpoint,
|
||||
titleRoutingKey,
|
||||
undefined,
|
||||
titleWorker.apiKey,
|
||||
config.provider.retry,
|
||||
(config.provider.timeoutMinutes ?? 10) * 60 * 1000,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
// proxy mode so title/classification usage is recorded as
|
||||
// source='gateway' with the backendId route (not mislabeled 'direct').
|
||||
{ proxy: titleWorker.proxy === true },
|
||||
);
|
||||
|
||||
generateTitle = async (body: string): Promise<string> => {
|
||||
generateTitle = async (body: string, ownerId?: string): Promise<string> => {
|
||||
const fallback = buildTitleFallback(body);
|
||||
const prompt = buildTitlePrompt(body);
|
||||
if (!prompt) return fallback;
|
||||
let title = '';
|
||||
try {
|
||||
for await (const event of titleClient!.chat([{ role: 'user', content: prompt }])) {
|
||||
for await (const event of titleClient!.chat([{ role: 'user', content: prompt }], undefined, undefined, { userId: ownerId })) {
|
||||
if (event.type === 'text') title += event.text;
|
||||
if (event.type === 'error') return fallback;
|
||||
if (event.type === 'done') break;
|
||||
@@ -232,7 +256,7 @@ export async function start(opts: StartWorkerOptions = {}): Promise<void> {
|
||||
const selectPiece = titleClient
|
||||
? async (body: string, fileNames: string[], userId?: string): Promise<string> => {
|
||||
const pieces = pieceCatalog.getForUser(userId ?? 'local');
|
||||
const result = await classifyPiece(titleClient!, body, pieces, fileNames);
|
||||
const result = await classifyPiece(titleClient!, body, pieces, fileNames, undefined, userId ?? 'local');
|
||||
return result ?? 'chat';
|
||||
}
|
||||
: undefined;
|
||||
|
||||
+3
-1
@@ -818,7 +818,7 @@ export class Worker {
|
||||
];
|
||||
|
||||
let answer = '';
|
||||
for await (const event of llmClient.chat(messages)) {
|
||||
for await (const event of llmClient.chat(messages, undefined, undefined, { userId: parentJob?.ownerId ?? 'local' })) {
|
||||
if (event.type === 'text') {
|
||||
answer += event.text;
|
||||
} else if (event.type === 'error') {
|
||||
@@ -1779,6 +1779,8 @@ export class Worker {
|
||||
// Same credential as normal task LLM calls — a key-enforcing
|
||||
// gateway 401s reflection without it.
|
||||
llmApiKey: this.getWorkerDef().apiKey,
|
||||
llmProxy: this.getWorkerDef().proxy === true,
|
||||
llmContextLimitTokens: this.contextLimitTokens,
|
||||
},
|
||||
job
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user