feat: initial public release (MAESTRO)
This commit is contained in:
oss-sync
@@ -0,0 +1,83 @@
|
||||
import * as http from 'http';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { logger } from '../logger.js';
|
||||
|
||||
const MIME_TYPES: Record<string, string> = {
|
||||
'.html': 'text/html; charset=utf-8',
|
||||
'.htm': 'text/html; charset=utf-8',
|
||||
'.txt': 'text/plain; charset=utf-8',
|
||||
'.md': 'text/plain; charset=utf-8',
|
||||
'.json': 'application/json; charset=utf-8',
|
||||
'.css': 'text/css; charset=utf-8',
|
||||
'.js': 'application/javascript; charset=utf-8',
|
||||
'.png': 'image/png',
|
||||
'.jpg': 'image/jpeg',
|
||||
};
|
||||
|
||||
export interface FixtureServer {
|
||||
port: number;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Serve files under `rootDir` over HTTP on a random localhost port.
|
||||
* Path traversal is rejected. Used for benchmark fixtures so tasks
|
||||
* that need WebFetch/BrowseWeb stay reproducible without external network.
|
||||
*/
|
||||
export async function startFixtureServer(rootDir: string): Promise<FixtureServer> {
|
||||
const root = path.resolve(rootDir);
|
||||
if (!fs.existsSync(root) || !fs.statSync(root).isDirectory()) {
|
||||
throw new Error(`Fixture server root does not exist or is not a directory: ${root}`);
|
||||
}
|
||||
|
||||
const server = http.createServer((req, res) => {
|
||||
try {
|
||||
const url = new URL(req.url ?? '/', 'http://localhost');
|
||||
const requested = decodeURIComponent(url.pathname);
|
||||
// Resolve and ensure the result still lives under root.
|
||||
const resolved = path.resolve(root, '.' + (requested === '/' ? '/index.html' : requested));
|
||||
if (!resolved.startsWith(root + path.sep) && resolved !== root) {
|
||||
res.statusCode = 403;
|
||||
res.end('forbidden');
|
||||
return;
|
||||
}
|
||||
if (!fs.existsSync(resolved) || !fs.statSync(resolved).isFile()) {
|
||||
res.statusCode = 404;
|
||||
res.end('not found');
|
||||
return;
|
||||
}
|
||||
const ext = path.extname(resolved).toLowerCase();
|
||||
res.statusCode = 200;
|
||||
res.setHeader('Content-Type', MIME_TYPES[ext] ?? 'application/octet-stream');
|
||||
const stream = fs.createReadStream(resolved);
|
||||
stream.pipe(res);
|
||||
} catch (err) {
|
||||
res.statusCode = 500;
|
||||
res.end(`error: ${(err as Error).message}`);
|
||||
}
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.once('error', reject);
|
||||
server.listen(0, '127.0.0.1', () => {
|
||||
server.removeListener('error', reject);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
|
||||
const address = server.address();
|
||||
if (typeof address !== 'object' || address === null) {
|
||||
throw new Error('Fixture server did not bind to an address');
|
||||
}
|
||||
const port = address.port;
|
||||
logger.info(`[bench/fixture-server] listening on http://127.0.0.1:${port} root=${root}`);
|
||||
|
||||
return {
|
||||
port,
|
||||
async close(): Promise<void> {
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
logger.info(`[bench/fixture-server] closed port=${port}`);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
computeTotal,
|
||||
gradeChecklist,
|
||||
gradeInstructionsProgrammatic,
|
||||
gradeTools,
|
||||
} from './grader.js';
|
||||
import type { BenchTask, RawJobResult, ToolCallObservation } from './types.js';
|
||||
|
||||
function tc(name: string, summary: string): ToolCallObservation {
|
||||
const filePath = /(input|output)\/[\w\-./ ]+/.exec(summary)?.[0];
|
||||
return { name, inputSummary: summary, filePath };
|
||||
}
|
||||
|
||||
function makeRaw(overrides: Partial<RawJobResult>): RawJobResult {
|
||||
return {
|
||||
taskId: 1,
|
||||
jobId: 'job-1',
|
||||
status: 'succeeded',
|
||||
iterations: null,
|
||||
promptTokens: null,
|
||||
completionTokens: null,
|
||||
workspacePath: '/tmp/x',
|
||||
activityLog: '',
|
||||
toolCalls: [],
|
||||
outputFiles: {},
|
||||
durationMs: 1000,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
const baseTask: BenchTask = {
|
||||
id: 'unit-test',
|
||||
title: 't',
|
||||
prompt: 'p',
|
||||
expected: { must_use_tools: [], forbidden_tools: [] },
|
||||
};
|
||||
|
||||
describe('gradeTools', () => {
|
||||
it('rewards must_use_tools and penalizes forbidden tools', () => {
|
||||
const task: BenchTask = {
|
||||
...baseTask,
|
||||
expected: { must_use_tools: ['ReadExcel', 'Write'], forbidden_tools: ['Bash'] },
|
||||
};
|
||||
const raw = makeRaw({ toolCalls: [tc('ReadExcel', 'input/x.xlsx'), tc('Write', 'output/x.md')] });
|
||||
const r = gradeTools(task, raw);
|
||||
expect(r.score).toBe(1);
|
||||
});
|
||||
|
||||
it('penalizes forbidden tool use', () => {
|
||||
const task: BenchTask = {
|
||||
...baseTask,
|
||||
expected: { must_use_tools: ['Write'], forbidden_tools: ['Bash'] },
|
||||
};
|
||||
const raw = makeRaw({ toolCalls: [tc('Write', 'output/x.md'), tc('Bash', 'rm -rf /tmp/x')] });
|
||||
const r = gradeTools(task, raw);
|
||||
expect(r.score).toBeLessThan(1);
|
||||
});
|
||||
|
||||
it('detects forbidden_tool_for_ext (Read on .xlsx)', () => {
|
||||
const task: BenchTask = {
|
||||
...baseTask,
|
||||
expected: {
|
||||
must_use_tools: [],
|
||||
forbidden_tool_for_ext: { Read: ['.xlsx'] },
|
||||
},
|
||||
};
|
||||
const raw = makeRaw({ toolCalls: [tc('Read', 'input/data.xlsx')] });
|
||||
const r = gradeTools(task, raw);
|
||||
expect(r.score).toBeLessThan(1);
|
||||
expect(r.details.some((d) => d.includes('Read on .xlsx'))).toBe(true);
|
||||
});
|
||||
|
||||
it('does not penalize Read on .txt when xlsx is forbidden', () => {
|
||||
const task: BenchTask = {
|
||||
...baseTask,
|
||||
expected: {
|
||||
must_use_tools: ['Read'],
|
||||
forbidden_tool_for_ext: { Read: ['.xlsx'] },
|
||||
},
|
||||
};
|
||||
const raw = makeRaw({ toolCalls: [tc('Read', 'input/notes.md')] });
|
||||
const r = gradeTools(task, raw);
|
||||
expect(r.score).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('gradeChecklist', () => {
|
||||
const task: BenchTask = {
|
||||
...baseTask,
|
||||
checklist: { required_tools: ['CreateChecklist', 'CheckItem', 'GetChecklist'], min_check_item_calls: 3 },
|
||||
};
|
||||
|
||||
it('full credit when all 3 required tools used and CheckItem >= min', () => {
|
||||
const raw = makeRaw({
|
||||
toolCalls: [
|
||||
tc('CreateChecklist', '...'),
|
||||
tc('CheckItem', '...'),
|
||||
tc('CheckItem', '...'),
|
||||
tc('CheckItem', '...'),
|
||||
tc('GetChecklist', '...'),
|
||||
],
|
||||
});
|
||||
expect(gradeChecklist(task, raw).score).toBe(1);
|
||||
});
|
||||
|
||||
it('partial credit when CheckItem under min', () => {
|
||||
const raw = makeRaw({
|
||||
toolCalls: [tc('CreateChecklist', '...'), tc('CheckItem', '...'), tc('GetChecklist', '...')],
|
||||
});
|
||||
expect(gradeChecklist(task, raw).score).toBeCloseTo(2 / 3, 5);
|
||||
});
|
||||
|
||||
it('zero when no checklist tool used', () => {
|
||||
expect(gradeChecklist(task, makeRaw({})).score).toBe(0);
|
||||
});
|
||||
|
||||
it('returns 1 when checklist is not configured', () => {
|
||||
expect(gradeChecklist(baseTask, makeRaw({})).score).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('gradeInstructionsProgrammatic', () => {
|
||||
it('penalizes when status is not in expected', () => {
|
||||
const task: BenchTask = {
|
||||
...baseTask,
|
||||
expected: { completion_status: ['succeeded'] },
|
||||
};
|
||||
const raw = makeRaw({ status: 'failed' });
|
||||
const r = gradeInstructionsProgrammatic(task, raw);
|
||||
expect(r.score).toBe(0);
|
||||
});
|
||||
|
||||
it('checks file existence and constraints', () => {
|
||||
const task: BenchTask = {
|
||||
...baseTask,
|
||||
expected: { must_produce_files: ['output/report.md'] },
|
||||
grading: {
|
||||
programmatic: {
|
||||
constraints: [
|
||||
{ type: 'file_first_line_equals', file: 'output/report.md', line: '# サマリー' },
|
||||
{ type: 'file_must_contain_in_order', file: 'output/report.md', sections: ['## A', '## B'] },
|
||||
{ type: 'file_section_max_lines', file: 'output/report.md', section: 'A', max: 2 },
|
||||
{ type: 'file_line_starts_with', file: 'output/report.md', prefix: '-', min_lines: 2, section: 'B' },
|
||||
{ type: 'file_line_max_chars', file: 'output/report.md', max: 30, section: 'B' },
|
||||
{ type: 'file_no_pattern', file: 'output/report.md', pattern: '!\\[' },
|
||||
],
|
||||
},
|
||||
},
|
||||
};
|
||||
const goodOutput = [
|
||||
'# サマリー',
|
||||
'',
|
||||
'## A',
|
||||
'short line',
|
||||
'short line 2',
|
||||
'',
|
||||
'## B',
|
||||
'- 短い行 1',
|
||||
'- 短い行 2',
|
||||
].join('\n');
|
||||
const raw = makeRaw({ outputFiles: { 'output/report.md': goodOutput } });
|
||||
expect(gradeInstructionsProgrammatic(task, raw).score).toBe(1);
|
||||
});
|
||||
|
||||
it('flags out-of-order sections', () => {
|
||||
const task: BenchTask = {
|
||||
...baseTask,
|
||||
grading: {
|
||||
programmatic: {
|
||||
constraints: [
|
||||
{ type: 'file_must_contain_in_order', file: 'out.md', sections: ['## A', '## B', '## C'] },
|
||||
],
|
||||
},
|
||||
},
|
||||
};
|
||||
const wrongOrder = '# H\n## C\n## B\n## A\n';
|
||||
const raw = makeRaw({ outputFiles: { 'out.md': wrongOrder } });
|
||||
const r = gradeInstructionsProgrammatic(task, raw);
|
||||
expect(r.score).toBeLessThan(1);
|
||||
});
|
||||
|
||||
it('flags forbidden Markdown image patterns', () => {
|
||||
const task: BenchTask = {
|
||||
...baseTask,
|
||||
grading: {
|
||||
programmatic: {
|
||||
constraints: [{ type: 'file_no_pattern', file: 'out.md', pattern: '!\\[' }],
|
||||
},
|
||||
},
|
||||
};
|
||||
const withImg = '# H\n\n\n';
|
||||
const raw = makeRaw({ outputFiles: { 'out.md': withImg } });
|
||||
const r = gradeInstructionsProgrammatic(task, raw);
|
||||
// 1 status check (true) + 1 forbidden pattern check (false) = 1/2 = 0.5
|
||||
expect(r.score).toBeLessThan(1);
|
||||
expect(r.details.some((d) => d.includes('no_pattern'))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('computeTotal', () => {
|
||||
it('weights axes 30/15/30/25', () => {
|
||||
const total = computeTotal({
|
||||
tools: { score: 1, details: [] },
|
||||
checklist: { score: 1, details: [] },
|
||||
instructions: { score: 1, details: [] },
|
||||
reasoning: { score: 1, details: [] },
|
||||
});
|
||||
expect(total).toBe(100);
|
||||
});
|
||||
|
||||
it('partial credit example', () => {
|
||||
const total = computeTotal({
|
||||
tools: { score: 0.9, details: [] }, // 27
|
||||
checklist: { score: 1.0, details: [] }, // 15
|
||||
instructions: { score: 0.7, details: [] }, // 21
|
||||
reasoning: { score: 0.7, details: [] }, // 17.5 → 17 or 18 after rounding
|
||||
});
|
||||
// 27 + 15 + 21 + 17.5 = 80.5 → 81
|
||||
expect(total).toBe(81);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,266 @@
|
||||
import * as path from 'path';
|
||||
import type {
|
||||
AxisScore,
|
||||
BenchTask,
|
||||
ProgrammaticConstraint,
|
||||
RawJobResult,
|
||||
} from './types.js';
|
||||
|
||||
function clamp01(v: number): number {
|
||||
return Math.max(0, Math.min(1, v));
|
||||
}
|
||||
|
||||
function avg(xs: number[]): number {
|
||||
if (xs.length === 0) return 0;
|
||||
return xs.reduce((a, b) => a + b, 0) / xs.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Axis A — tool calling correctness.
|
||||
*
|
||||
* Components:
|
||||
* - +1 for each must_use tool actually called (averaged)
|
||||
* - -1 for each forbidden_tool used
|
||||
* - -1 for each forbidden_tool_for_ext violation (e.g. Read on .xlsx)
|
||||
*/
|
||||
export function gradeTools(task: BenchTask, raw: RawJobResult): AxisScore {
|
||||
const must = task.expected.must_use_tools ?? [];
|
||||
const forbidden = task.expected.forbidden_tools ?? [];
|
||||
const forbiddenForExt = task.expected.forbidden_tool_for_ext ?? {};
|
||||
|
||||
const used = new Set(raw.toolCalls.map((c) => c.name));
|
||||
const details: string[] = [];
|
||||
|
||||
let mustHits = 0;
|
||||
for (const t of must) {
|
||||
if (used.has(t)) {
|
||||
mustHits++;
|
||||
details.push(`✓ used ${t}`);
|
||||
} else {
|
||||
details.push(`✗ missing ${t}`);
|
||||
}
|
||||
}
|
||||
let forbiddenViolations = 0;
|
||||
for (const t of forbidden) {
|
||||
if (used.has(t)) {
|
||||
forbiddenViolations++;
|
||||
details.push(`✗ forbidden tool used: ${t}`);
|
||||
}
|
||||
}
|
||||
let extViolations = 0;
|
||||
for (const [tool, exts] of Object.entries(forbiddenForExt)) {
|
||||
for (const call of raw.toolCalls) {
|
||||
if (call.name !== tool) continue;
|
||||
const fp = call.filePath ?? call.inputSummary;
|
||||
const ext = path.extname(fp).toLowerCase();
|
||||
if (exts.includes(ext)) {
|
||||
extViolations++;
|
||||
details.push(`✗ ${tool} on ${ext}: ${fp}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const mustScore = must.length === 0 ? 1 : mustHits / must.length;
|
||||
const penalty = (forbiddenViolations + extViolations) * 0.5;
|
||||
const score = clamp01(mustScore - penalty);
|
||||
return { score, details };
|
||||
}
|
||||
|
||||
/**
|
||||
* Axis B — checklist tool usage.
|
||||
*
|
||||
* +1/3 for CreateChecklist used, +1/3 for GetChecklist used, +1/3 if
|
||||
* CheckItem invoked at least min_check_item_calls times.
|
||||
*/
|
||||
export function gradeChecklist(task: BenchTask, raw: RawJobResult): AxisScore {
|
||||
const cfg = task.checklist;
|
||||
if (!cfg) return { score: 1, details: ['(checklist not required)'] };
|
||||
|
||||
const calls = raw.toolCalls.map((c) => c.name);
|
||||
const required = cfg.required_tools;
|
||||
const checkItemCalls = calls.filter((n) => n === 'CheckItem').length;
|
||||
const details: string[] = [];
|
||||
|
||||
let satisfied = 0;
|
||||
for (const t of required) {
|
||||
if (t === 'CheckItem') {
|
||||
if (checkItemCalls >= cfg.min_check_item_calls) {
|
||||
satisfied++;
|
||||
details.push(`✓ CheckItem ${checkItemCalls}/${cfg.min_check_item_calls}`);
|
||||
} else {
|
||||
details.push(`✗ CheckItem ${checkItemCalls}/${cfg.min_check_item_calls}`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (calls.includes(t)) {
|
||||
satisfied++;
|
||||
details.push(`✓ ${t}`);
|
||||
} else {
|
||||
details.push(`✗ missing ${t}`);
|
||||
}
|
||||
}
|
||||
return { score: required.length === 0 ? 1 : satisfied / required.length, details };
|
||||
}
|
||||
|
||||
interface OutputView {
|
||||
text: string;
|
||||
lines: string[];
|
||||
/** Line index keyed by section header (## ...). End is exclusive. */
|
||||
sections: Record<string, { start: number; end: number }>;
|
||||
}
|
||||
|
||||
function buildOutputView(text: string): OutputView {
|
||||
const lines = text.split('\n');
|
||||
const sections: Record<string, { start: number; end: number }> = {};
|
||||
let currentHeader: string | null = null;
|
||||
let currentStart = 0;
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const m = /^##\s+(.+)\s*$/.exec(lines[i]!);
|
||||
if (m) {
|
||||
if (currentHeader !== null) {
|
||||
sections[currentHeader] = { start: currentStart, end: i };
|
||||
}
|
||||
currentHeader = m[1]!.trim();
|
||||
currentStart = i + 1;
|
||||
}
|
||||
}
|
||||
if (currentHeader !== null) {
|
||||
sections[currentHeader] = { start: currentStart, end: lines.length };
|
||||
}
|
||||
return { text, lines, sections };
|
||||
}
|
||||
|
||||
function evaluateConstraint(
|
||||
constraint: ProgrammaticConstraint,
|
||||
outputs: Record<string, string>,
|
||||
): { passed: boolean; detail: string } {
|
||||
const file = (constraint as { file: string }).file;
|
||||
const text = outputs[file];
|
||||
if (text === undefined) {
|
||||
return { passed: false, detail: `[${constraint.type}] file missing: ${file}` };
|
||||
}
|
||||
const view = buildOutputView(text);
|
||||
|
||||
switch (constraint.type) {
|
||||
case 'file_first_line_equals': {
|
||||
const ok = (view.lines[0] ?? '').trim() === constraint.line.trim();
|
||||
return { passed: ok, detail: `[first_line] ${file}: ${ok ? '✓' : `got "${view.lines[0]}"`}` };
|
||||
}
|
||||
case 'file_must_contain_in_order': {
|
||||
let cursor = 0;
|
||||
const missing: string[] = [];
|
||||
for (const sec of constraint.sections) {
|
||||
const idx = view.text.indexOf(sec, cursor);
|
||||
if (idx === -1) missing.push(sec);
|
||||
else cursor = idx + sec.length;
|
||||
}
|
||||
return {
|
||||
passed: missing.length === 0,
|
||||
detail: `[order] ${file}: ${missing.length === 0 ? '✓' : `missing/out-of-order: ${missing.join(', ')}`}`,
|
||||
};
|
||||
}
|
||||
case 'file_line_starts_with': {
|
||||
const range = constraint.section ? view.sections[constraint.section] : { start: 0, end: view.lines.length };
|
||||
if (!range) {
|
||||
return { passed: false, detail: `[starts_with] section "${constraint.section}" not found` };
|
||||
}
|
||||
const target = view.lines.slice(range.start, range.end).filter((l) => l.trim().length > 0);
|
||||
const matched = target.filter((l) => l.trim().startsWith(constraint.prefix)).length;
|
||||
const ok = matched >= constraint.min_lines;
|
||||
return {
|
||||
passed: ok,
|
||||
detail: `[starts_with "${constraint.prefix}"] ${constraint.section ?? file}: ${matched}/${constraint.min_lines}`,
|
||||
};
|
||||
}
|
||||
case 'file_line_max_chars': {
|
||||
const range = constraint.section ? view.sections[constraint.section] : { start: 0, end: view.lines.length };
|
||||
if (!range) {
|
||||
return { passed: false, detail: `[max_chars] section "${constraint.section}" not found` };
|
||||
}
|
||||
const target = view.lines.slice(range.start, range.end).filter((l) => l.trim().length > 0);
|
||||
const violations = target.filter((l) => [...l.trim()].length > constraint.max);
|
||||
return {
|
||||
passed: violations.length === 0,
|
||||
detail: `[max_chars ${constraint.max}] ${constraint.section ?? file}: ${violations.length === 0 ? '✓' : `${violations.length} violations`}`,
|
||||
};
|
||||
}
|
||||
case 'file_section_max_lines': {
|
||||
const range = view.sections[constraint.section];
|
||||
if (!range) {
|
||||
return { passed: false, detail: `[max_lines] section "${constraint.section}" not found` };
|
||||
}
|
||||
const nonEmpty = view.lines.slice(range.start, range.end).filter((l) => l.trim().length > 0).length;
|
||||
return {
|
||||
passed: nonEmpty <= constraint.max,
|
||||
detail: `[max_lines ${constraint.max}] ${constraint.section}: ${nonEmpty} lines`,
|
||||
};
|
||||
}
|
||||
case 'file_no_pattern': {
|
||||
const re = new RegExp(constraint.pattern, 'm');
|
||||
const ok = !re.test(view.text);
|
||||
return { passed: ok, detail: `[no_pattern /${constraint.pattern}/] ${file}: ${ok ? '✓' : '✗'}` };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Axis C — instruction adherence (programmatic part).
|
||||
*
|
||||
* Combines:
|
||||
* - must_produce_files coverage
|
||||
* - each programmatic.constraints check
|
||||
* - completion_status acceptance
|
||||
*/
|
||||
export function gradeInstructionsProgrammatic(task: BenchTask, raw: RawJobResult): AxisScore {
|
||||
const details: string[] = [];
|
||||
const checks: boolean[] = [];
|
||||
|
||||
const acceptable = task.expected.completion_status ?? ['succeeded'];
|
||||
const statusOk = acceptable.includes(raw.status as 'succeeded');
|
||||
checks.push(statusOk);
|
||||
details.push(`[status] ${raw.status} ${statusOk ? '✓' : `(expected one of ${acceptable.join(',')})`}`);
|
||||
|
||||
const mustFiles = task.expected.must_produce_files ?? [];
|
||||
for (const f of mustFiles) {
|
||||
const ok = raw.outputFiles[f] !== undefined && raw.outputFiles[f].length > 0;
|
||||
checks.push(ok);
|
||||
details.push(`[file] ${f} ${ok ? '✓' : '✗ (empty/missing)'}`);
|
||||
}
|
||||
|
||||
for (const c of task.grading?.programmatic?.constraints ?? []) {
|
||||
const r = evaluateConstraint(c, raw.outputFiles);
|
||||
checks.push(r.passed);
|
||||
details.push(r.detail);
|
||||
}
|
||||
|
||||
return { score: checks.length === 0 ? 1 : checks.filter(Boolean).length / checks.length, details };
|
||||
}
|
||||
|
||||
/**
|
||||
* Combine programmatic + judge scores for axis C / D, weighted by config.
|
||||
*/
|
||||
export function combineAxisScores(programmatic: AxisScore, judge: AxisScore | null, judgeWeight: number): AxisScore {
|
||||
if (!judge) return programmatic;
|
||||
const w = clamp01(judgeWeight);
|
||||
return {
|
||||
score: (1 - w) * programmatic.score + w * judge.score,
|
||||
details: [...programmatic.details, ...judge.details.map((d) => `[judge] ${d}`)],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Total score in 0..100 with the described axis weights.
|
||||
*
|
||||
* tools 30 / checklist 15 / instructions 30 / reasoning 25
|
||||
*/
|
||||
export function computeTotal(axes: {
|
||||
tools: AxisScore;
|
||||
checklist: AxisScore;
|
||||
instructions: AxisScore;
|
||||
reasoning: AxisScore;
|
||||
}): number {
|
||||
const sum = axes.tools.score * 30 + axes.checklist.score * 15 + axes.instructions.score * 30 + axes.reasoning.score * 25;
|
||||
return Math.round(sum);
|
||||
}
|
||||
|
||||
export { avg, clamp01 };
|
||||
@@ -0,0 +1,147 @@
|
||||
import type { AxisScore, BenchTask, RawJobResult } from './types.js';
|
||||
import { logger } from '../logger.js';
|
||||
|
||||
export interface JudgeConfig {
|
||||
endpoint: string; // OpenAI-compat /chat/completions base, e.g. http://localhost:11434/v1
|
||||
model: string;
|
||||
apiKey?: string;
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
interface JudgeRubricResult {
|
||||
name: string;
|
||||
score: number;
|
||||
max: number;
|
||||
rationale: string;
|
||||
}
|
||||
|
||||
const SYSTEM_PROMPT = [
|
||||
'You are an evaluation assistant for an autonomous agent benchmark.',
|
||||
'Score each rubric item with an integer in [0, max_score].',
|
||||
'Be strict but fair. Score 0 means the rubric goal is not met at all; max_score means fully met.',
|
||||
'Return ONLY a JSON object of the form {"results":[{"name":"...","score":N,"rationale":"..."}, ...]}.',
|
||||
'No prose outside the JSON. No code fences.',
|
||||
].join('\n');
|
||||
|
||||
function buildJudgePrompt(task: BenchTask, raw: RawJobResult): string {
|
||||
const rubrics = task.grading?.llm_judge?.rubrics ?? [];
|
||||
const outputView = Object.entries(raw.outputFiles)
|
||||
.slice(0, 5) // bound size; we don't expect many files
|
||||
.map(([name, body]) => `--- ${name} ---\n${body.slice(0, 4000)}`)
|
||||
.join('\n\n');
|
||||
|
||||
return [
|
||||
`## Original task prompt`,
|
||||
task.prompt,
|
||||
'',
|
||||
`## Job status`,
|
||||
`status=${raw.status} duration_ms=${raw.durationMs}`,
|
||||
'',
|
||||
`## Output files`,
|
||||
outputView || '(no output files)',
|
||||
'',
|
||||
'## Rubrics',
|
||||
...rubrics.map((r, i) => `${i + 1}. name="${r.name}" max_score=${r.max_score ?? 10}\n ${r.prompt}`),
|
||||
'',
|
||||
`## Output`,
|
||||
`Return JSON with one entry per rubric, in the same order.`,
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
async function callJudge(config: JudgeConfig, userPrompt: string): Promise<string> {
|
||||
const body = {
|
||||
model: config.model,
|
||||
stream: false,
|
||||
messages: [
|
||||
{ role: 'system', content: SYSTEM_PROMPT },
|
||||
{ role: 'user', content: userPrompt },
|
||||
],
|
||||
};
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), config.timeoutMs ?? 120_000);
|
||||
try {
|
||||
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
|
||||
if (config.apiKey) headers['Authorization'] = `Bearer ${config.apiKey}`;
|
||||
const res = await fetch(`${config.endpoint.replace(/\/+$/, '')}/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify(body),
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (!res.ok) {
|
||||
throw new Error(`judge endpoint ${res.status}: ${(await res.text()).slice(0, 400)}`);
|
||||
}
|
||||
const data = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> };
|
||||
return data.choices?.[0]?.message?.content ?? '';
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
function parseJudgeJson(content: string): JudgeRubricResult[] {
|
||||
const trimmed = content.trim().replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/i, '');
|
||||
// Some models still embed prose; extract the first {...} block.
|
||||
const start = trimmed.indexOf('{');
|
||||
const end = trimmed.lastIndexOf('}');
|
||||
if (start < 0 || end <= start) throw new Error(`judge response not JSON: ${content.slice(0, 200)}`);
|
||||
const parsed = JSON.parse(trimmed.slice(start, end + 1)) as { results?: unknown };
|
||||
if (!Array.isArray(parsed.results)) throw new Error('judge response missing results[]');
|
||||
return parsed.results.map((r: unknown) => {
|
||||
const o = r as Record<string, unknown>;
|
||||
return {
|
||||
name: String(o['name'] ?? ''),
|
||||
score: Number(o['score'] ?? 0),
|
||||
max: Number(o['max'] ?? o['max_score'] ?? 10),
|
||||
rationale: String(o['rationale'] ?? ''),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the LLM judge for axis D (reasoning). When the task has no llm_judge
|
||||
* config or `config` is null, returns a 1.0 score so that absence of judge
|
||||
* does not penalize.
|
||||
*/
|
||||
export async function gradeReasoning(
|
||||
task: BenchTask,
|
||||
raw: RawJobResult,
|
||||
config: JudgeConfig | null,
|
||||
): Promise<AxisScore> {
|
||||
const rubrics = task.grading?.llm_judge?.rubrics ?? [];
|
||||
if (!config || rubrics.length === 0) {
|
||||
return { score: 1, details: ['(LLM judge skipped — no config or no rubrics)'] };
|
||||
}
|
||||
const prompt = buildJudgePrompt(task, raw);
|
||||
let raw_response: string;
|
||||
try {
|
||||
raw_response = await callJudge(config, prompt);
|
||||
} catch (err) {
|
||||
logger.warn(`[bench/judge] call failed: ${(err as Error).message}`);
|
||||
return { score: 0, details: [`judge error: ${(err as Error).message}`] };
|
||||
}
|
||||
let results: JudgeRubricResult[];
|
||||
try {
|
||||
results = parseJudgeJson(raw_response);
|
||||
} catch (err) {
|
||||
logger.warn(`[bench/judge] parse failed: ${(err as Error).message}`);
|
||||
return { score: 0, details: [`judge parse error: ${(err as Error).message}`, `raw=${raw_response.slice(0, 200)}`] };
|
||||
}
|
||||
if (results.length === 0) {
|
||||
return { score: 0, details: ['judge returned no results'] };
|
||||
}
|
||||
const normalized = results.map((r) => Math.max(0, Math.min(1, r.score / Math.max(1, r.max))));
|
||||
const score = normalized.reduce((a, b) => a + b, 0) / normalized.length;
|
||||
const details = results.map((r) => `${r.name}: ${r.score}/${r.max} — ${r.rationale}`);
|
||||
return { score, details };
|
||||
}
|
||||
|
||||
export function loadJudgeConfigFromEnv(fallback: { endpoint: string; model: string; apiKey?: string }): JudgeConfig | null {
|
||||
const enabled = process.env['BENCH_JUDGE'] !== 'off';
|
||||
if (!enabled) return null;
|
||||
return {
|
||||
endpoint: process.env['BENCH_JUDGE_ENDPOINT'] ?? fallback.endpoint,
|
||||
model: process.env['BENCH_JUDGE_MODEL'] ?? fallback.model,
|
||||
apiKey: process.env['BENCH_JUDGE_API_KEY'] ?? fallback.apiKey,
|
||||
timeoutMs: 120_000,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { logger } from '../logger.js';
|
||||
import type { BenchTask, RawJobResult, ToolCallObservation } from './types.js';
|
||||
|
||||
const TERMINAL_STATUSES = new Set(['succeeded', 'failed', 'aborted', 'cancelled', 'waiting_human']);
|
||||
|
||||
export interface RunnerOptions {
|
||||
serverUrl: string;
|
||||
benchRoot: string;
|
||||
pollIntervalMs?: number;
|
||||
/** Substituted into prompt {WEB_PORT}. */
|
||||
webPort: number;
|
||||
}
|
||||
|
||||
interface CreateTaskResponse {
|
||||
task: { id: number; workspacePath: string | null };
|
||||
jobId: string;
|
||||
}
|
||||
|
||||
interface TaskDetailResponse {
|
||||
task: {
|
||||
id: number;
|
||||
workspacePath: string | null;
|
||||
latestJob?: {
|
||||
id: string;
|
||||
status: string;
|
||||
contextPromptTokens: number | null;
|
||||
contextLimitTokens: number | null;
|
||||
} | null;
|
||||
};
|
||||
}
|
||||
|
||||
function expandPromptTokens(prompt: string, tokens: Record<string, string>): string {
|
||||
return prompt.replace(/\{(\w+)\}/g, (_, key) => tokens[key] ?? `{${key}}`);
|
||||
}
|
||||
|
||||
async function postJson(url: string, body: unknown): Promise<unknown> {
|
||||
const res = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
if (!res.ok) throw new Error(`POST ${url} → ${res.status}: ${(await res.text()).slice(0, 400)}`);
|
||||
return res.json();
|
||||
}
|
||||
|
||||
async function getJson(url: string): Promise<unknown> {
|
||||
const res = await fetch(url);
|
||||
if (!res.ok) throw new Error(`GET ${url} → ${res.status}: ${(await res.text()).slice(0, 400)}`);
|
||||
return res.json();
|
||||
}
|
||||
|
||||
function buildAttachments(task: BenchTask, benchRoot: string): Array<{ name: string; contentBase64: string }> {
|
||||
const attachments: Array<{ name: string; contentBase64: string }> = [];
|
||||
for (const fx of task.fixtures ?? []) {
|
||||
if (!fx.dest.startsWith('input/')) continue; // web/ fixtures are served by HTTP server, not uploaded
|
||||
const sourcePath = path.resolve(benchRoot, fx.source);
|
||||
if (!fs.existsSync(sourcePath)) {
|
||||
throw new Error(`Fixture source missing: ${sourcePath}`);
|
||||
}
|
||||
const name = fx.dest.slice('input/'.length);
|
||||
attachments.push({ name, contentBase64: fs.readFileSync(sourcePath).toString('base64') });
|
||||
}
|
||||
return attachments;
|
||||
}
|
||||
|
||||
function parseToolCalls(activityLog: string): ToolCallObservation[] {
|
||||
const out: ToolCallObservation[] = [];
|
||||
// Format from summarizeToolInput: "[time] [worker:..] [mode:..] ToolName: arg-summary"
|
||||
// tool line shape after metadata strip: "ToolName: <input summary>"
|
||||
const lines = activityLog.split('\n').map((l) => l.trim()).filter(Boolean);
|
||||
for (const raw of lines) {
|
||||
const stripped = raw
|
||||
.replace(/^\[[^\]]+\]\s+/, '')
|
||||
.replace(/\[worker:[^\]]+\]\s*/g, '')
|
||||
.replace(/\[mode:[^\]]+\]\s*/g, '')
|
||||
.trim();
|
||||
// Skip non-tool lines (preflight, [movement] start/complete/preview, final, ask, context-action)
|
||||
if (
|
||||
stripped.startsWith('preflight:') ||
|
||||
stripped.startsWith('[llm-preflight:') ||
|
||||
stripped.startsWith('context-action:') ||
|
||||
stripped.startsWith('final:') ||
|
||||
stripped.startsWith('ask:') ||
|
||||
stripped.startsWith('[')
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const m = /^([A-Z][A-Za-z0-9_]+):\s*(.+)$/.exec(stripped);
|
||||
if (!m) continue;
|
||||
const name = m[1]!;
|
||||
const inputSummary = m[2]!;
|
||||
// Heuristic: extract first quoted/unquoted "input/..." or "output/..." as filePath
|
||||
const filePath = /(input|output)\/[\w\-./ ]+/.exec(inputSummary)?.[0];
|
||||
out.push({ name, inputSummary, filePath });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function listOutputFiles(workspacePath: string): Record<string, string> {
|
||||
const outDir = path.join(workspacePath, 'output');
|
||||
if (!fs.existsSync(outDir)) return {};
|
||||
const result: Record<string, string> = {};
|
||||
function walk(dir: string, prefix: string): void {
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
const abs = path.join(dir, entry.name);
|
||||
const rel = prefix ? `${prefix}/${entry.name}` : entry.name;
|
||||
if (entry.isDirectory()) {
|
||||
walk(abs, rel);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const buf = fs.readFileSync(abs);
|
||||
// Only return small text files (skip binary/huge)
|
||||
if (buf.length > 200_000 || buf.includes(0)) continue;
|
||||
result[rel] = buf.toString('utf-8');
|
||||
} catch {
|
||||
/* skip unreadable */
|
||||
}
|
||||
}
|
||||
}
|
||||
walk(outDir, '');
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function runOneTask(task: BenchTask, opts: RunnerOptions): Promise<RawJobResult> {
|
||||
const startedAt = Date.now();
|
||||
const promptTokens: Record<string, string> = {
|
||||
WEB_PORT: String(opts.webPort),
|
||||
...(task.prompt_tokens ?? {}),
|
||||
};
|
||||
const expandedPrompt = expandPromptTokens(task.prompt, promptTokens);
|
||||
const attachments = buildAttachments(task, opts.benchRoot);
|
||||
|
||||
logger.info(`[bench/runner] submitting task=${task.id} attachments=${attachments.length}`);
|
||||
const create = (await postJson(`${opts.serverUrl}/api/local/tasks`, {
|
||||
title: `[bench] ${task.title}`,
|
||||
body: expandedPrompt,
|
||||
piece: task.piece_hint ?? 'chat',
|
||||
profile: 'auto',
|
||||
outputFormat: 'markdown',
|
||||
askPolicy: 'low',
|
||||
priority: 'medium',
|
||||
visibility: 'private',
|
||||
attachments,
|
||||
})) as CreateTaskResponse;
|
||||
const taskId = create.task.id;
|
||||
const jobId = create.jobId;
|
||||
logger.info(`[bench/runner] created task=${task.id} taskId=${taskId} jobId=${jobId}`);
|
||||
|
||||
const timeoutMs = (task.timeout_minutes ?? 10) * 60_000;
|
||||
const pollMs = opts.pollIntervalMs ?? 3_000;
|
||||
let detail: TaskDetailResponse['task'] | null = null;
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
await new Promise((r) => setTimeout(r, pollMs));
|
||||
try {
|
||||
const got = (await getJson(`${opts.serverUrl}/api/local/tasks/${taskId}`)) as TaskDetailResponse;
|
||||
detail = got.task;
|
||||
const status = detail.latestJob?.status ?? 'queued';
|
||||
logger.info(`[bench/runner] task=${task.id} status=${status}`);
|
||||
if (TERMINAL_STATUSES.has(status)) break;
|
||||
} catch (err) {
|
||||
logger.warn(`[bench/runner] poll error: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
if (!detail) throw new Error(`Task ${taskId} never reported state`);
|
||||
if (!detail.latestJob) throw new Error(`Task ${taskId} has no latestJob`);
|
||||
|
||||
const workspacePath = detail.workspacePath ?? '';
|
||||
const activityLogPath = path.join(workspacePath, 'logs', 'activity.log');
|
||||
const activityLog = fs.existsSync(activityLogPath) ? fs.readFileSync(activityLogPath, 'utf-8') : '';
|
||||
const toolCalls = parseToolCalls(activityLog);
|
||||
const outputFiles = listOutputFiles(workspacePath);
|
||||
|
||||
return {
|
||||
taskId,
|
||||
jobId: detail.latestJob.id,
|
||||
status: detail.latestJob.status,
|
||||
iterations: null,
|
||||
promptTokens: detail.latestJob.contextPromptTokens,
|
||||
completionTokens: null,
|
||||
workspacePath,
|
||||
activityLog,
|
||||
toolCalls,
|
||||
outputFiles,
|
||||
durationMs: Date.now() - startedAt,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import type { BenchResult } from './types.js';
|
||||
|
||||
function pad(s: string, n: number): string {
|
||||
return s.length >= n ? s : s + ' '.repeat(n - s.length);
|
||||
}
|
||||
|
||||
function bar(score: number): string {
|
||||
const filled = Math.round(score * 10);
|
||||
return '█'.repeat(filled) + '░'.repeat(10 - filled);
|
||||
}
|
||||
|
||||
function formatAxis(name: string, score: number, weight: number): string {
|
||||
return `${pad(name, 16)} ${bar(score)} ${(score * 100).toFixed(0).padStart(3)}% (weight ${weight})`;
|
||||
}
|
||||
|
||||
export function formatResultMarkdown(result: BenchResult): string {
|
||||
const r = result;
|
||||
const minutes = (r.raw.durationMs / 60_000).toFixed(1);
|
||||
return [
|
||||
`## ${r.taskTitle} (id: \`${r.taskId}\`)`,
|
||||
'',
|
||||
`- Started: ${r.startedAt}`,
|
||||
`- Finished: ${r.finishedAt}`,
|
||||
`- Status: \`${r.raw.status}\``,
|
||||
`- Duration: ${minutes} min`,
|
||||
`- Tool calls: ${r.raw.toolCalls.length}`,
|
||||
r.raw.promptTokens !== null ? `- Last prompt tokens: ${r.raw.promptTokens?.toLocaleString()}` : null,
|
||||
`- Workspace: \`${r.raw.workspacePath}\``,
|
||||
'',
|
||||
'### Scores',
|
||||
'',
|
||||
'```',
|
||||
formatAxis('A. Tools', r.axes.tools.score, 30),
|
||||
formatAxis('B. Checklist', r.axes.checklist.score, 15),
|
||||
formatAxis('C. Instructions', r.axes.instructions.score, 30),
|
||||
formatAxis('D. Reasoning', r.axes.reasoning.score, 25),
|
||||
'```',
|
||||
'',
|
||||
`**Total: ${r.total} / 100**`,
|
||||
'',
|
||||
'### Details',
|
||||
'',
|
||||
'<details><summary>A. Tools</summary>',
|
||||
'',
|
||||
...r.axes.tools.details.map((d) => `- ${d}`),
|
||||
'',
|
||||
'</details>',
|
||||
'',
|
||||
'<details><summary>B. Checklist</summary>',
|
||||
'',
|
||||
...r.axes.checklist.details.map((d) => `- ${d}`),
|
||||
'',
|
||||
'</details>',
|
||||
'',
|
||||
'<details><summary>C. Instructions</summary>',
|
||||
'',
|
||||
...r.axes.instructions.details.map((d) => `- ${d}`),
|
||||
'',
|
||||
'</details>',
|
||||
'',
|
||||
'<details><summary>D. Reasoning</summary>',
|
||||
'',
|
||||
...r.axes.reasoning.details.map((d) => `- ${d}`),
|
||||
'',
|
||||
'</details>',
|
||||
'',
|
||||
'<details><summary>Tool call sequence</summary>',
|
||||
'',
|
||||
'```',
|
||||
...r.raw.toolCalls.map((c) => `${c.name}: ${c.inputSummary}`),
|
||||
'```',
|
||||
'',
|
||||
'</details>',
|
||||
'',
|
||||
]
|
||||
.filter((x) => x !== null)
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
export function writeRunSummary(resultDir: string, results: BenchResult[]): string {
|
||||
const overallTotal = results.length === 0 ? 0 : Math.round(results.reduce((a, r) => a + r.total, 0) / results.length);
|
||||
const summary = [
|
||||
`# Bench run @ ${new Date().toISOString()}`,
|
||||
'',
|
||||
`**Overall: ${overallTotal} / 100** (avg of ${results.length} task${results.length === 1 ? '' : 's'})`,
|
||||
'',
|
||||
'| Task | Status | Total | A | B | C | D |',
|
||||
'| --- | --- | ---: | ---: | ---: | ---: | ---: |',
|
||||
...results.map(
|
||||
(r) =>
|
||||
`| \`${r.taskId}\` | ${r.raw.status} | ${r.total} | ${(r.axes.tools.score * 100).toFixed(0)}% | ${(r.axes.checklist.score * 100).toFixed(0)}% | ${(r.axes.instructions.score * 100).toFixed(0)}% | ${(r.axes.reasoning.score * 100).toFixed(0)}% |`,
|
||||
),
|
||||
'',
|
||||
'---',
|
||||
'',
|
||||
...results.map((r) => formatResultMarkdown(r)),
|
||||
].join('\n');
|
||||
|
||||
const summaryPath = path.join(resultDir, 'summary.md');
|
||||
fs.writeFileSync(summaryPath, summary, 'utf-8');
|
||||
return summaryPath;
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
export interface BenchFixtureSpec {
|
||||
/** Path under bench/ root, e.g. "fixtures/sales.xlsx". */
|
||||
source: string;
|
||||
/** Destination relative to the task workspace. Either "input/<name>" (uploaded as attachment) or "web/<path>" (served by HTTP server, no upload). */
|
||||
dest: string;
|
||||
}
|
||||
|
||||
export interface BenchExpectations {
|
||||
must_use_tools?: string[];
|
||||
forbidden_tools?: string[];
|
||||
/** Tools forbidden against specific file extensions: e.g. { Read: [".xlsx", ".docx"] }. */
|
||||
forbidden_tool_for_ext?: Record<string, string[]>;
|
||||
must_produce_files?: string[];
|
||||
/** Acceptable terminal job statuses. Default: ["succeeded"]. */
|
||||
completion_status?: Array<'succeeded' | 'waiting_human' | 'failed' | 'aborted' | 'cancelled'>;
|
||||
}
|
||||
|
||||
export type ProgrammaticConstraint =
|
||||
| { type: 'file_first_line_equals'; file: string; line: string }
|
||||
| { type: 'file_must_contain_in_order'; file: string; sections: string[] }
|
||||
| { type: 'file_line_starts_with'; file: string; prefix: string; min_lines: number; section?: string }
|
||||
| { type: 'file_line_max_chars'; file: string; max: number; section?: string }
|
||||
| { type: 'file_section_max_lines'; file: string; section: string; max: number }
|
||||
| { type: 'file_no_pattern'; file: string; pattern: string };
|
||||
|
||||
export interface BenchGrading {
|
||||
programmatic?: { weight?: number; constraints?: ProgrammaticConstraint[] };
|
||||
llm_judge?: {
|
||||
weight?: number;
|
||||
rubrics: Array<{
|
||||
name: string;
|
||||
prompt: string;
|
||||
max_score?: number; // default 10
|
||||
}>;
|
||||
};
|
||||
}
|
||||
|
||||
export interface BenchTask {
|
||||
id: string;
|
||||
title: string;
|
||||
prompt: string;
|
||||
piece_hint?: string;
|
||||
fixtures?: BenchFixtureSpec[];
|
||||
/** Tokens substituted into the prompt at runtime: e.g. {WEB_PORT}. */
|
||||
prompt_tokens?: Record<string, string>;
|
||||
expected: BenchExpectations;
|
||||
grading?: BenchGrading;
|
||||
/** Required checklist tools and minimum CheckItem count for axis B. */
|
||||
checklist?: { required_tools: string[]; min_check_item_calls: number };
|
||||
timeout_minutes?: number;
|
||||
}
|
||||
|
||||
export interface ToolCallObservation {
|
||||
name: string;
|
||||
/** Approximate input shown in activity.log; not the full tool input. */
|
||||
inputSummary: string;
|
||||
/** Tool first-arg-as-path heuristic, when available. */
|
||||
filePath?: string;
|
||||
}
|
||||
|
||||
export interface RawJobResult {
|
||||
taskId: number;
|
||||
jobId: string;
|
||||
status: string;
|
||||
iterations?: number | null;
|
||||
promptTokens?: number | null;
|
||||
completionTokens?: number | null;
|
||||
workspacePath: string;
|
||||
activityLog: string;
|
||||
toolCalls: ToolCallObservation[];
|
||||
outputFiles: Record<string, string>;
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
export interface AxisScore {
|
||||
/** 0..1 normalized score. */
|
||||
score: number;
|
||||
/** Human readable detail entries. */
|
||||
details: string[];
|
||||
}
|
||||
|
||||
export interface BenchResult {
|
||||
taskId: string;
|
||||
taskTitle: string;
|
||||
startedAt: string;
|
||||
finishedAt: string;
|
||||
raw: RawJobResult;
|
||||
axes: {
|
||||
tools: AxisScore;
|
||||
checklist: AxisScore;
|
||||
instructions: AxisScore;
|
||||
reasoning: AxisScore;
|
||||
};
|
||||
/** Weighted total 0..100. */
|
||||
total: number;
|
||||
}
|
||||
Reference in New Issue
Block a user