import * as fs from 'fs'; import * as path from 'path'; import type { BenchResult } from './types.js'; function pad(s: string, n: number): string { return s.length >= n ? s : s + ' '.repeat(n - s.length); } function bar(score: number): string { const filled = Math.round(score * 10); return '█'.repeat(filled) + '░'.repeat(10 - filled); } function formatAxis(name: string, score: number, weight: number): string { return `${pad(name, 16)} ${bar(score)} ${(score * 100).toFixed(0).padStart(3)}% (weight ${weight})`; } export function formatResultMarkdown(result: BenchResult): string { const r = result; const minutes = (r.raw.durationMs / 60_000).toFixed(1); return [ `## ${r.taskTitle} (id: \`${r.taskId}\`)`, '', `- Started: ${r.startedAt}`, `- Finished: ${r.finishedAt}`, `- Status: \`${r.raw.status}\``, `- Duration: ${minutes} min`, `- Tool calls: ${r.raw.toolCalls.length}`, r.raw.promptTokens !== null ? `- Last prompt tokens: ${r.raw.promptTokens?.toLocaleString()}` : null, `- Workspace: \`${r.raw.workspacePath}\``, '', '### Scores', '', '```', formatAxis('A. Tools', r.axes.tools.score, 30), formatAxis('B. Checklist', r.axes.checklist.score, 15), formatAxis('C. Instructions', r.axes.instructions.score, 30), formatAxis('D. Reasoning', r.axes.reasoning.score, 25), '```', '', `**Total: ${r.total} / 100**`, '', '### Details', '', '
A. Tools', '', ...r.axes.tools.details.map((d) => `- ${d}`), '', '
', '', '
B. Checklist', '', ...r.axes.checklist.details.map((d) => `- ${d}`), '', '
', '', '
C. Instructions', '', ...r.axes.instructions.details.map((d) => `- ${d}`), '', '
', '', '
D. Reasoning', '', ...r.axes.reasoning.details.map((d) => `- ${d}`), '', '
', '', '
Tool call sequence', '', '```', ...r.raw.toolCalls.map((c) => `${c.name}: ${c.inputSummary}`), '```', '', '
', '', ] .filter((x) => x !== null) .join('\n'); } export function writeRunSummary(resultDir: string, results: BenchResult[]): string { const overallTotal = results.length === 0 ? 0 : Math.round(results.reduce((a, r) => a + r.total, 0) / results.length); const summary = [ `# Bench run @ ${new Date().toISOString()}`, '', `**Overall: ${overallTotal} / 100** (avg of ${results.length} task${results.length === 1 ? '' : 's'})`, '', '| Task | Status | Total | A | B | C | D |', '| --- | --- | ---: | ---: | ---: | ---: | ---: |', ...results.map( (r) => `| \`${r.taskId}\` | ${r.raw.status} | ${r.total} | ${(r.axes.tools.score * 100).toFixed(0)}% | ${(r.axes.checklist.score * 100).toFixed(0)}% | ${(r.axes.instructions.score * 100).toFixed(0)}% | ${(r.axes.reasoning.score * 100).toFixed(0)}% |`, ), '', '---', '', ...results.map((r) => formatResultMarkdown(r)), ].join('\n'); const summaryPath = path.join(resultDir, 'summary.md'); fs.writeFileSync(summaryPath, summary, 'utf-8'); return summaryPath; }