105 lines
3.2 KiB
TypeScript
105 lines
3.2 KiB
TypeScript
import * as fs from 'fs';
|
|
import * as path from 'path';
|
|
import type { BenchResult } from './types.js';
|
|
|
|
function pad(s: string, n: number): string {
|
|
return s.length >= n ? s : s + ' '.repeat(n - s.length);
|
|
}
|
|
|
|
function bar(score: number): string {
|
|
const filled = Math.round(score * 10);
|
|
return '█'.repeat(filled) + '░'.repeat(10 - filled);
|
|
}
|
|
|
|
function formatAxis(name: string, score: number, weight: number): string {
|
|
return `${pad(name, 16)} ${bar(score)} ${(score * 100).toFixed(0).padStart(3)}% (weight ${weight})`;
|
|
}
|
|
|
|
export function formatResultMarkdown(result: BenchResult): string {
|
|
const r = result;
|
|
const minutes = (r.raw.durationMs / 60_000).toFixed(1);
|
|
return [
|
|
`## ${r.taskTitle} (id: \`${r.taskId}\`)`,
|
|
'',
|
|
`- Started: ${r.startedAt}`,
|
|
`- Finished: ${r.finishedAt}`,
|
|
`- Status: \`${r.raw.status}\``,
|
|
`- Duration: ${minutes} min`,
|
|
`- Tool calls: ${r.raw.toolCalls.length}`,
|
|
r.raw.promptTokens !== null ? `- Last prompt tokens: ${r.raw.promptTokens?.toLocaleString()}` : null,
|
|
`- Workspace: \`${r.raw.workspacePath}\``,
|
|
'',
|
|
'### Scores',
|
|
'',
|
|
'```',
|
|
formatAxis('A. Tools', r.axes.tools.score, 30),
|
|
formatAxis('B. Checklist', r.axes.checklist.score, 15),
|
|
formatAxis('C. Instructions', r.axes.instructions.score, 30),
|
|
formatAxis('D. Reasoning', r.axes.reasoning.score, 25),
|
|
'```',
|
|
'',
|
|
`**Total: ${r.total} / 100**`,
|
|
'',
|
|
'### Details',
|
|
'',
|
|
'<details><summary>A. Tools</summary>',
|
|
'',
|
|
...r.axes.tools.details.map((d) => `- ${d}`),
|
|
'',
|
|
'</details>',
|
|
'',
|
|
'<details><summary>B. Checklist</summary>',
|
|
'',
|
|
...r.axes.checklist.details.map((d) => `- ${d}`),
|
|
'',
|
|
'</details>',
|
|
'',
|
|
'<details><summary>C. Instructions</summary>',
|
|
'',
|
|
...r.axes.instructions.details.map((d) => `- ${d}`),
|
|
'',
|
|
'</details>',
|
|
'',
|
|
'<details><summary>D. Reasoning</summary>',
|
|
'',
|
|
...r.axes.reasoning.details.map((d) => `- ${d}`),
|
|
'',
|
|
'</details>',
|
|
'',
|
|
'<details><summary>Tool call sequence</summary>',
|
|
'',
|
|
'```',
|
|
...r.raw.toolCalls.map((c) => `${c.name}: ${c.inputSummary}`),
|
|
'```',
|
|
'',
|
|
'</details>',
|
|
'',
|
|
]
|
|
.filter((x) => x !== null)
|
|
.join('\n');
|
|
}
|
|
|
|
export function writeRunSummary(resultDir: string, results: BenchResult[]): string {
|
|
const overallTotal = results.length === 0 ? 0 : Math.round(results.reduce((a, r) => a + r.total, 0) / results.length);
|
|
const summary = [
|
|
`# Bench run @ ${new Date().toISOString()}`,
|
|
'',
|
|
`**Overall: ${overallTotal} / 100** (avg of ${results.length} task${results.length === 1 ? '' : 's'})`,
|
|
'',
|
|
'| Task | Status | Total | A | B | C | D |',
|
|
'| --- | --- | ---: | ---: | ---: | ---: | ---: |',
|
|
...results.map(
|
|
(r) =>
|
|
`| \`${r.taskId}\` | ${r.raw.status} | ${r.total} | ${(r.axes.tools.score * 100).toFixed(0)}% | ${(r.axes.checklist.score * 100).toFixed(0)}% | ${(r.axes.instructions.score * 100).toFixed(0)}% | ${(r.axes.reasoning.score * 100).toFixed(0)}% |`,
|
|
),
|
|
'',
|
|
'---',
|
|
'',
|
|
...results.map((r) => formatResultMarkdown(r)),
|
|
].join('\n');
|
|
|
|
const summaryPath = path.join(resultDir, 'summary.md');
|
|
fs.writeFileSync(summaryPath, summary, 'utf-8');
|
|
return summaryPath;
|
|
}
|