feat: initial public release (MAESTRO)

2026-06-03 05:08:00 +00:00
commit f5c7666f6b
823 changed files with 184150 additions and 0 deletions
@@ -0,0 +1,104 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import type { BenchResult } from './types.js';
+
+function pad(s: string, n: number): string {
+  return s.length >= n ? s : s + ' '.repeat(n - s.length);
+}
+
+function bar(score: number): string {
+  const filled = Math.round(score * 10);
+  return '█'.repeat(filled) + '░'.repeat(10 - filled);
+}
+
+function formatAxis(name: string, score: number, weight: number): string {
+  return `${pad(name, 16)} ${bar(score)} ${(score * 100).toFixed(0).padStart(3)}%  (weight ${weight})`;
+}
+
+export function formatResultMarkdown(result: BenchResult): string {
+  const r = result;
+  const minutes = (r.raw.durationMs / 60_000).toFixed(1);
+  return [
+    `## ${r.taskTitle}  (id: \`${r.taskId}\`)`,
+    '',
+    `- Started: ${r.startedAt}`,
+    `- Finished: ${r.finishedAt}`,
+    `- Status: \`${r.raw.status}\``,
+    `- Duration: ${minutes} min`,
+    `- Tool calls: ${r.raw.toolCalls.length}`,
+    r.raw.promptTokens !== null ? `- Last prompt tokens: ${r.raw.promptTokens?.toLocaleString()}` : null,
+    `- Workspace: \`${r.raw.workspacePath}\``,
+    '',
+    '### Scores',
+    '',
+    '```',
+    formatAxis('A. Tools', r.axes.tools.score, 30),
+    formatAxis('B. Checklist', r.axes.checklist.score, 15),
+    formatAxis('C. Instructions', r.axes.instructions.score, 30),
+    formatAxis('D. Reasoning', r.axes.reasoning.score, 25),
+    '```',
+    '',
+    `**Total: ${r.total} / 100**`,
+    '',
+    '### Details',
+    '',
+    '<details><summary>A. Tools</summary>',
+    '',
+    ...r.axes.tools.details.map((d) => `- ${d}`),
+    '',
+    '</details>',
+    '',
+    '<details><summary>B. Checklist</summary>',
+    '',
+    ...r.axes.checklist.details.map((d) => `- ${d}`),
+    '',
+    '</details>',
+    '',
+    '<details><summary>C. Instructions</summary>',
+    '',
+    ...r.axes.instructions.details.map((d) => `- ${d}`),
+    '',
+    '</details>',
+    '',
+    '<details><summary>D. Reasoning</summary>',
+    '',
+    ...r.axes.reasoning.details.map((d) => `- ${d}`),
+    '',
+    '</details>',
+    '',
+    '<details><summary>Tool call sequence</summary>',
+    '',
+    '```',
+    ...r.raw.toolCalls.map((c) => `${c.name}: ${c.inputSummary}`),
+    '```',
+    '',
+    '</details>',
+    '',
+  ]
+    .filter((x) => x !== null)
+    .join('\n');
+}
+
+export function writeRunSummary(resultDir: string, results: BenchResult[]): string {
+  const overallTotal = results.length === 0 ? 0 : Math.round(results.reduce((a, r) => a + r.total, 0) / results.length);
+  const summary = [
+    `# Bench run @ ${new Date().toISOString()}`,
+    '',
+    `**Overall: ${overallTotal} / 100**  (avg of ${results.length} task${results.length === 1 ? '' : 's'})`,
+    '',
+    '| Task | Status | Total | A | B | C | D |',
+    '| --- | --- | ---: | ---: | ---: | ---: | ---: |',
+    ...results.map(
+      (r) =>
+        `| \`${r.taskId}\` | ${r.raw.status} | ${r.total} | ${(r.axes.tools.score * 100).toFixed(0)}% | ${(r.axes.checklist.score * 100).toFixed(0)}% | ${(r.axes.instructions.score * 100).toFixed(0)}% | ${(r.axes.reasoning.score * 100).toFixed(0)}% |`,
+    ),
+    '',
+    '---',
+    '',
+    ...results.map((r) => formatResultMarkdown(r)),
+  ].join('\n');
+
+  const summaryPath = path.join(resultDir, 'summary.md');
+  fs.writeFileSync(summaryPath, summary, 'utf-8');
+  return summaryPath;
+}