sync: update from private repo (2336949a)

scripts/bench-run.ts

@@ -5,7 +5,7 @@
  *   npm run bench                 # all tasks
  *   npm run bench -- --task=composite-mini-report
  *   npm run bench -- --server=http://127.0.0.1:9876
- *   BENCH_JUDGE=off npm run bench # skip LLM judge (axis D fixed at 1.0)
+ *   BENCH_JUDGE=off npm run bench # skip LLM judge (axis D not graded, excluded from the total)
  *
  * Prereq: scripts/server.sh start (or `npm start`) running on the configured port.
  */
@@ -14,15 +14,10 @@ import * as path from 'path';
 import { parse as parseYaml } from 'yaml';
 import { startFixtureServer } from '../src/bench/fixture-server.js';
 import { runOneTask } from '../src/bench/runner.js';
-import {
-  computeTotal,
-  gradeChecklist,
-  gradeInstructionsProgrammatic,
-  gradeTools,
-} from '../src/bench/grader.js';
-import { gradeReasoning, loadJudgeConfigFromEnv } from '../src/bench/judge.js';
-import { writeRunSummary } from '../src/bench/summary.js';
-import type { BenchResult, BenchTask, RawJobResult } from '../src/bench/types.js';
+import { exitCodeForResults, runAndGradeTask } from '../src/bench/grade-run.js';
+import { loadJudgeConfigFromProvider } from '../src/bench/judge.js';
+import { formatAxisCell, formatTotalCell, writeRunSummary } from '../src/bench/summary.js';
+import type { BenchResult, BenchTask } from '../src/bench/types.js';
 import { loadConfig } from '../src/config.js';
 import { logger } from '../src/logger.js';
 
@@ -87,12 +82,7 @@ function copyWorkspaceArtifacts(rawWorkspace: string, dest: string): void {
 async function main(): Promise<void> {
   const args = parseArgs();
   const config = loadConfig();
-  const fallbackWorker = config.provider.workers?.[0];
-  const judgeConfig = loadJudgeConfigFromEnv({
-    endpoint: fallbackWorker?.endpoint ?? config.provider.baseUrl ?? 'http://127.0.0.1:11434/v1',
-    model: fallbackWorker?.model ?? config.provider.model ?? '',
-    apiKey: config.provider.apiKey ?? undefined,
-  });
+  const judgeConfig = loadJudgeConfigFromProvider(config.provider);
 
   const tasks = loadTasks(args.benchRoot, args.taskFilter);
   if (tasks.length === 0) {
@@ -108,37 +98,25 @@ async function main(): Promise<void> {
   const results: BenchResult[] = [];
   try {
     for (const task of tasks) {
-      const startedAt = new Date().toISOString();
       logger.info(`[bench] === task: ${task.id} ===`);
-      let raw: RawJobResult;
-      try {
-        raw = await runOneTask(task, {
-          serverUrl: args.serverUrl,
-          benchRoot: args.benchRoot,
-          webPort: fixtureServer.port,
-        });
-      } catch (err) {
-        logger.error(`[bench] runner failed for ${task.id}: ${(err as Error).message}`);
-        continue;
-      }
-      const tools = gradeTools(task, raw);
-      const checklist = gradeChecklist(task, raw);
-      const instructions = gradeInstructionsProgrammatic(task, raw);
-      const reasoning = await gradeReasoning(task, raw, judgeConfig);
-      const finishedAt = new Date().toISOString();
-      const result: BenchResult = {
-        taskId: task.id,
-        taskTitle: task.title,
-        startedAt,
-        finishedAt,
-        raw,
-        axes: { tools, checklist, instructions, reasoning },
-        total: 0,
-      };
-      result.total = computeTotal(result.axes);
       const taskResultDir = path.join(args.resultDir, task.id);
       fs.mkdirSync(taskResultDir, { recursive: true });
-      copyWorkspaceArtifacts(raw.workspacePath, path.join(taskResultDir, 'workspace'));
+
+      const result = await runAndGradeTask(task, {
+        runTask: (t) =>
+          runOneTask(t, {
+            serverUrl: args.serverUrl,
+            benchRoot: args.benchRoot,
+            webPort: fixtureServer.port,
+          }),
+        judgeConfig,
+      });
+
+      if (result.outcome === 'failed') {
+        logger.error(`[bench] runner failed for ${task.id}: ${result.error}`);
+      } else if (result.raw) {
+        copyWorkspaceArtifacts(result.raw.workspacePath, path.join(taskResultDir, 'workspace'));
+      }
       fs.writeFileSync(path.join(taskResultDir, 'result.json'), JSON.stringify(result, null, 2), 'utf-8');
       results.push(result);
     }
@@ -149,10 +127,28 @@ async function main(): Promise<void> {
   const summaryPath = writeRunSummary(args.resultDir, results);
   logger.info(`[bench] wrote ${summaryPath}`);
   // Concise stdout summary so CI / humans see totals without opening the file.
+  // Axes that were not graded print their status, never a percentage, and the
+  // scored weight travels with the total so two runs are comparable on purpose.
   for (const r of results) {
+    if (r.outcome === 'failed') {
+      // eslint-disable-next-line no-console
+      console.log(`${r.taskId}: FAILED (not scored)  error=${r.error ?? 'unknown'}`);
+      continue;
+    }
+    const weight = r.scoring ? `${r.scoring.scoredWeight}/${r.scoring.maxWeight}` : '?';
     // eslint-disable-next-line no-console
-    console.log(`${r.taskId}: total=${r.total}/100  status=${r.raw.status}  tools=${(r.axes.tools.score * 100).toFixed(0)}%  checklist=${(r.axes.checklist.score * 100).toFixed(0)}%  instr=${(r.axes.instructions.score * 100).toFixed(0)}%  reasoning=${(r.axes.reasoning.score * 100).toFixed(0)}%`);
+    console.log(
+      `${r.taskId}: total=${formatTotalCell(r)}/100 (scored weight ${weight})  status=${r.raw?.status ?? '-'}  ` +
+        `tools=${formatAxisCell(r.axes?.tools)}  checklist=${formatAxisCell(r.axes?.checklist)}  ` +
+        `instr=${formatAxisCell(r.axes?.instructions)}  reasoning=${formatAxisCell(r.axes?.reasoning)}`,
+    );
   }
+  const failedCount = results.filter((r) => r.outcome === 'failed').length;
+  if (failedCount > 0) {
+    // eslint-disable-next-line no-console
+    console.log(`${failedCount} of ${results.length} task(s) failed to run and were not scored.`);
+  }
+  process.exitCode = exitCodeForResults(results);
 }
 
 main().catch((err) => {