maestro/scripts/bench-run.ts

#!/usr/bin/env -S npx tsx
/**
 * CLI entry for the benchmark runner.
 *
 *   npm run bench                 # all tasks
 *   npm run bench -- --task=composite-mini-report
 *   npm run bench -- --server=http://127.0.0.1:9876
 *   BENCH_JUDGE=off npm run bench # skip LLM judge (axis D not graded, excluded from the total)
 *
 * Prereq: scripts/server.sh start (or `npm start`) running on the configured port.
 */
import * as fs from 'fs';
import * as path from 'path';
import { parse as parseYaml } from 'yaml';
import { startFixtureServer } from '../src/bench/fixture-server.js';
import { runOneTask } from '../src/bench/runner.js';
import { exitCodeForResults, runAndGradeTask } from '../src/bench/grade-run.js';
import { loadJudgeConfigFromProvider } from '../src/bench/judge.js';
import { formatAxisCell, formatTotalCell, writeRunSummary } from '../src/bench/summary.js';
import type { BenchResult, BenchTask } from '../src/bench/types.js';
import { loadConfig } from '../src/config.js';
import { logger } from '../src/logger.js';

interface Args {
  taskFilter?: string;
  serverUrl: string;
  benchRoot: string;
  resultDir: string;
}

function parseArgs(): Args {
  const argv = process.argv.slice(2);
  let taskFilter: string | undefined;
  let serverUrl = process.env['BENCH_SERVER_URL'] ?? 'http://127.0.0.1:9876';
  for (const a of argv) {
    if (a.startsWith('--task=')) taskFilter = a.slice('--task='.length);
    else if (a.startsWith('--server=')) serverUrl = a.slice('--server='.length);
  }
  const benchRoot = path.resolve(process.cwd(), 'bench');
  const runId = new Date().toISOString().replace(/[:.]/g, '-');
  const resultDir = path.join(benchRoot, 'results', runId);
  return { taskFilter, serverUrl, benchRoot, resultDir };
}

function loadTasks(benchRoot: string, filter?: string): BenchTask[] {
  const taskDir = path.join(benchRoot, 'tasks');
  if (!fs.existsSync(taskDir)) return [];
  const files = fs.readdirSync(taskDir).filter((f) => f.endsWith('.yaml') || f.endsWith('.yml'));
  const tasks: BenchTask[] = [];
  for (const f of files) {
    const parsed = parseYaml(fs.readFileSync(path.join(taskDir, f), 'utf-8')) as BenchTask;
    if (filter && parsed.id !== filter) continue;
    tasks.push(parsed);
  }
  return tasks;
}

function copyWorkspaceArtifacts(rawWorkspace: string, dest: string): void {
  if (!rawWorkspace || !fs.existsSync(rawWorkspace)) return;
  fs.mkdirSync(dest, { recursive: true });
  function copyDir(src: string, target: string): void {
    fs.mkdirSync(target, { recursive: true });
    for (const entry of fs.readdirSync(src, { withFileTypes: true })) {
      const s = path.join(src, entry.name);
      const t = path.join(target, entry.name);
      if (entry.isDirectory()) copyDir(s, t);
      else if (entry.isFile()) {
        try {
          fs.copyFileSync(s, t);
        } catch (err) {
          logger.warn(`[bench] copy failed ${s} -> ${t}: ${(err as Error).message}`);
        }
      }
    }
  }
  for (const sub of ['logs', 'output']) {
    const src = path.join(rawWorkspace, sub);
    if (fs.existsSync(src)) copyDir(src, path.join(dest, sub));
  }
}

async function main(): Promise<void> {
  const args = parseArgs();
  const config = loadConfig();
  const judgeConfig = loadJudgeConfigFromProvider(config.provider);

  const tasks = loadTasks(args.benchRoot, args.taskFilter);
  if (tasks.length === 0) {
    logger.error(`[bench] no tasks found (filter=${args.taskFilter ?? 'none'})`);
    process.exit(1);
  }
  fs.mkdirSync(args.resultDir, { recursive: true });
  logger.info(`[bench] run dir: ${args.resultDir}`);
  logger.info(`[bench] server: ${args.serverUrl}`);
  logger.info(`[bench] judge: ${judgeConfig ? `${judgeConfig.endpoint} model=${judgeConfig.model}` : 'off'}`);

  const fixtureServer = await startFixtureServer(path.join(args.benchRoot, 'fixtures', 'web'));
  const results: BenchResult[] = [];
  try {
    for (const task of tasks) {
      logger.info(`[bench] === task: ${task.id} ===`);
      const taskResultDir = path.join(args.resultDir, task.id);
      fs.mkdirSync(taskResultDir, { recursive: true });

      const result = await runAndGradeTask(task, {
        runTask: (t) =>
          runOneTask(t, {
            serverUrl: args.serverUrl,
            benchRoot: args.benchRoot,
            webPort: fixtureServer.port,
          }),
        judgeConfig,
      });

      if (result.outcome === 'failed') {
        logger.error(`[bench] runner failed for ${task.id}: ${result.error}`);
      } else if (result.raw) {
        copyWorkspaceArtifacts(result.raw.workspacePath, path.join(taskResultDir, 'workspace'));
      }
      fs.writeFileSync(path.join(taskResultDir, 'result.json'), JSON.stringify(result, null, 2), 'utf-8');
      results.push(result);
    }
  } finally {
    await fixtureServer.close();
  }

  const summaryPath = writeRunSummary(args.resultDir, results);
  logger.info(`[bench] wrote ${summaryPath}`);
  // Concise stdout summary so CI / humans see totals without opening the file.
  // Axes that were not graded print their status, never a percentage, and the
  // scored weight travels with the total so two runs are comparable on purpose.
  for (const r of results) {
    if (r.outcome === 'failed') {
      // eslint-disable-next-line no-console
      console.log(`${r.taskId}: FAILED (not scored)  error=${r.error ?? 'unknown'}`);
      continue;
    }
    const weight = r.scoring ? `${r.scoring.scoredWeight}/${r.scoring.maxWeight}` : '?';
    // eslint-disable-next-line no-console
    console.log(
      `${r.taskId}: total=${formatTotalCell(r)}/100 (scored weight ${weight})  status=${r.raw?.status ?? '-'}  ` +
        `tools=${formatAxisCell(r.axes?.tools)}  checklist=${formatAxisCell(r.axes?.checklist)}  ` +
        `instr=${formatAxisCell(r.axes?.instructions)}  reasoning=${formatAxisCell(r.axes?.reasoning)}`,
    );
  }
  const failedCount = results.filter((r) => r.outcome === 'failed').length;
  if (failedCount > 0) {
    // eslint-disable-next-line no-console
    console.log(`${failedCount} of ${results.length} task(s) failed to run and were not scored.`);
  }
  process.exitCode = exitCodeForResults(results);
}

main().catch((err) => {
  // eslint-disable-next-line no-console
  console.error(err);
  process.exit(1);
});