feat: initial public release (MAESTRO)

src/bench/types.ts

@@ -0,0 +1,96 @@
+export interface BenchFixtureSpec {
+  /** Path under bench/ root, e.g. "fixtures/sales.xlsx". */
+  source: string;
+  /** Destination relative to the task workspace. Either "input/<name>" (uploaded as attachment) or "web/<path>" (served by HTTP server, no upload). */
+  dest: string;
+}
+
+export interface BenchExpectations {
+  must_use_tools?: string[];
+  forbidden_tools?: string[];
+  /** Tools forbidden against specific file extensions: e.g. { Read: [".xlsx", ".docx"] }. */
+  forbidden_tool_for_ext?: Record<string, string[]>;
+  must_produce_files?: string[];
+  /** Acceptable terminal job statuses. Default: ["succeeded"]. */
+  completion_status?: Array<'succeeded' | 'waiting_human' | 'failed' | 'aborted' | 'cancelled'>;
+}
+
+export type ProgrammaticConstraint =
+  | { type: 'file_first_line_equals'; file: string; line: string }
+  | { type: 'file_must_contain_in_order'; file: string; sections: string[] }
+  | { type: 'file_line_starts_with'; file: string; prefix: string; min_lines: number; section?: string }
+  | { type: 'file_line_max_chars'; file: string; max: number; section?: string }
+  | { type: 'file_section_max_lines'; file: string; section: string; max: number }
+  | { type: 'file_no_pattern'; file: string; pattern: string };
+
+export interface BenchGrading {
+  programmatic?: { weight?: number; constraints?: ProgrammaticConstraint[] };
+  llm_judge?: {
+    weight?: number;
+    rubrics: Array<{
+      name: string;
+      prompt: string;
+      max_score?: number;  // default 10
+    }>;
+  };
+}
+
+export interface BenchTask {
+  id: string;
+  title: string;
+  prompt: string;
+  piece_hint?: string;
+  fixtures?: BenchFixtureSpec[];
+  /** Tokens substituted into the prompt at runtime: e.g. {WEB_PORT}. */
+  prompt_tokens?: Record<string, string>;
+  expected: BenchExpectations;
+  grading?: BenchGrading;
+  /** Required checklist tools and minimum CheckItem count for axis B. */
+  checklist?: { required_tools: string[]; min_check_item_calls: number };
+  timeout_minutes?: number;
+}
+
+export interface ToolCallObservation {
+  name: string;
+  /** Approximate input shown in activity.log; not the full tool input. */
+  inputSummary: string;
+  /** Tool first-arg-as-path heuristic, when available. */
+  filePath?: string;
+}
+
+export interface RawJobResult {
+  taskId: number;
+  jobId: string;
+  status: string;
+  iterations?: number | null;
+  promptTokens?: number | null;
+  completionTokens?: number | null;
+  workspacePath: string;
+  activityLog: string;
+  toolCalls: ToolCallObservation[];
+  outputFiles: Record<string, string>;
+  durationMs: number;
+}
+
+export interface AxisScore {
+  /** 0..1 normalized score. */
+  score: number;
+  /** Human readable detail entries. */
+  details: string[];
+}
+
+export interface BenchResult {
+  taskId: string;
+  taskTitle: string;
+  startedAt: string;
+  finishedAt: string;
+  raw: RawJobResult;
+  axes: {
+    tools: AxisScore;
+    checklist: AxisScore;
+    instructions: AxisScore;
+    reasoning: AxisScore;
+  };
+  /** Weighted total 0..100. */
+  total: number;
+}