97 lines
2.9 KiB
TypeScript
97 lines
2.9 KiB
TypeScript
export interface BenchFixtureSpec {
|
|
/** Path under bench/ root, e.g. "fixtures/sales.xlsx". */
|
|
source: string;
|
|
/** Destination relative to the task workspace. Either "input/<name>" (uploaded as attachment) or "web/<path>" (served by HTTP server, no upload). */
|
|
dest: string;
|
|
}
|
|
|
|
export interface BenchExpectations {
|
|
must_use_tools?: string[];
|
|
forbidden_tools?: string[];
|
|
/** Tools forbidden against specific file extensions: e.g. { Read: [".xlsx", ".docx"] }. */
|
|
forbidden_tool_for_ext?: Record<string, string[]>;
|
|
must_produce_files?: string[];
|
|
/** Acceptable terminal job statuses. Default: ["succeeded"]. */
|
|
completion_status?: Array<'succeeded' | 'waiting_human' | 'failed' | 'aborted' | 'cancelled'>;
|
|
}
|
|
|
|
export type ProgrammaticConstraint =
|
|
| { type: 'file_first_line_equals'; file: string; line: string }
|
|
| { type: 'file_must_contain_in_order'; file: string; sections: string[] }
|
|
| { type: 'file_line_starts_with'; file: string; prefix: string; min_lines: number; section?: string }
|
|
| { type: 'file_line_max_chars'; file: string; max: number; section?: string }
|
|
| { type: 'file_section_max_lines'; file: string; section: string; max: number }
|
|
| { type: 'file_no_pattern'; file: string; pattern: string };
|
|
|
|
export interface BenchGrading {
|
|
programmatic?: { weight?: number; constraints?: ProgrammaticConstraint[] };
|
|
llm_judge?: {
|
|
weight?: number;
|
|
rubrics: Array<{
|
|
name: string;
|
|
prompt: string;
|
|
max_score?: number; // default 10
|
|
}>;
|
|
};
|
|
}
|
|
|
|
export interface BenchTask {
|
|
id: string;
|
|
title: string;
|
|
prompt: string;
|
|
piece_hint?: string;
|
|
fixtures?: BenchFixtureSpec[];
|
|
/** Tokens substituted into the prompt at runtime: e.g. {WEB_PORT}. */
|
|
prompt_tokens?: Record<string, string>;
|
|
expected: BenchExpectations;
|
|
grading?: BenchGrading;
|
|
/** Required checklist tools and minimum CheckItem count for axis B. */
|
|
checklist?: { required_tools: string[]; min_check_item_calls: number };
|
|
timeout_minutes?: number;
|
|
}
|
|
|
|
export interface ToolCallObservation {
|
|
name: string;
|
|
/** Approximate input shown in activity.log; not the full tool input. */
|
|
inputSummary: string;
|
|
/** Tool first-arg-as-path heuristic, when available. */
|
|
filePath?: string;
|
|
}
|
|
|
|
export interface RawJobResult {
|
|
taskId: number;
|
|
jobId: string;
|
|
status: string;
|
|
iterations?: number | null;
|
|
promptTokens?: number | null;
|
|
completionTokens?: number | null;
|
|
workspacePath: string;
|
|
activityLog: string;
|
|
toolCalls: ToolCallObservation[];
|
|
outputFiles: Record<string, string>;
|
|
durationMs: number;
|
|
}
|
|
|
|
export interface AxisScore {
|
|
/** 0..1 normalized score. */
|
|
score: number;
|
|
/** Human readable detail entries. */
|
|
details: string[];
|
|
}
|
|
|
|
export interface BenchResult {
|
|
taskId: string;
|
|
taskTitle: string;
|
|
startedAt: string;
|
|
finishedAt: string;
|
|
raw: RawJobResult;
|
|
axes: {
|
|
tools: AxisScore;
|
|
checklist: AxisScore;
|
|
instructions: AxisScore;
|
|
reasoning: AxisScore;
|
|
};
|
|
/** Weighted total 0..100. */
|
|
total: number;
|
|
}
|