feat: initial public release (MAESTRO)
This commit is contained in:
oss-sync
@@ -0,0 +1,347 @@
|
||||
// src/engine/reflection/applier.fuzz.test.ts
|
||||
//
|
||||
// Property / fuzz tests for applyReflection using fast-check.
|
||||
//
|
||||
// Properties asserted for 200 seeded runs:
|
||||
// 1. applyReflection never throws on any generated ReflectionResult
|
||||
// 2. Every memoryDecision has accepted:true OR a known ReflectionRejectionCode
|
||||
// 3. When outcome === 'rejected', the memory dir on disk is byte-for-byte
|
||||
// identical to before the call.
|
||||
// 4. memoryDecisions.length <= 3 regardless of input size.
|
||||
|
||||
import { describe, it } from 'vitest';
|
||||
import * as fc from 'fast-check';
|
||||
import { mkdtempSync, mkdirSync, rmSync, readdirSync, readFileSync, existsSync, writeFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
import { applyReflection, type ApplierDeps } from './applier.js';
|
||||
import { upsertMemoryEntry, readMemoryEntry } from '../../user-folder/memory.js';
|
||||
import { bodyRevision } from './revisions.js';
|
||||
import { Repository } from '../../db/repository.js';
|
||||
import { PieceCatalog } from '../piece-catalog.js';
|
||||
import type { ReflectionInput, ReflectionResult, ReflectionRejectionCode } from './types.js';
|
||||
|
||||
// ── Constants ─────────────────────────────────────────────────────────────────
|
||||
|
||||
const USER_ID = 'u-fuzz';
|
||||
const MAX_BODY = 8192;
|
||||
|
||||
/** All known rejection codes — exactly the 10-member union from types.ts. */
|
||||
const KNOWN_REJECTION_CODES = new Set<ReflectionRejectionCode>([
|
||||
'rejected_unknown_type',
|
||||
'rejected_bad_name',
|
||||
'rejected_body_too_large',
|
||||
'rejected_missing_target',
|
||||
'rejected_stale_target',
|
||||
'rejected_name_collision',
|
||||
'rejected_target_piece_mismatch',
|
||||
'rejected_invalid_yaml',
|
||||
'rejected_invalid_piece',
|
||||
'rejected_dangerous_piece',
|
||||
]);
|
||||
|
||||
// ── Snapshot helper ───────────────────────────────────────────────────────────
|
||||
|
||||
/** Take a deterministic snapshot of a directory (path → content). */
|
||||
function snapshotDir(dir: string): Map<string, Buffer> {
|
||||
const snap = new Map<string, Buffer>();
|
||||
if (!existsSync(dir)) return snap;
|
||||
|
||||
function walk(current: string, rel: string): void {
|
||||
for (const entry of readdirSync(current, { withFileTypes: true })) {
|
||||
const fullPath = join(current, entry.name);
|
||||
const relPath = rel ? `${rel}/${entry.name}` : entry.name;
|
||||
if (entry.isDirectory()) {
|
||||
walk(fullPath, relPath);
|
||||
} else {
|
||||
snap.set(relPath, readFileSync(fullPath));
|
||||
}
|
||||
}
|
||||
}
|
||||
walk(dir, '');
|
||||
return snap;
|
||||
}
|
||||
|
||||
/** Returns true iff two snapshots have identical keys and byte-identical values. */
|
||||
function snapshotsEqual(a: Map<string, Buffer>, b: Map<string, Buffer>): boolean {
|
||||
if (a.size !== b.size) return false;
|
||||
for (const [key, valA] of a) {
|
||||
const valB = b.get(key);
|
||||
if (!valB) return false;
|
||||
if (!valA.equals(valB)) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// ── Fixture setup ─────────────────────────────────────────────────────────────
|
||||
|
||||
/** Pre-existing entry name present in every property run. */
|
||||
const EXISTING_ENTRY_NAME = 'existing_a';
|
||||
|
||||
/**
|
||||
* Seed the temp dir with one pre-existing memory entry so that:
|
||||
* - collision checks (add + existing name) fire
|
||||
* - missing-target checks (update/merge_into/remove + unknown target) fire
|
||||
* - CAS checks for correct vs. stale revisions can be exercised
|
||||
*
|
||||
* Returns the known revision of the seeded body (post gray-matter round-trip).
|
||||
*/
|
||||
function seedFixture(dataDir: string): string {
|
||||
upsertMemoryEntry(dataDir, USER_ID, {
|
||||
name: EXISTING_ENTRY_NAME,
|
||||
type: 'user',
|
||||
description: 'pre-existing fuzz fixture',
|
||||
body: 'original body for fuzz',
|
||||
});
|
||||
const stored = readMemoryEntry(dataDir, USER_ID, EXISTING_ENTRY_NAME)!;
|
||||
return bodyRevision(stored.body);
|
||||
}
|
||||
|
||||
function makeDeps(dataDir: string): ApplierDeps {
|
||||
// Build the same shape as applier.test.ts: real Repository (SQLite),
|
||||
// real PieceCatalog with a tiny builtin pieces dir. Without these, every
|
||||
// piece_change with should_edit=true would throw inside writePiece and the
|
||||
// applier's catch would silently swallow it — making the fuzz vacuously
|
||||
// pass Property 1 (no throws) for the piece path. Codex final-review MAJOR-2.
|
||||
const builtinDir = join(dataDir, 'pieces');
|
||||
mkdirSync(builtinDir, { recursive: true });
|
||||
writeFileSync(
|
||||
join(builtinDir, 'chat.yaml'),
|
||||
'name: chat\nmovements:\n - name: m1\n rules: []\n',
|
||||
);
|
||||
const repo = new Repository(join(dataDir, 'db.sqlite'));
|
||||
const catalog = new PieceCatalog(builtinDir, dataDir);
|
||||
return {
|
||||
dataDir,
|
||||
maxBodyBytes: MAX_BODY,
|
||||
repo,
|
||||
catalog,
|
||||
builtinDir,
|
||||
cooldownHours: 24,
|
||||
};
|
||||
}
|
||||
|
||||
function makeInput(
|
||||
dataDir: string,
|
||||
knownRevision: string,
|
||||
overrides: Partial<ReflectionInput> = {},
|
||||
): ReflectionInput {
|
||||
return {
|
||||
originalJobId: 'j-fuzz',
|
||||
userId: USER_ID,
|
||||
pieceName: 'chat',
|
||||
pieceSource: 'builtin',
|
||||
outcome: 'succeeded',
|
||||
taskTitle: 'fuzz task',
|
||||
taskBody: 'fuzz body',
|
||||
activityLogSummary: '',
|
||||
postCompletionComments: [],
|
||||
feedback: { rating: null, comment: null, tags: [] },
|
||||
resultText: 'done',
|
||||
// Expose the pre-existing entry so validator can see it.
|
||||
observedRevisions: { [EXISTING_ENTRY_NAME]: knownRevision },
|
||||
memoryIndex: '',
|
||||
memoryEntries: [
|
||||
{
|
||||
name: EXISTING_ENTRY_NAME,
|
||||
description: 'pre-existing fuzz fixture',
|
||||
type: 'user',
|
||||
body: 'original body for fuzz\n',
|
||||
},
|
||||
],
|
||||
pieceYaml: 'name: chat\nmovements:\n - name: m1\n rules: []\n',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Arbitraries ───────────────────────────────────────────────────────────────
|
||||
|
||||
/** Generate strings that include both valid and adversarial patterns. */
|
||||
const anyString = fc.oneof(
|
||||
fc.string(), // unicode, any length
|
||||
fc.constant(''),
|
||||
fc.constant('../evil'),
|
||||
fc.constant('/etc/passwd'),
|
||||
fc.constant('a'.repeat(200)),
|
||||
fc.hexaString({ minLength: 0, maxLength: 16 }),
|
||||
fc.fullUnicodeString({ minLength: 0, maxLength: 50 }),
|
||||
);
|
||||
|
||||
/** Generate op values — valid and adversarial. */
|
||||
const anyOp = fc.oneof(
|
||||
fc.constantFrom('add', 'update', 'merge_into', 'remove'),
|
||||
fc.string({ minLength: 0, maxLength: 20 }), // unknown ops
|
||||
);
|
||||
|
||||
/** Generate type values — valid and adversarial. */
|
||||
const anyType = fc.oneof(
|
||||
fc.constantFrom('user', 'feedback', 'project', 'reference'),
|
||||
fc.string({ minLength: 0, maxLength: 20 }), // unknown types
|
||||
);
|
||||
|
||||
/** Generate a name — valid, invalid, and edge-case. */
|
||||
const anyName = fc.oneof(
|
||||
// Valid names
|
||||
fc.stringMatching(/^[a-zA-Z0-9_-]{1,64}$/),
|
||||
// The pre-existing entry (triggers collision for 'add', valid target otherwise)
|
||||
fc.constant(EXISTING_ENTRY_NAME),
|
||||
// Unknown entry name (valid for 'add', triggers missing_target for others)
|
||||
fc.constant('unknown_entry_xyz'),
|
||||
// Adversarial names
|
||||
fc.constant(''),
|
||||
fc.constant('!invalid!'),
|
||||
fc.constant('../escape'),
|
||||
fc.constant('a'.repeat(65)),
|
||||
anyString,
|
||||
);
|
||||
|
||||
/** A single MemoryChange — any combination of fields. */
|
||||
const anyMemoryChange = fc.record({
|
||||
op: anyOp,
|
||||
name: anyName,
|
||||
type: anyType,
|
||||
description: anyString,
|
||||
body: fc.oneof(
|
||||
fc.string({ minLength: 0, maxLength: 100 }),
|
||||
// Oversized body (exceeds maxBodyBytes)
|
||||
fc.constant('x'.repeat(MAX_BODY + 1)),
|
||||
),
|
||||
merge_target: fc.option(anyName, { nil: undefined }),
|
||||
}) as fc.Arbitrary<{
|
||||
op: string;
|
||||
name: string;
|
||||
type: string;
|
||||
description: string;
|
||||
body: string;
|
||||
merge_target?: string;
|
||||
}>;
|
||||
|
||||
/** PieceChanges — valid and adversarial. */
|
||||
const anyPieceChanges = fc.record({
|
||||
should_edit: fc.boolean(),
|
||||
target_piece: fc.option(
|
||||
fc.oneof(
|
||||
fc.constant('chat'), // matches pieceName → may pass
|
||||
fc.constant('other_piece'), // mismatch → rejected
|
||||
anyString,
|
||||
),
|
||||
{ nil: undefined },
|
||||
),
|
||||
new_yaml: fc.option(
|
||||
fc.oneof(
|
||||
// Valid minimal piece yaml
|
||||
fc.constant('name: chat\nmovements:\n - name: m1\n rules: []\n'),
|
||||
// Missing movements → rejected_invalid_piece
|
||||
fc.constant('name: chat'),
|
||||
// Dangerous sentinel in rules
|
||||
fc.constant(
|
||||
'name: chat\nmovements:\n - name: m1\n rules:\n - next: COMPLETE\n',
|
||||
),
|
||||
// Garbage YAML
|
||||
fc.constant(': : : invalid yaml :::'),
|
||||
// null / empty
|
||||
fc.constant(''),
|
||||
anyString,
|
||||
),
|
||||
{ nil: null },
|
||||
),
|
||||
diff_summary: fc.option(anyString, { nil: undefined }),
|
||||
}) as fc.Arbitrary<{
|
||||
should_edit: boolean;
|
||||
target_piece?: string;
|
||||
new_yaml?: string | null;
|
||||
diff_summary?: string;
|
||||
}>;
|
||||
|
||||
/** A full ReflectionResult — any combination. */
|
||||
const anyReflectionResult = fc.record({
|
||||
memory_changes: fc.array(anyMemoryChange, { minLength: 0, maxLength: 10 }),
|
||||
piece_changes: anyPieceChanges,
|
||||
reasoning: anyString,
|
||||
abstain_reason: fc.option(anyString, { nil: undefined }),
|
||||
}) as fc.Arbitrary<ReflectionResult>;
|
||||
|
||||
// ── Property tests ─────────────────────────────────────────────────────────────
|
||||
|
||||
describe('applyReflection — property / fuzz tests (fast-check)', () => {
|
||||
it(
|
||||
'holds all 4 properties for 200 seeded runs',
|
||||
async () => {
|
||||
await fc.assert(
|
||||
fc.asyncProperty(anyReflectionResult, async (result) => {
|
||||
// ── Setup: fresh temp dir + fixture for each run ──────────────────────
|
||||
const dataDir = mkdtempSync(join(tmpdir(), 'applier-fuzz-'));
|
||||
try {
|
||||
const knownRevision = seedFixture(dataDir);
|
||||
const deps = makeDeps(dataDir);
|
||||
const input = makeInput(dataDir, knownRevision);
|
||||
|
||||
// Snapshot the memory dir BEFORE the call (for property 3).
|
||||
const memDir = join(dataDir, USER_ID, 'memory');
|
||||
const beforeSnap = snapshotDir(memDir);
|
||||
|
||||
// ── Property 1: never throws ─────────────────────────────────────────
|
||||
let applyResult: Awaited<ReturnType<typeof applyReflection>>;
|
||||
try {
|
||||
applyResult = await applyReflection(deps, input, result);
|
||||
} catch (e) {
|
||||
// Property 1 violated: applyReflection must not throw.
|
||||
throw new Error(
|
||||
`applyReflection threw unexpectedly: ${String(e)}\n` +
|
||||
`result=${JSON.stringify(result)}`,
|
||||
);
|
||||
}
|
||||
|
||||
// ── Property 2: each decision has accepted:true OR a known code ──────
|
||||
for (const decision of applyResult.memoryDecisions) {
|
||||
if (!decision.accepted) {
|
||||
if (
|
||||
decision.code === undefined ||
|
||||
!KNOWN_REJECTION_CODES.has(decision.code as ReflectionRejectionCode)
|
||||
) {
|
||||
throw new Error(
|
||||
`memoryDecision has accepted:false but code="${decision.code}" is ` +
|
||||
`not in the known ReflectionRejectionCode union.\n` +
|
||||
`decision=${JSON.stringify(decision)}\n` +
|
||||
`result=${JSON.stringify(result)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Property 3: if outcome === 'rejected', disk unchanged ─────────────
|
||||
if (applyResult.outcome === 'rejected') {
|
||||
const afterSnap = snapshotDir(memDir);
|
||||
if (!snapshotsEqual(beforeSnap, afterSnap)) {
|
||||
throw new Error(
|
||||
`outcome=rejected but memory dir changed on disk.\n` +
|
||||
`before keys=[${[...beforeSnap.keys()].join(', ')}]\n` +
|
||||
`after keys=[${[...afterSnap.keys()].join(', ')}]\n` +
|
||||
`result=${JSON.stringify(result)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Property 4: cap honored — at most 3 decisions ────────────────────
|
||||
if (applyResult.memoryDecisions.length > 3) {
|
||||
throw new Error(
|
||||
`memoryDecisions.length=${applyResult.memoryDecisions.length} > 3 ` +
|
||||
`(input had ${result.memory_changes.length} changes).\n` +
|
||||
`result=${JSON.stringify(result)}`,
|
||||
);
|
||||
}
|
||||
|
||||
} finally {
|
||||
rmSync(dataDir, { recursive: true, force: true });
|
||||
}
|
||||
}),
|
||||
{
|
||||
numRuns: 200,
|
||||
seed: 1,
|
||||
verbose: false,
|
||||
},
|
||||
);
|
||||
},
|
||||
60_000, // 60s timeout (well within the <30s target for 200 runs)
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user