// src/engine/reflection/applier.fuzz.test.ts // // Property / fuzz tests for applyReflection using fast-check. // // Properties asserted for 200 seeded runs: // 1. applyReflection never throws on any generated ReflectionResult // 2. Every memoryDecision has accepted:true OR a known ReflectionRejectionCode // 3. When outcome === 'rejected', the memory dir on disk is byte-for-byte // identical to before the call. // 4. memoryDecisions.length <= 3 regardless of input size. import { describe, it } from 'vitest'; import * as fc from 'fast-check'; import { mkdtempSync, mkdirSync, rmSync, readdirSync, readFileSync, existsSync, writeFileSync } from 'fs'; import { join } from 'path'; import { tmpdir } from 'os'; import { applyReflection, type ApplierDeps } from './applier.js'; import { upsertMemoryEntry, readMemoryEntry } from '../../user-folder/memory.js'; import { bodyRevision } from './revisions.js'; import { Repository } from '../../db/repository.js'; import { PieceCatalog } from '../piece-catalog.js'; import type { ReflectionInput, ReflectionResult, ReflectionRejectionCode } from './types.js'; // ── Constants ───────────────────────────────────────────────────────────────── const USER_ID = 'u-fuzz'; const MAX_BODY = 8192; /** All known rejection codes — exactly the 10-member union from types.ts. */ const KNOWN_REJECTION_CODES = new Set([ 'rejected_unknown_type', 'rejected_bad_name', 'rejected_body_too_large', 'rejected_missing_target', 'rejected_stale_target', 'rejected_name_collision', 'rejected_target_piece_mismatch', 'rejected_invalid_yaml', 'rejected_invalid_piece', 'rejected_dangerous_piece', ]); // ── Snapshot helper ─────────────────────────────────────────────────────────── /** Take a deterministic snapshot of a directory (path → content). */ function snapshotDir(dir: string): Map { const snap = new Map(); if (!existsSync(dir)) return snap; function walk(current: string, rel: string): void { for (const entry of readdirSync(current, { withFileTypes: true })) { const fullPath = join(current, entry.name); const relPath = rel ? `${rel}/${entry.name}` : entry.name; if (entry.isDirectory()) { walk(fullPath, relPath); } else { snap.set(relPath, readFileSync(fullPath)); } } } walk(dir, ''); return snap; } /** Returns true iff two snapshots have identical keys and byte-identical values. */ function snapshotsEqual(a: Map, b: Map): boolean { if (a.size !== b.size) return false; for (const [key, valA] of a) { const valB = b.get(key); if (!valB) return false; if (!valA.equals(valB)) return false; } return true; } // ── Fixture setup ───────────────────────────────────────────────────────────── /** Pre-existing entry name present in every property run. */ const EXISTING_ENTRY_NAME = 'existing_a'; /** * Seed the temp dir with one pre-existing memory entry so that: * - collision checks (add + existing name) fire * - missing-target checks (update/merge_into/remove + unknown target) fire * - CAS checks for correct vs. stale revisions can be exercised * * Returns the known revision of the seeded body (post gray-matter round-trip). */ function seedFixture(dataDir: string): string { upsertMemoryEntry(dataDir, USER_ID, { name: EXISTING_ENTRY_NAME, type: 'user', description: 'pre-existing fuzz fixture', body: 'original body for fuzz', }); const stored = readMemoryEntry(dataDir, USER_ID, EXISTING_ENTRY_NAME)!; return bodyRevision(stored.body); } function makeDeps(dataDir: string): ApplierDeps { // Build the same shape as applier.test.ts: real Repository (SQLite), // real PieceCatalog with a tiny builtin pieces dir. Without these, every // piece_change with should_edit=true would throw inside writePiece and the // applier's catch would silently swallow it — making the fuzz vacuously // pass Property 1 (no throws) for the piece path. Codex final-review MAJOR-2. const builtinDir = join(dataDir, 'pieces'); mkdirSync(builtinDir, { recursive: true }); writeFileSync( join(builtinDir, 'chat.yaml'), 'name: chat\nmovements:\n - name: m1\n rules: []\n', ); const repo = new Repository(join(dataDir, 'db.sqlite')); const catalog = new PieceCatalog(builtinDir, dataDir); return { dataDir, maxBodyBytes: MAX_BODY, repo, catalog, builtinDir, cooldownHours: 24, }; } function makeInput( dataDir: string, knownRevision: string, overrides: Partial = {}, ): ReflectionInput { return { originalJobId: 'j-fuzz', userId: USER_ID, pieceName: 'chat', pieceSource: 'builtin', outcome: 'succeeded', taskTitle: 'fuzz task', taskBody: 'fuzz body', activityLogSummary: '', postCompletionComments: [], feedback: { rating: null, comment: null, tags: [] }, resultText: 'done', // Expose the pre-existing entry so validator can see it. observedRevisions: { [EXISTING_ENTRY_NAME]: knownRevision }, memoryIndex: '', memoryEntries: [ { name: EXISTING_ENTRY_NAME, description: 'pre-existing fuzz fixture', type: 'user', body: 'original body for fuzz\n', }, ], pieceYaml: 'name: chat\nmovements:\n - name: m1\n rules: []\n', ...overrides, }; } // ── Arbitraries ─────────────────────────────────────────────────────────────── /** Generate strings that include both valid and adversarial patterns. */ const anyString = fc.oneof( fc.string(), // unicode, any length fc.constant(''), fc.constant('../evil'), fc.constant('/etc/passwd'), fc.constant('a'.repeat(200)), fc.hexaString({ minLength: 0, maxLength: 16 }), fc.fullUnicodeString({ minLength: 0, maxLength: 50 }), ); /** Generate op values — valid and adversarial. */ const anyOp = fc.oneof( fc.constantFrom('add', 'update', 'merge_into', 'remove'), fc.string({ minLength: 0, maxLength: 20 }), // unknown ops ); /** Generate type values — valid and adversarial. */ const anyType = fc.oneof( fc.constantFrom('user', 'feedback', 'project', 'reference'), fc.string({ minLength: 0, maxLength: 20 }), // unknown types ); /** Generate a name — valid, invalid, and edge-case. */ const anyName = fc.oneof( // Valid names fc.stringMatching(/^[a-zA-Z0-9_-]{1,64}$/), // The pre-existing entry (triggers collision for 'add', valid target otherwise) fc.constant(EXISTING_ENTRY_NAME), // Unknown entry name (valid for 'add', triggers missing_target for others) fc.constant('unknown_entry_xyz'), // Adversarial names fc.constant(''), fc.constant('!invalid!'), fc.constant('../escape'), fc.constant('a'.repeat(65)), anyString, ); /** A single MemoryChange — any combination of fields. */ const anyMemoryChange = fc.record({ op: anyOp, name: anyName, type: anyType, description: anyString, body: fc.oneof( fc.string({ minLength: 0, maxLength: 100 }), // Oversized body (exceeds maxBodyBytes) fc.constant('x'.repeat(MAX_BODY + 1)), ), merge_target: fc.option(anyName, { nil: undefined }), }) as fc.Arbitrary<{ op: string; name: string; type: string; description: string; body: string; merge_target?: string; }>; /** PieceChanges — valid and adversarial. */ const anyPieceChanges = fc.record({ should_edit: fc.boolean(), target_piece: fc.option( fc.oneof( fc.constant('chat'), // matches pieceName → may pass fc.constant('other_piece'), // mismatch → rejected anyString, ), { nil: undefined }, ), new_yaml: fc.option( fc.oneof( // Valid minimal piece yaml fc.constant('name: chat\nmovements:\n - name: m1\n rules: []\n'), // Missing movements → rejected_invalid_piece fc.constant('name: chat'), // Dangerous sentinel in rules fc.constant( 'name: chat\nmovements:\n - name: m1\n rules:\n - next: COMPLETE\n', ), // Garbage YAML fc.constant(': : : invalid yaml :::'), // null / empty fc.constant(''), anyString, ), { nil: null }, ), diff_summary: fc.option(anyString, { nil: undefined }), }) as fc.Arbitrary<{ should_edit: boolean; target_piece?: string; new_yaml?: string | null; diff_summary?: string; }>; /** A full ReflectionResult — any combination. */ const anyReflectionResult = fc.record({ memory_changes: fc.array(anyMemoryChange, { minLength: 0, maxLength: 10 }), piece_changes: anyPieceChanges, reasoning: anyString, abstain_reason: fc.option(anyString, { nil: undefined }), }) as fc.Arbitrary; // ── Property tests ───────────────────────────────────────────────────────────── describe('applyReflection — property / fuzz tests (fast-check)', () => { it( 'holds all 4 properties for 200 seeded runs', async () => { await fc.assert( fc.asyncProperty(anyReflectionResult, async (result) => { // ── Setup: fresh temp dir + fixture for each run ────────────────────── const dataDir = mkdtempSync(join(tmpdir(), 'applier-fuzz-')); try { const knownRevision = seedFixture(dataDir); const deps = makeDeps(dataDir); const input = makeInput(dataDir, knownRevision); // Snapshot the memory dir BEFORE the call (for property 3). const memDir = join(dataDir, USER_ID, 'memory'); const beforeSnap = snapshotDir(memDir); // ── Property 1: never throws ───────────────────────────────────────── let applyResult: Awaited>; try { applyResult = await applyReflection(deps, input, result); } catch (e) { // Property 1 violated: applyReflection must not throw. throw new Error( `applyReflection threw unexpectedly: ${String(e)}\n` + `result=${JSON.stringify(result)}`, ); } // ── Property 2: each decision has accepted:true OR a known code ────── for (const decision of applyResult.memoryDecisions) { if (!decision.accepted) { if ( decision.code === undefined || !KNOWN_REJECTION_CODES.has(decision.code as ReflectionRejectionCode) ) { throw new Error( `memoryDecision has accepted:false but code="${decision.code}" is ` + `not in the known ReflectionRejectionCode union.\n` + `decision=${JSON.stringify(decision)}\n` + `result=${JSON.stringify(result)}`, ); } } } // ── Property 3: if outcome === 'rejected', disk unchanged ───────────── if (applyResult.outcome === 'rejected') { const afterSnap = snapshotDir(memDir); if (!snapshotsEqual(beforeSnap, afterSnap)) { throw new Error( `outcome=rejected but memory dir changed on disk.\n` + `before keys=[${[...beforeSnap.keys()].join(', ')}]\n` + `after keys=[${[...afterSnap.keys()].join(', ')}]\n` + `result=${JSON.stringify(result)}`, ); } } // ── Property 4: cap honored — at most 3 decisions ──────────────────── if (applyResult.memoryDecisions.length > 3) { throw new Error( `memoryDecisions.length=${applyResult.memoryDecisions.length} > 3 ` + `(input had ${result.memory_changes.length} changes).\n` + `result=${JSON.stringify(result)}`, ); } } finally { rmSync(dataDir, { recursive: true, force: true }); } }), { numRuns: 200, seed: 1, verbose: false, }, ); }, 60_000, // 60s timeout (well within the <30s target for 200 runs) ); });