maestro/src/engine/reflection/applier.fuzz.test.ts

// src/engine/reflection/applier.fuzz.test.ts
//
// Property / fuzz tests for applyReflection using fast-check.
//
// Properties asserted for 200 seeded runs:
//   1. applyReflection never throws on any generated ReflectionResult
//   2. Every memoryDecision has accepted:true OR a known ReflectionRejectionCode
//   3. When outcome === 'rejected', the memory dir on disk is byte-for-byte
//      identical to before the call.
//   4. memoryDecisions.length <= 3 regardless of input size.

import { describe, it } from 'vitest';
import * as fc from 'fast-check';
import { mkdtempSync, mkdirSync, rmSync, readdirSync, readFileSync, existsSync, writeFileSync } from 'fs';
import { join } from 'path';
import { tmpdir } from 'os';
import { applyReflection, type ApplierDeps } from './applier.js';
import { upsertMemoryEntry, readMemoryEntry } from '../../user-folder/memory.js';
import { bodyRevision } from './revisions.js';
import { Repository } from '../../db/repository.js';
import { PieceCatalog } from '../piece-catalog.js';
import type { ReflectionInput, ReflectionResult, ReflectionRejectionCode } from './types.js';

// ── Constants ─────────────────────────────────────────────────────────────────

const USER_ID = 'u-fuzz';
const MAX_BODY = 8192;

/** All known rejection codes — exactly the 10-member union from types.ts. */
const KNOWN_REJECTION_CODES = new Set<ReflectionRejectionCode>([
  'rejected_unknown_type',
  'rejected_bad_name',
  'rejected_body_too_large',
  'rejected_missing_target',
  'rejected_stale_target',
  'rejected_name_collision',
  'rejected_target_piece_mismatch',
  'rejected_invalid_yaml',
  'rejected_invalid_piece',
  'rejected_dangerous_piece',
]);

// ── Snapshot helper ───────────────────────────────────────────────────────────

/** Take a deterministic snapshot of a directory (path → content). */
function snapshotDir(dir: string): Map<string, Buffer> {
  const snap = new Map<string, Buffer>();
  if (!existsSync(dir)) return snap;

  function walk(current: string, rel: string): void {
    for (const entry of readdirSync(current, { withFileTypes: true })) {
      const fullPath = join(current, entry.name);
      const relPath = rel ? `${rel}/${entry.name}` : entry.name;
      if (entry.isDirectory()) {
        walk(fullPath, relPath);
      } else {
        snap.set(relPath, readFileSync(fullPath));
      }
    }
  }
  walk(dir, '');
  return snap;
}

/** Returns true iff two snapshots have identical keys and byte-identical values. */
function snapshotsEqual(a: Map<string, Buffer>, b: Map<string, Buffer>): boolean {
  if (a.size !== b.size) return false;
  for (const [key, valA] of a) {
    const valB = b.get(key);
    if (!valB) return false;
    if (!valA.equals(valB)) return false;
  }
  return true;
}

// ── Fixture setup ─────────────────────────────────────────────────────────────

/** Pre-existing entry name present in every property run. */
const EXISTING_ENTRY_NAME = 'existing_a';

/**
 * Seed the temp dir with one pre-existing memory entry so that:
 *   - collision checks (add + existing name) fire
 *   - missing-target checks (update/merge_into/remove + unknown target) fire
 *   - CAS checks for correct vs. stale revisions can be exercised
 *
 * Returns the known revision of the seeded body (post gray-matter round-trip).
 */
function seedFixture(dataDir: string): string {
  upsertMemoryEntry(dataDir, USER_ID, {
    name: EXISTING_ENTRY_NAME,
    type: 'user',
    description: 'pre-existing fuzz fixture',
    body: 'original body for fuzz',
  });
  const stored = readMemoryEntry(dataDir, USER_ID, EXISTING_ENTRY_NAME)!;
  return bodyRevision(stored.body);
}

function makeDeps(dataDir: string): ApplierDeps {
  // Build the same shape as applier.test.ts: real Repository (SQLite),
  // real PieceCatalog with a tiny builtin pieces dir. Without these, every
  // piece_change with should_edit=true would throw inside writePiece and the
  // applier's catch would silently swallow it — making the fuzz vacuously
  // pass Property 1 (no throws) for the piece path. Codex final-review MAJOR-2.
  const builtinDir = join(dataDir, 'pieces');
  mkdirSync(builtinDir, { recursive: true });
  writeFileSync(
    join(builtinDir, 'chat.yaml'),
    'name: chat\nmovements:\n  - name: m1\n    rules: []\n',
  );
  const repo = new Repository(join(dataDir, 'db.sqlite'));
  const catalog = new PieceCatalog(builtinDir, dataDir);
  return {
    dataDir,
    maxBodyBytes: MAX_BODY,
    repo,
    catalog,
    builtinDir,
    cooldownHours: 24,
  };
}

function makeInput(
  dataDir: string,
  knownRevision: string,
  overrides: Partial<ReflectionInput> = {},
): ReflectionInput {
  return {
    originalJobId: 'j-fuzz',
    userId: USER_ID,
    pieceName: 'chat',
    pieceSource: 'builtin',
    outcome: 'succeeded',
    taskTitle: 'fuzz task',
    taskBody: 'fuzz body',
    activityLogSummary: '',
    postCompletionComments: [],
    feedback: { rating: null, comment: null, tags: [] },
    resultText: 'done',
    // Expose the pre-existing entry so validator can see it.
    observedRevisions: { [EXISTING_ENTRY_NAME]: knownRevision },
    memoryIndex: '',
    memoryEntries: [
      {
        name: EXISTING_ENTRY_NAME,
        description: 'pre-existing fuzz fixture',
        type: 'user',
        body: 'original body for fuzz\n',
      },
    ],
    pieceYaml: 'name: chat\nmovements:\n  - name: m1\n    rules: []\n',
    ...overrides,
  };
}

// ── Arbitraries ───────────────────────────────────────────────────────────────

/** Generate strings that include both valid and adversarial patterns. */
const anyString = fc.oneof(
  fc.string(),           // unicode, any length
  fc.constant(''),
  fc.constant('../evil'),
  fc.constant('/etc/passwd'),
  fc.constant('a'.repeat(200)),
  fc.hexaString({ minLength: 0, maxLength: 16 }),
  fc.fullUnicodeString({ minLength: 0, maxLength: 50 }),
);

/** Generate op values — valid and adversarial. */
const anyOp = fc.oneof(
  fc.constantFrom('add', 'update', 'merge_into', 'remove'),
  fc.string({ minLength: 0, maxLength: 20 }),   // unknown ops
);

/** Generate type values — valid and adversarial. */
const anyType = fc.oneof(
  fc.constantFrom('user', 'feedback', 'project', 'reference'),
  fc.string({ minLength: 0, maxLength: 20 }),   // unknown types
);

/** Generate a name — valid, invalid, and edge-case. */
const anyName = fc.oneof(
  // Valid names
  fc.stringMatching(/^[a-zA-Z0-9_-]{1,64}$/),
  // The pre-existing entry (triggers collision for 'add', valid target otherwise)
  fc.constant(EXISTING_ENTRY_NAME),
  // Unknown entry name (valid for 'add', triggers missing_target for others)
  fc.constant('unknown_entry_xyz'),
  // Adversarial names
  fc.constant(''),
  fc.constant('!invalid!'),
  fc.constant('../escape'),
  fc.constant('a'.repeat(65)),
  anyString,
);

/** A single MemoryChange — any combination of fields. */
const anyMemoryChange = fc.record({
  op: anyOp,
  name: anyName,
  type: anyType,
  description: anyString,
  body: fc.oneof(
    fc.string({ minLength: 0, maxLength: 100 }),
    // Oversized body (exceeds maxBodyBytes)
    fc.constant('x'.repeat(MAX_BODY + 1)),
  ),
  merge_target: fc.option(anyName, { nil: undefined }),
}) as fc.Arbitrary<{
  op: string;
  name: string;
  type: string;
  description: string;
  body: string;
  merge_target?: string;
}>;

/** PieceChanges — valid and adversarial. */
const anyPieceChanges = fc.record({
  should_edit: fc.boolean(),
  target_piece: fc.option(
    fc.oneof(
      fc.constant('chat'),              // matches pieceName → may pass
      fc.constant('other_piece'),        // mismatch → rejected
      anyString,
    ),
    { nil: undefined },
  ),
  new_yaml: fc.option(
    fc.oneof(
      // Valid minimal piece yaml
      fc.constant('name: chat\nmovements:\n  - name: m1\n    rules: []\n'),
      // Missing movements → rejected_invalid_piece
      fc.constant('name: chat'),
      // Dangerous sentinel in rules
      fc.constant(
        'name: chat\nmovements:\n  - name: m1\n    rules:\n      - next: COMPLETE\n',
      ),
      // Garbage YAML
      fc.constant(': : : invalid yaml :::'),
      // null / empty
      fc.constant(''),
      anyString,
    ),
    { nil: null },
  ),
  diff_summary: fc.option(anyString, { nil: undefined }),
}) as fc.Arbitrary<{
  should_edit: boolean;
  target_piece?: string;
  new_yaml?: string | null;
  diff_summary?: string;
}>;

/** A full ReflectionResult — any combination. */
const anyReflectionResult = fc.record({
  memory_changes: fc.array(anyMemoryChange, { minLength: 0, maxLength: 10 }),
  piece_changes: anyPieceChanges,
  reasoning: anyString,
  abstain_reason: fc.option(anyString, { nil: undefined }),
}) as fc.Arbitrary<ReflectionResult>;

// ── Property tests ─────────────────────────────────────────────────────────────

describe('applyReflection — property / fuzz tests (fast-check)', () => {
  it(
    'holds all 4 properties for 200 seeded runs',
    async () => {
      await fc.assert(
        fc.asyncProperty(anyReflectionResult, async (result) => {
          // ── Setup: fresh temp dir + fixture for each run ──────────────────────
          const dataDir = mkdtempSync(join(tmpdir(), 'applier-fuzz-'));
          try {
            const knownRevision = seedFixture(dataDir);
            const deps = makeDeps(dataDir);
            const input = makeInput(dataDir, knownRevision);

            // Snapshot the memory dir BEFORE the call (for property 3).
            const memDir = join(dataDir, USER_ID, 'memory');
            const beforeSnap = snapshotDir(memDir);

            // ── Property 1: never throws ─────────────────────────────────────────
            let applyResult: Awaited<ReturnType<typeof applyReflection>>;
            try {
              applyResult = await applyReflection(deps, input, result);
            } catch (e) {
              // Property 1 violated: applyReflection must not throw.
              throw new Error(
                `applyReflection threw unexpectedly: ${String(e)}\n` +
                `result=${JSON.stringify(result)}`,
              );
            }

            // ── Property 2: each decision has accepted:true OR a known code ──────
            for (const decision of applyResult.memoryDecisions) {
              if (!decision.accepted) {
                if (
                  decision.code === undefined ||
                  !KNOWN_REJECTION_CODES.has(decision.code as ReflectionRejectionCode)
                ) {
                  throw new Error(
                    `memoryDecision has accepted:false but code="${decision.code}" is ` +
                    `not in the known ReflectionRejectionCode union.\n` +
                    `decision=${JSON.stringify(decision)}\n` +
                    `result=${JSON.stringify(result)}`,
                  );
                }
              }
            }

            // ── Property 3: if outcome === 'rejected', disk unchanged ─────────────
            if (applyResult.outcome === 'rejected') {
              const afterSnap = snapshotDir(memDir);
              if (!snapshotsEqual(beforeSnap, afterSnap)) {
                throw new Error(
                  `outcome=rejected but memory dir changed on disk.\n` +
                  `before keys=[${[...beforeSnap.keys()].join(', ')}]\n` +
                  `after  keys=[${[...afterSnap.keys()].join(', ')}]\n` +
                  `result=${JSON.stringify(result)}`,
                );
              }
            }

            // ── Property 4: cap honored — at most 3 decisions ────────────────────
            if (applyResult.memoryDecisions.length > 3) {
              throw new Error(
                `memoryDecisions.length=${applyResult.memoryDecisions.length} > 3 ` +
                `(input had ${result.memory_changes.length} changes).\n` +
                `result=${JSON.stringify(result)}`,
              );
            }

          } finally {
            rmSync(dataDir, { recursive: true, force: true });
          }
        }),
        {
          numRuns: 200,
          seed: 1,
          verbose: false,
        },
      );
    },
    60_000, // 60s timeout (well within the <30s target for 200 runs)
  );
});