feat: initial public release (MAESTRO)
This commit is contained in:
oss-sync
@@ -0,0 +1,273 @@
|
||||
// src/engine/reflection/retention.test.ts
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import {
|
||||
mkdtempSync,
|
||||
rmSync,
|
||||
mkdirSync,
|
||||
writeFileSync,
|
||||
existsSync,
|
||||
} from 'fs';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
import {
|
||||
pruneOldSnapshots,
|
||||
enforceDiskCap,
|
||||
runReflectionRetentionSweep,
|
||||
type RetentionDeps,
|
||||
} from './retention.js';
|
||||
|
||||
// ── Fixtures ──────────────────────────────────────────────────────────────────
|
||||
|
||||
const USER_ID = 'u-retention-test';
|
||||
|
||||
/** Format a Date as the snapshot directory name prefix (YYYYMMDDTHHmmssZ). */
|
||||
function fmtTs(d: Date): string {
|
||||
const pad2 = (n: number) => String(n).padStart(2, '0');
|
||||
return (
|
||||
`${d.getUTCFullYear()}` +
|
||||
`${pad2(d.getUTCMonth() + 1)}` +
|
||||
`${pad2(d.getUTCDate())}` +
|
||||
`T${pad2(d.getUTCHours())}` +
|
||||
`${pad2(d.getUTCMinutes())}` +
|
||||
`${pad2(d.getUTCSeconds())}` +
|
||||
`Z`
|
||||
);
|
||||
}
|
||||
|
||||
/** Create a fake snapshot dir for the given age in days (relative to now). */
|
||||
function makeSnapshot(
|
||||
histDir: string,
|
||||
ageDays: number,
|
||||
jobId: string,
|
||||
fileSizeBytes = 100,
|
||||
): string {
|
||||
const d = new Date(Date.now() - ageDays * 86_400_000);
|
||||
const name = `${fmtTs(d)}-${jobId}`;
|
||||
const dir = join(histDir, name);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
// Write a dummy meta.json + a content file of the requested size
|
||||
writeFileSync(
|
||||
join(dir, 'meta.json'),
|
||||
JSON.stringify({ snapshotId: name, ts: d.toISOString(), pieceName: 'chat' }),
|
||||
'utf-8',
|
||||
);
|
||||
writeFileSync(join(dir, 'data.bin'), Buffer.alloc(fileSizeBytes), 'binary');
|
||||
return name;
|
||||
}
|
||||
|
||||
function makeDeps(dataDir: string): RetentionDeps {
|
||||
return { dataDir };
|
||||
}
|
||||
|
||||
function histDir(dataDir: string, userId: string): string {
|
||||
return join(dataDir, userId, '.reflection-history');
|
||||
}
|
||||
|
||||
// ── Test suite ────────────────────────────────────────────────────────────────
|
||||
|
||||
describe('pruneOldSnapshots', () => {
|
||||
let tmpDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = mkdtempSync(join(tmpdir(), 'retention-age-'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('does NOT prune a snapshot newer than retentionDays', async () => {
|
||||
const hDir = histDir(tmpDir, USER_ID);
|
||||
mkdirSync(hDir, { recursive: true });
|
||||
const snapId = makeSnapshot(hDir, 5, 'j-new'); // 5 days old
|
||||
|
||||
const result = await pruneOldSnapshots(makeDeps(tmpDir), USER_ID, 90);
|
||||
|
||||
expect(result.pruned).toHaveLength(0);
|
||||
expect(existsSync(join(hDir, snapId))).toBe(true);
|
||||
});
|
||||
|
||||
it('prunes a snapshot older than retentionDays', async () => {
|
||||
const hDir = histDir(tmpDir, USER_ID);
|
||||
mkdirSync(hDir, { recursive: true });
|
||||
const snapId = makeSnapshot(hDir, 100, 'j-old'); // 100 days old, retention=90
|
||||
|
||||
const result = await pruneOldSnapshots(makeDeps(tmpDir), USER_ID, 90);
|
||||
|
||||
expect(result.pruned).toContain(snapId);
|
||||
expect(existsSync(join(hDir, snapId))).toBe(false);
|
||||
});
|
||||
|
||||
it('prunes old but keeps new when both present', async () => {
|
||||
const hDir = histDir(tmpDir, USER_ID);
|
||||
mkdirSync(hDir, { recursive: true });
|
||||
const oldId = makeSnapshot(hDir, 200, 'j-old'); // old
|
||||
const newId = makeSnapshot(hDir, 10, 'j-new'); // new
|
||||
|
||||
const result = await pruneOldSnapshots(makeDeps(tmpDir), USER_ID, 90);
|
||||
|
||||
expect(result.pruned).toContain(oldId);
|
||||
expect(result.pruned).not.toContain(newId);
|
||||
expect(existsSync(join(hDir, oldId))).toBe(false);
|
||||
expect(existsSync(join(hDir, newId))).toBe(true);
|
||||
});
|
||||
|
||||
it('returns empty pruned when no history dir exists', async () => {
|
||||
const result = await pruneOldSnapshots(makeDeps(tmpDir), 'u-ghost', 90);
|
||||
expect(result.pruned).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('does not prune index.jsonl (not a parseable snapshot dir name)', async () => {
|
||||
const hDir = histDir(tmpDir, USER_ID);
|
||||
mkdirSync(hDir, { recursive: true });
|
||||
// Write index.jsonl as a file (not a dir), should be ignored
|
||||
writeFileSync(join(hDir, 'index.jsonl'), '', 'utf-8');
|
||||
const snapId = makeSnapshot(hDir, 5, 'j-ok');
|
||||
|
||||
const result = await pruneOldSnapshots(makeDeps(tmpDir), USER_ID, 90);
|
||||
|
||||
expect(result.pruned).toHaveLength(0);
|
||||
expect(existsSync(join(hDir, 'index.jsonl'))).toBe(true);
|
||||
expect(existsSync(join(hDir, snapId))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('enforceDiskCap', () => {
|
||||
let tmpDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = mkdtempSync(join(tmpdir(), 'retention-cap-'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('does not prune when total bytes is under cap', async () => {
|
||||
const hDir = histDir(tmpDir, USER_ID);
|
||||
mkdirSync(hDir, { recursive: true });
|
||||
// Each snapshot ≈ 200 bytes (100 data + small meta.json); cap = 10 MiB
|
||||
const snapId = makeSnapshot(hDir, 5, 'j-small', 100);
|
||||
|
||||
const cap = 10 * 1024 * 1024; // 10 MiB
|
||||
const result = await enforceDiskCap(makeDeps(tmpDir), USER_ID, cap);
|
||||
|
||||
expect(result.pruned).toHaveLength(0);
|
||||
expect(existsSync(join(hDir, snapId))).toBe(true);
|
||||
});
|
||||
|
||||
it('prunes oldest snapshots first when over cap', async () => {
|
||||
const hDir = histDir(tmpDir, USER_ID);
|
||||
mkdirSync(hDir, { recursive: true });
|
||||
|
||||
// Create 3 snapshots: oldest, middle, newest — each ~1024 bytes of data
|
||||
const oldestId = makeSnapshot(hDir, 30, 'j-oldest', 1024);
|
||||
const middleId = makeSnapshot(hDir, 20, 'j-middle', 1024);
|
||||
const newestId = makeSnapshot(hDir, 10, 'j-newest', 1024);
|
||||
|
||||
// The total will be around 3 * (1024 + meta) bytes.
|
||||
// Set cap so that only 2 snapshots fit (i.e., total > 2 * entry_size).
|
||||
// Each entry is ~1024 + ~80 bytes meta ≈ 1104 bytes.
|
||||
// Cap at 2200 bytes → oldest must be pruned.
|
||||
const cap = 2200;
|
||||
|
||||
const result = await enforceDiskCap(makeDeps(tmpDir), USER_ID, cap);
|
||||
|
||||
// oldest should be pruned, newest should survive
|
||||
expect(result.pruned).toContain(oldestId);
|
||||
expect(existsSync(join(hDir, oldestId))).toBe(false);
|
||||
expect(existsSync(join(hDir, newestId))).toBe(true);
|
||||
// middle may or may not be pruned depending on actual sizes, but newest must survive
|
||||
void middleId; // referenced to suppress unused variable warning
|
||||
});
|
||||
|
||||
it('newest snapshot survives when only one exists and under cap', async () => {
|
||||
const hDir = histDir(tmpDir, USER_ID);
|
||||
mkdirSync(hDir, { recursive: true });
|
||||
const snapId = makeSnapshot(hDir, 5, 'j-only', 100);
|
||||
|
||||
const cap = 10 * 1024 * 1024; // 10 MiB — easily above 100 bytes
|
||||
const result = await enforceDiskCap(makeDeps(tmpDir), USER_ID, cap);
|
||||
|
||||
expect(result.pruned).toHaveLength(0);
|
||||
expect(existsSync(join(hDir, snapId))).toBe(true);
|
||||
});
|
||||
|
||||
it('returns empty pruned when no history dir exists', async () => {
|
||||
const result = await enforceDiskCap(makeDeps(tmpDir), 'u-ghost', 1024);
|
||||
expect(result.pruned).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('runReflectionRetentionSweep', () => {
|
||||
let tmpDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = mkdtempSync(join(tmpdir(), 'retention-sweep-'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('sweeps all users and prunes old snapshots', async () => {
|
||||
const uid1 = 'u-sweep-1';
|
||||
const uid2 = 'u-sweep-2';
|
||||
|
||||
const hDir1 = histDir(tmpDir, uid1);
|
||||
const hDir2 = histDir(tmpDir, uid2);
|
||||
mkdirSync(hDir1, { recursive: true });
|
||||
mkdirSync(hDir2, { recursive: true });
|
||||
|
||||
const old1 = makeSnapshot(hDir1, 100, 'j-old1'); // old
|
||||
const new1 = makeSnapshot(hDir1, 5, 'j-new1'); // new
|
||||
const old2 = makeSnapshot(hDir2, 200, 'j-old2'); // old
|
||||
|
||||
await runReflectionRetentionSweep(makeDeps(tmpDir), {
|
||||
snapshotRetentionDays: 90,
|
||||
snapshotMaxBytesPerUser: 100 * 1024 * 1024, // 100 MiB — won't trigger
|
||||
});
|
||||
|
||||
expect(existsSync(join(hDir1, old1))).toBe(false);
|
||||
expect(existsSync(join(hDir1, new1))).toBe(true);
|
||||
expect(existsSync(join(hDir2, old2))).toBe(false);
|
||||
});
|
||||
|
||||
it('skips users with no .reflection-history dir', async () => {
|
||||
// Create a user dir without a history dir
|
||||
const uid = 'u-no-history';
|
||||
mkdirSync(join(tmpDir, uid), { recursive: true });
|
||||
|
||||
// Should not throw
|
||||
await expect(
|
||||
runReflectionRetentionSweep(makeDeps(tmpDir), {
|
||||
snapshotRetentionDays: 90,
|
||||
snapshotMaxBytesPerUser: 100 * 1024 * 1024,
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('handles a non-existent dataDir gracefully', async () => {
|
||||
await expect(
|
||||
runReflectionRetentionSweep(
|
||||
{ dataDir: join(tmpDir, 'does-not-exist') },
|
||||
{ snapshotRetentionDays: 90, snapshotMaxBytesPerUser: 100 * 1024 * 1024 },
|
||||
),
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('single snapshot under cap and within retention is not touched', async () => {
|
||||
const uid = 'u-clean';
|
||||
const hDir = histDir(tmpDir, uid);
|
||||
mkdirSync(hDir, { recursive: true });
|
||||
const snapId = makeSnapshot(hDir, 5, 'j-only', 100);
|
||||
|
||||
await runReflectionRetentionSweep(makeDeps(tmpDir), {
|
||||
snapshotRetentionDays: 90,
|
||||
snapshotMaxBytesPerUser: 100 * 1024 * 1024,
|
||||
});
|
||||
|
||||
expect(existsSync(join(hDir, snapId))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user