feat: initial public release (MAESTRO)

2026-06-03 05:08:00 +00:00
commit f5c7666f6b
823 changed files with 184150 additions and 0 deletions
@@ -0,0 +1,133 @@
+import { describe, it, expect, vi } from 'vitest';
+import { createStickyBackendResolver, type StickyBackendLogger } from './sticky-backend.js';
+
+function makeLogger(): StickyBackendLogger & {
+  calls: { debug: string[]; info: string[]; warn: string[] };
+} {
+  const calls = { debug: [] as string[], info: [] as string[], warn: [] as string[] };
+  return {
+    calls,
+    debug: (m) => calls.debug.push(m),
+    info: (m) => calls.info.push(m),
+    warn: (m) => calls.warn.push(m),
+  };
+}
+
+describe('createStickyBackendResolver', () => {
+  it('persists the first backend, sets sticky, and logs at info', async () => {
+    const logger = makeLogger();
+    const persist = vi.fn().mockResolvedValue(undefined);
+    const resolve = createStickyBackendResolver({
+      initial: null,
+      persist,
+      logger,
+      workerId: 'w1',
+      jobId: 'j1',
+    });
+
+    await resolve({ backendId: 'gpu-a', cacheKey: null });
+
+    expect(persist).toHaveBeenCalledTimes(1);
+    expect(persist).toHaveBeenCalledWith('gpu-a');
+    expect(logger.calls.info).toHaveLength(1);
+    expect(logger.calls.info[0]).toContain('gpu-a');
+  });
+
+  it('short-circuits subsequent events once sticky is set', async () => {
+    const logger = makeLogger();
+    const persist = vi.fn().mockResolvedValue(undefined);
+    const resolve = createStickyBackendResolver({
+      initial: null,
+      persist,
+      logger,
+      workerId: 'w1',
+      jobId: 'j1',
+    });
+
+    await resolve({ backendId: 'gpu-a', cacheKey: null });
+    await resolve({ backendId: 'gpu-b', cacheKey: 'sha:xyz' });
+    await resolve({ backendId: 'gpu-a', cacheKey: null });
+
+    expect(persist).toHaveBeenCalledTimes(1);
+    // gpu-b ≠ sticky → debug; gpu-a == sticky → no log
+    expect(logger.calls.debug).toHaveLength(1);
+    expect(logger.calls.debug[0]).toContain('gpu-b');
+  });
+
+  it('honors initial sticky from DB without re-persisting', async () => {
+    const logger = makeLogger();
+    const persist = vi.fn().mockResolvedValue(undefined);
+    const resolve = createStickyBackendResolver({
+      initial: 'gpu-seed',
+      persist,
+      logger,
+      workerId: 'w1',
+      jobId: 'j1',
+    });
+
+    await resolve({ backendId: 'gpu-other', cacheKey: null });
+    await resolve({ backendId: 'gpu-seed', cacheKey: null });
+
+    expect(persist).not.toHaveBeenCalled();
+    expect(logger.calls.debug).toHaveLength(1);
+    expect(logger.calls.debug[0]).toContain('gpu-other');
+  });
+
+  it('does NOT set sticky when persist fails — next event retries', async () => {
+    const logger = makeLogger();
+    const persist = vi
+      .fn()
+      .mockRejectedValueOnce(new Error('SQLITE_BUSY'))
+      .mockResolvedValueOnce(undefined);
+    const resolve = createStickyBackendResolver({
+      initial: null,
+      persist,
+      logger,
+      workerId: 'w1',
+      jobId: 'j1',
+    });
+
+    // First call: DB write fails → sticky unset → warn logged
+    await resolve({ backendId: 'gpu-a', cacheKey: null });
+    expect(persist).toHaveBeenCalledTimes(1);
+    expect(logger.calls.warn).toHaveLength(1);
+    expect(logger.calls.warn[0]).toContain('SQLITE_BUSY');
+    expect(logger.calls.info).toHaveLength(0);
+
+    // Second call: DB write succeeds → sticky set
+    await resolve({ backendId: 'gpu-b', cacheKey: null });
+    expect(persist).toHaveBeenCalledTimes(2);
+    expect(persist).toHaveBeenLastCalledWith('gpu-b');
+    expect(logger.calls.info).toHaveLength(1);
+    expect(logger.calls.info[0]).toContain('gpu-b');
+
+    // Third call: sticky is now set → no further persist
+    await resolve({ backendId: 'gpu-c', cacheKey: null });
+    expect(persist).toHaveBeenCalledTimes(2);
+  });
+
+  it('retries on every event until persist succeeds (multiple failures)', async () => {
+    const logger = makeLogger();
+    const persist = vi
+      .fn()
+      .mockRejectedValueOnce(new Error('fail 1'))
+      .mockRejectedValueOnce(new Error('fail 2'))
+      .mockResolvedValueOnce(undefined);
+    const resolve = createStickyBackendResolver({
+      initial: null,
+      persist,
+      logger,
+      workerId: 'w1',
+      jobId: 'j1',
+    });
+
+    await resolve({ backendId: 'gpu-a', cacheKey: null });
+    await resolve({ backendId: 'gpu-b', cacheKey: null });
+    await resolve({ backendId: 'gpu-c', cacheKey: null });
+
+    expect(persist).toHaveBeenCalledTimes(3);
+    expect(logger.calls.warn).toHaveLength(2);
+    expect(logger.calls.info).toHaveLength(1);
+    expect(logger.calls.info[0]).toContain('gpu-c');
+  });
+});
@@ -0,0 +1,81 @@
+/**
+ * Sticky-backend resolver for proxy worker jobs.
+ *
+ * Design (per
+ *   docs/superpowers/specs/2026-05-18-multi-team-gpu-pool-and-node-status-design.md
+ * Open Question #3 case 1, "最初に確定したら以後 update しない"):
+ *
+ * - For a proxy worker, every LLM call may resolve to a different
+ *   physical backend (LiteLLM rebalances per request). The UI pet should
+ *   not flicker, so we record only the FIRST backend a job sees.
+ * - Persistence happens via `updateJob({ lastBackendId })`. If that DB
+ *   write FAILS, we must remain in the unset state so the next
+ *   `onBackendResolved` event has a chance to retry. If we set the local
+ *   sticky variable BEFORE persisting, a transient DB error would lose
+ *   the binding permanently for the lifetime of the job (sticky check
+ *   short-circuits all subsequent events) and the UI would never see
+ *   the worker → backend mapping.
+ *
+ * This module isolates the "set sticky only after persist succeeds"
+ * invariant from `Worker.buildPieceRunCallbacks`, which already has a
+ * dozen other concerns and is hard to unit-test in isolation.
+ */
+
+export interface StickyBackendLogger {
+  debug: (msg: string) => void;
+  info: (msg: string) => void;
+  warn: (msg: string) => void;
+}
+
+export interface StickyBackendEvent {
+  backendId: string;
+  cacheKey: string | null;
+}
+
+/**
+ * Build the `onBackendResolved` callback. `persist(backendId)` is the
+ * DB write (typically `repo.updateJob(jobId, { lastBackendId })`); it
+ * must reject on failure so we can keep sticky unset for retry.
+ *
+ * Returns an async function the agent-loop can call without awaiting —
+ * errors are caught internally and logged. The function resolves once
+ * either: (a) sticky was already set and we short-circuited, (b) the
+ * persist succeeded and sticky is now set, or (c) the persist failed
+ * and sticky remains unset for the next event to retry.
+ */
+export function createStickyBackendResolver(opts: {
+  initial: string | null;
+  persist: (backendId: string) => Promise<void>;
+  logger: StickyBackendLogger;
+  workerId: string;
+  jobId: string;
+}): (event: StickyBackendEvent) => Promise<void> {
+  const { initial, persist, logger, workerId, jobId } = opts;
+  let sticky: string | null = initial;
+
+  return async function onBackendResolved({ backendId, cacheKey }: StickyBackendEvent): Promise<void> {
+    if (sticky) {
+      if (sticky !== backendId) {
+        logger.debug(
+          `[worker:${workerId}] job ${jobId} backend re-resolved to ${backendId} (sticky=${sticky}, cache=${cacheKey ?? 'miss'}); keeping sticky`,
+        );
+      }
+      return;
+    }
+    try {
+      await persist(backendId);
+      // Only set sticky AFTER persist succeeds. If we set first and persist
+      // failed, the next event would short-circuit on the sticky check and
+      // we'd never recover — the UI would render "no backend" forever.
+      sticky = backendId;
+      logger.info(
+        `[worker:${workerId}] job ${jobId} backend resolved: ${backendId} cache=${cacheKey ?? 'miss'}`,
+      );
+    } catch (err) {
+      logger.warn(
+        `[worker:${workerId}] failed to persist lastBackendId for job ${jobId}: ${err} — sticky left unset for retry`,
+      );
+      // Intentionally do NOT set sticky. Next event retries.
+    }
+  };
+}