feat: initial public release (MAESTRO)
This commit is contained in:
oss-sync
@@ -0,0 +1,94 @@
|
||||
import { describe, it, expect, beforeEach } from 'vitest';
|
||||
import { Registry } from 'prom-client';
|
||||
import { createGatewayMetrics } from './gateway-metrics.js';
|
||||
|
||||
describe('metrics/gateway-metrics', () => {
|
||||
let reg: Registry;
|
||||
|
||||
beforeEach(() => {
|
||||
reg = new Registry();
|
||||
});
|
||||
|
||||
it('registers all 11 metrics under the supplied prefix', async () => {
|
||||
createGatewayMetrics(reg, 'aao_gateway_t');
|
||||
const out = await reg.metrics();
|
||||
for (const name of [
|
||||
'aao_gateway_t_requests_total',
|
||||
'aao_gateway_t_request_duration_seconds',
|
||||
'aao_gateway_t_tokens_total',
|
||||
'aao_gateway_t_backend_busy_slots',
|
||||
'aao_gateway_t_backend_total_slots',
|
||||
'aao_gateway_t_backend_online',
|
||||
'aao_gateway_t_cache_hit_total',
|
||||
'aao_gateway_t_cache_miss_total',
|
||||
'aao_gateway_t_virtual_key_budget_used_ratio',
|
||||
'aao_gateway_t_rate_limit_rejections_total',
|
||||
'aao_gateway_t_active_streams',
|
||||
]) {
|
||||
expect(out).toContain(name);
|
||||
}
|
||||
});
|
||||
|
||||
it('counters emit a series per unique label set', async () => {
|
||||
const m = createGatewayMetrics(reg, 'aao_gateway_c');
|
||||
m.requestsTotal.inc({ team: 'alpha', backend: 'gpu-a', model: 'qwen3:8b', status: 'success' }, 3);
|
||||
m.requestsTotal.inc({ team: 'alpha', backend: 'gpu-a', model: 'qwen3:8b', status: 'success' }, 2);
|
||||
m.requestsTotal.inc({ team: 'bravo', backend: 'gpu-b', model: 'qwen3:8b', status: 'success' }, 1);
|
||||
const out = await reg.metrics();
|
||||
// 3 + 2 = 5 on the alpha series
|
||||
expect(out).toMatch(/aao_gateway_c_requests_total\{[^}]*team="alpha"[^}]*\} 5/);
|
||||
expect(out).toMatch(/aao_gateway_c_requests_total\{[^}]*team="bravo"[^}]*\} 1/);
|
||||
});
|
||||
|
||||
it('histogram observe records a single sample', async () => {
|
||||
const m = createGatewayMetrics(reg, 'aao_gateway_h');
|
||||
m.requestDurationSeconds.observe(
|
||||
{ team: 'alpha', backend: 'gpu-a', model: 'qwen3:8b', status: 'success' },
|
||||
0.42,
|
||||
);
|
||||
const out = await reg.metrics();
|
||||
expect(out).toContain('aao_gateway_h_request_duration_seconds_count{');
|
||||
expect(out).toMatch(/aao_gateway_h_request_duration_seconds_sum\{[^}]+\} 0\.42/);
|
||||
});
|
||||
|
||||
it('gauge set + reset returns latest value', async () => {
|
||||
const m = createGatewayMetrics(reg, 'aao_gateway_g');
|
||||
m.backendBusySlots.set({ backend: 'gpu-a' }, 4);
|
||||
m.backendBusySlots.set({ backend: 'gpu-a' }, 7);
|
||||
const out = await reg.metrics();
|
||||
expect(out).toMatch(/aao_gateway_g_backend_busy_slots\{backend="gpu-a"\} 7/);
|
||||
});
|
||||
|
||||
it('activeStreams gauge has no labels', async () => {
|
||||
const m = createGatewayMetrics(reg, 'aao_gateway_s');
|
||||
m.activeStreams.set(5);
|
||||
const out = await reg.metrics();
|
||||
expect(out).toContain('aao_gateway_s_active_streams 5');
|
||||
});
|
||||
|
||||
it('resetMetrics zeroes counters without unregistering', async () => {
|
||||
const m = createGatewayMetrics(reg, 'aao_gateway_r');
|
||||
m.cacheHitTotal.inc({ cache: 'key' }, 10);
|
||||
reg.resetMetrics();
|
||||
const out = await reg.metrics();
|
||||
expect(out).toContain('aao_gateway_r_cache_hit_total');
|
||||
// No data line after reset (counter not yet observed post-reset)
|
||||
expect(out).not.toMatch(/aao_gateway_r_cache_hit_total\{cache="key"\} 10/);
|
||||
});
|
||||
|
||||
it('is idempotent on (registry, prefix) — returns the cached handle, does NOT throw', () => {
|
||||
// CRITICAL-1 fix: prom-client throws on duplicate metric name
|
||||
// registration, but createGatewayMetrics is memoized so callers
|
||||
// (e.g. same-process gateway bounces with a shared bridge
|
||||
// Registry) can safely call it again. The cache is per (registry,
|
||||
// prefix) so a different prefix on the same registry still
|
||||
// registers fresh counters.
|
||||
const first = createGatewayMetrics(reg, 'aao_gateway_dup');
|
||||
expect(() => createGatewayMetrics(reg, 'aao_gateway_dup')).not.toThrow();
|
||||
const second = createGatewayMetrics(reg, 'aao_gateway_dup');
|
||||
expect(second).toBe(first); // same handle, no fresh Counter instances
|
||||
// Different prefix → fresh registration, no cache hit
|
||||
const other = createGatewayMetrics(reg, 'aao_gateway_other');
|
||||
expect(other).not.toBe(first);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,223 @@
|
||||
/**
|
||||
* Phase 3b — Gateway-side Prometheus metrics.
|
||||
*
|
||||
* 11 metrics, all prefixed with `aao_gateway_` by default. Labels are
|
||||
* kept narrow to bound cardinality: `team` and `backend` come from the
|
||||
* static config (small N), `model` from a request body (bounded by the
|
||||
* router's allowlist), `status` from a closed enum.
|
||||
*
|
||||
* Hot-path emission order:
|
||||
* - `requestsTotal` + `requestDurationSeconds`: fired in
|
||||
* stream-proxy.ts's finally for every response (success and error).
|
||||
* - `tokensTotal`: fired alongside requestsTotal when usage was
|
||||
* observed (skipped when zero — we don't want labels with 0 sum).
|
||||
* - `backendBusySlots` / `backendTotalSlots` / `backendOnline`: pushed
|
||||
* from a BackendStatusRegistry.subscribe() callback in bootstrap.
|
||||
* - `cacheHitTotal` / `cacheMissTotal`: bootstrap's dbLookup wrapper
|
||||
* inc()s these on each path through the keyCache.
|
||||
* - `budgetUsedRatio`: pushed when a usage write completes
|
||||
* (recordUsage callback in bootstrap).
|
||||
* - `rateLimitRejectionsTotal`: fired by the rate-limit middleware
|
||||
* reject branch.
|
||||
* - `activeStreams`: bootstrap subscribes a tick that reads the
|
||||
* StreamRegistry size every 5s (gauge — not delta).
|
||||
*
|
||||
* Cardinality budget (5 backends × 3 teams × 5 models × 5 statuses):
|
||||
* - requestsTotal: 375 series — fine
|
||||
* - requestDurationSeconds: same × Prom default 10 buckets = 3750
|
||||
* observation series — still well under the 100k informal cap
|
||||
*/
|
||||
import { Counter, Gauge, Histogram, type Registry } from 'prom-client';
|
||||
|
||||
export interface GatewayMetrics {
|
||||
/** Total chat/completions requests fired. Labels: team, backend, model, status. */
|
||||
requestsTotal: Counter<'team' | 'backend' | 'model' | 'status'>;
|
||||
/** Request processing latency seconds (default buckets). */
|
||||
requestDurationSeconds: Histogram<'team' | 'backend' | 'model' | 'status'>;
|
||||
/** Cumulative tokens routed through the gateway. direction is in/out. */
|
||||
tokensTotal: Counter<'team' | 'backend' | 'model' | 'direction'>;
|
||||
/** Latest BackendStatusRegistry busySlots for each backend. */
|
||||
backendBusySlots: Gauge<'backend'>;
|
||||
/** Latest totalSlots for each backend (constant per backend, but expose for completeness). */
|
||||
backendTotalSlots: Gauge<'backend'>;
|
||||
/** 1 = online, 0 = offline (registry probe failed). */
|
||||
backendOnline: Gauge<'backend'>;
|
||||
/** keyCache hit counter. Label `cache` keeps room for budget / rate / key sub-caches. */
|
||||
cacheHitTotal: Counter<'cache'>;
|
||||
/** keyCache miss counter. */
|
||||
cacheMissTotal: Counter<'cache'>;
|
||||
/**
|
||||
* Per-key budget usage ratio (used / budget) in [0, 1+]. Reset monthly
|
||||
* when the underlying gateway_key_usage period rolls over.
|
||||
*/
|
||||
budgetUsedRatio: Gauge<'team' | 'key_prefix'>;
|
||||
/** 429s emitted by the rate-limit middleware. */
|
||||
rateLimitRejectionsTotal: Counter<'team'>;
|
||||
/** Currently in-flight SSE streams (StreamRegistry size). */
|
||||
activeStreams: Gauge;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-(registry, prefix) memoization cache.
|
||||
*
|
||||
* prom-client throws on duplicate metric name registration. In
|
||||
* same-process gateway mode the bridge owns one shared Registry but
|
||||
* `createGatewayMetrics` is invoked again every time `startGateway()`
|
||||
* runs (e.g. `enabled: false → true → false → true` bounce, or a
|
||||
* backend list edit triggering a stop+start cycle). Without
|
||||
* memoization the second start throws and crashes the bridge process.
|
||||
*
|
||||
* Cache key: the Registry instance × the metric-name prefix. Using a
|
||||
* WeakMap on the Registry lets the cache GC naturally when the
|
||||
* registry is dropped (tests typically discard registries between
|
||||
* cases). The inner `Map<prefix, GatewayMetrics>` allows multiple
|
||||
* prefixes against the same registry (rare but legitimate — e.g. two
|
||||
* gateway mounts on different prefixes in test).
|
||||
*
|
||||
* Stop()/teardown semantics: callers MUST NOT delete metric instances
|
||||
* from the registry on stop — doing so would break the next start
|
||||
* (the cache would still hold a Counter whose internal registry
|
||||
* pointer dangles). Stop should reset label values via `.remove(label)`
|
||||
* or `.reset()` only.
|
||||
*/
|
||||
const metricsCache = new WeakMap<Registry, Map<string, GatewayMetrics>>();
|
||||
|
||||
/**
|
||||
* Wire all 11 metrics onto the supplied registry and return the typed
|
||||
* handle. Pass the same registry instance to every call site so a
|
||||
* `/metrics` scrape sees a single coherent snapshot.
|
||||
*
|
||||
* `prefix` only changes the metric name root; we keep it as a function
|
||||
* arg so tests can use a unique prefix per case and avoid the
|
||||
* "duplicate registration" prom-client error.
|
||||
*
|
||||
* Idempotent on `(registry, prefix)` — a second call with the same
|
||||
* args returns the cached handle instead of re-registering counters
|
||||
* (which prom-client would reject by throwing). See `metricsCache`
|
||||
* doc for why same-process gateway bounces need this.
|
||||
*/
|
||||
export function createGatewayMetrics(
|
||||
registry: Registry,
|
||||
prefix: string = 'aao_gateway',
|
||||
): GatewayMetrics {
|
||||
let perRegistry = metricsCache.get(registry);
|
||||
if (perRegistry) {
|
||||
const cached = perRegistry.get(prefix);
|
||||
if (cached) return cached;
|
||||
} else {
|
||||
perRegistry = new Map<string, GatewayMetrics>();
|
||||
metricsCache.set(registry, perRegistry);
|
||||
}
|
||||
|
||||
const handle = buildGatewayMetrics(registry, prefix);
|
||||
perRegistry.set(prefix, handle);
|
||||
return handle;
|
||||
}
|
||||
|
||||
function buildGatewayMetrics(registry: Registry, prefix: string): GatewayMetrics {
|
||||
const requestsTotal = new Counter({
|
||||
name: `${prefix}_requests_total`,
|
||||
help: 'Total chat/completions requests processed by the gateway.',
|
||||
labelNames: ['team', 'backend', 'model', 'status'] as const,
|
||||
registers: [registry],
|
||||
});
|
||||
|
||||
const requestDurationSeconds = new Histogram({
|
||||
name: `${prefix}_request_duration_seconds`,
|
||||
help: 'Gateway request processing time in seconds (incl. upstream).',
|
||||
labelNames: ['team', 'backend', 'model', 'status'] as const,
|
||||
// Prom-client default buckets are tuned for HTTP latency.
|
||||
registers: [registry],
|
||||
});
|
||||
|
||||
const tokensTotal = new Counter({
|
||||
name: `${prefix}_tokens_total`,
|
||||
help: 'Cumulative tokens routed through the gateway (in / out).',
|
||||
labelNames: ['team', 'backend', 'model', 'direction'] as const,
|
||||
registers: [registry],
|
||||
});
|
||||
|
||||
const backendBusySlots = new Gauge({
|
||||
name: `${prefix}_backend_busy_slots`,
|
||||
help: 'BackendStatusRegistry busySlots for each backend.',
|
||||
labelNames: ['backend'] as const,
|
||||
registers: [registry],
|
||||
});
|
||||
|
||||
const backendTotalSlots = new Gauge({
|
||||
name: `${prefix}_backend_total_slots`,
|
||||
help: 'BackendStatusRegistry totalSlots for each backend.',
|
||||
labelNames: ['backend'] as const,
|
||||
registers: [registry],
|
||||
});
|
||||
|
||||
const backendOnline = new Gauge({
|
||||
name: `${prefix}_backend_online`,
|
||||
help: 'Backend health: 1 = reachable, 0 = probe failed.',
|
||||
labelNames: ['backend'] as const,
|
||||
registers: [registry],
|
||||
});
|
||||
|
||||
const cacheHitTotal = new Counter({
|
||||
name: `${prefix}_cache_hit_total`,
|
||||
help: 'Key cache hits by sub-cache (key/backends).',
|
||||
labelNames: ['cache'] as const,
|
||||
registers: [registry],
|
||||
});
|
||||
|
||||
const cacheMissTotal = new Counter({
|
||||
name: `${prefix}_cache_miss_total`,
|
||||
help: 'Key cache misses by sub-cache.',
|
||||
labelNames: ['cache'] as const,
|
||||
registers: [registry],
|
||||
});
|
||||
|
||||
const budgetUsedRatio = new Gauge({
|
||||
name: `${prefix}_virtual_key_budget_used_ratio`,
|
||||
help: 'Per-virtual-key budget usage ratio (used / budget); 0 means unlimited.',
|
||||
labelNames: ['team', 'key_prefix'] as const,
|
||||
registers: [registry],
|
||||
});
|
||||
|
||||
const rateLimitRejectionsTotal = new Counter({
|
||||
name: `${prefix}_rate_limit_rejections_total`,
|
||||
help: 'Total 429 rejections by the rate-limit middleware.',
|
||||
labelNames: ['team'] as const,
|
||||
registers: [registry],
|
||||
});
|
||||
|
||||
const activeStreams = new Gauge({
|
||||
name: `${prefix}_active_streams`,
|
||||
help: 'In-flight SSE streams managed by the StreamRegistry.',
|
||||
registers: [registry],
|
||||
});
|
||||
|
||||
return {
|
||||
requestsTotal,
|
||||
requestDurationSeconds,
|
||||
tokensTotal,
|
||||
backendBusySlots,
|
||||
backendTotalSlots,
|
||||
backendOnline,
|
||||
cacheHitTotal,
|
||||
cacheMissTotal,
|
||||
budgetUsedRatio,
|
||||
rateLimitRejectionsTotal,
|
||||
activeStreams,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Closed enum of request status labels — exported so call sites get a
|
||||
* compile-time check rather than scattering string literals.
|
||||
*/
|
||||
export type GatewayRequestStatus =
|
||||
| 'success'
|
||||
| 'upstream_error'
|
||||
| 'gateway_timeout'
|
||||
| 'gateway_shutdown'
|
||||
| 'client_aborted'
|
||||
| 'auth_fail'
|
||||
| 'budget_exhausted'
|
||||
| 'rate_limited'
|
||||
| 'no_backend';
|
||||
@@ -0,0 +1,170 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import express from 'express';
|
||||
import request from 'supertest';
|
||||
import { Registry } from 'prom-client';
|
||||
import { createMetricsHandler } from './http-handler.js';
|
||||
import { createGatewayMetrics } from './gateway-metrics.js';
|
||||
|
||||
describe('metrics/http-handler', () => {
|
||||
function makeApp(): { app: express.Express; reg: Registry } {
|
||||
const app = express();
|
||||
const reg = new Registry();
|
||||
createGatewayMetrics(reg, 'aao_gateway_h');
|
||||
// Default (no auth opts) → localhost-only allowlist; supertest
|
||||
// connects from 127.0.0.1 so it passes.
|
||||
app.get('/metrics', createMetricsHandler(reg));
|
||||
return { app, reg };
|
||||
}
|
||||
|
||||
it('responds 200 with text/plain version=0.0.4', async () => {
|
||||
const { app } = makeApp();
|
||||
const res = await request(app).get('/metrics');
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers['content-type']).toMatch(/^text\/plain.*version=0\.0\.4/);
|
||||
});
|
||||
|
||||
it('payload contains registered metric names', async () => {
|
||||
const { app } = makeApp();
|
||||
const res = await request(app).get('/metrics');
|
||||
expect(res.text).toContain('aao_gateway_h_requests_total');
|
||||
});
|
||||
|
||||
it('returns 500 with text/plain when render throws', async () => {
|
||||
const reg = {
|
||||
contentType: 'text/plain',
|
||||
// Force a rejection out of metrics()
|
||||
metrics: () => Promise.reject(new Error('boom')),
|
||||
} as unknown as Registry;
|
||||
const app = express();
|
||||
app.get('/metrics', createMetricsHandler(reg));
|
||||
const res = await request(app).get('/metrics');
|
||||
expect(res.status).toBe(500);
|
||||
expect(res.headers['content-type']).toMatch(/text\/plain/);
|
||||
});
|
||||
|
||||
describe('auth gating (Phase 3b post-review)', () => {
|
||||
it('default config allows 127.0.0.1 (localhost)', async () => {
|
||||
const reg = new Registry();
|
||||
createGatewayMetrics(reg, 'aao_gateway_l1');
|
||||
const app = express();
|
||||
app.get('/metrics', createMetricsHandler(reg));
|
||||
const res = await request(app).get('/metrics');
|
||||
expect(res.status).toBe(200);
|
||||
});
|
||||
|
||||
it('default config rejects a non-localhost client IP with 403', async () => {
|
||||
const reg = new Registry();
|
||||
createGatewayMetrics(reg, 'aao_gateway_l2');
|
||||
const app = express();
|
||||
// Simulate an external IP via a stub middleware before the
|
||||
// handler. Defining `req.ip` after Express assigned it requires
|
||||
// setting the underlying socket; we override the getter directly.
|
||||
app.use((req, _res, next) => {
|
||||
Object.defineProperty(req, 'ip', { value: '203.0.113.5', configurable: true });
|
||||
// Also override socket.remoteAddress so the fallback check
|
||||
// doesn't see the real supertest socket (127.0.0.1).
|
||||
Object.defineProperty(req, 'socket', {
|
||||
value: { remoteAddress: '203.0.113.5' },
|
||||
configurable: true,
|
||||
});
|
||||
next();
|
||||
});
|
||||
app.get('/metrics', createMetricsHandler(reg));
|
||||
const res = await request(app).get('/metrics');
|
||||
expect(res.status).toBe(403);
|
||||
});
|
||||
|
||||
it('bearer token: 200 on correct Authorization header', async () => {
|
||||
const reg = new Registry();
|
||||
createGatewayMetrics(reg, 'aao_gateway_t1');
|
||||
const app = express();
|
||||
app.get('/metrics', createMetricsHandler(reg, { bearerToken: 'sk-metrics-secret' }));
|
||||
const res = await request(app).get('/metrics').set('Authorization', 'Bearer sk-metrics-secret');
|
||||
expect(res.status).toBe(200);
|
||||
});
|
||||
|
||||
it('bearer token: 401 with WWW-Authenticate when missing/wrong', async () => {
|
||||
const reg = new Registry();
|
||||
createGatewayMetrics(reg, 'aao_gateway_t2');
|
||||
const app = express();
|
||||
app.get('/metrics', createMetricsHandler(reg, { bearerToken: 'sk-metrics-secret' }));
|
||||
const r1 = await request(app).get('/metrics');
|
||||
expect(r1.status).toBe(401);
|
||||
expect(r1.headers['www-authenticate']).toMatch(/Bearer/);
|
||||
const r2 = await request(app).get('/metrics').set('Authorization', 'Bearer wrong-token');
|
||||
expect(r2.status).toBe(401);
|
||||
});
|
||||
|
||||
it('bearer token wins over IP allowlist (correct token from external IP → 200)', async () => {
|
||||
const reg = new Registry();
|
||||
createGatewayMetrics(reg, 'aao_gateway_t3');
|
||||
const app = express();
|
||||
app.use((req, _res, next) => {
|
||||
Object.defineProperty(req, 'ip', { value: '203.0.113.5', configurable: true });
|
||||
// Also override socket.remoteAddress so the fallback check
|
||||
// doesn't see the real supertest socket (127.0.0.1).
|
||||
Object.defineProperty(req, 'socket', {
|
||||
value: { remoteAddress: '203.0.113.5' },
|
||||
configurable: true,
|
||||
});
|
||||
next();
|
||||
});
|
||||
app.get('/metrics', createMetricsHandler(reg, {
|
||||
bearerToken: 'tok',
|
||||
allowedHosts: ['127.0.0.1'],
|
||||
}));
|
||||
const res = await request(app).get('/metrics').set('Authorization', 'Bearer tok');
|
||||
expect(res.status).toBe(200);
|
||||
});
|
||||
|
||||
it('allowedHosts containing 0.0.0.0 disables IP checks', async () => {
|
||||
const reg = new Registry();
|
||||
createGatewayMetrics(reg, 'aao_gateway_h0');
|
||||
const app = express();
|
||||
app.use((req, _res, next) => {
|
||||
Object.defineProperty(req, 'ip', { value: '198.51.100.7', configurable: true });
|
||||
Object.defineProperty(req, 'socket', {
|
||||
value: { remoteAddress: '198.51.100.7' },
|
||||
configurable: true,
|
||||
});
|
||||
next();
|
||||
});
|
||||
app.get('/metrics', createMetricsHandler(reg, { allowedHosts: ['0.0.0.0'] }));
|
||||
const res = await request(app).get('/metrics');
|
||||
expect(res.status).toBe(200);
|
||||
});
|
||||
|
||||
it('allowedHosts custom list accepts listed IPs and rejects others', async () => {
|
||||
const reg = new Registry();
|
||||
createGatewayMetrics(reg, 'aao_gateway_h2');
|
||||
// Listed IP → 200
|
||||
const appOk = express();
|
||||
appOk.use((req, _res, next) => {
|
||||
Object.defineProperty(req, 'ip', { value: '10.0.0.5', configurable: true });
|
||||
Object.defineProperty(req, 'socket', {
|
||||
value: { remoteAddress: '10.0.0.5' },
|
||||
configurable: true,
|
||||
});
|
||||
next();
|
||||
});
|
||||
appOk.get('/metrics', createMetricsHandler(reg, { allowedHosts: ['10.0.0.5'] }));
|
||||
const r1 = await request(appOk).get('/metrics');
|
||||
expect(r1.status).toBe(200);
|
||||
// Unlisted IP → 403
|
||||
const reg2 = new Registry();
|
||||
createGatewayMetrics(reg2, 'aao_gateway_h3');
|
||||
const appBlocked = express();
|
||||
appBlocked.use((req, _res, next) => {
|
||||
Object.defineProperty(req, 'ip', { value: '10.0.0.99', configurable: true });
|
||||
Object.defineProperty(req, 'socket', {
|
||||
value: { remoteAddress: '10.0.0.99' },
|
||||
configurable: true,
|
||||
});
|
||||
next();
|
||||
});
|
||||
appBlocked.get('/metrics', createMetricsHandler(reg2, { allowedHosts: ['10.0.0.5'] }));
|
||||
const r2 = await request(appBlocked).get('/metrics');
|
||||
expect(r2.status).toBe(403);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,133 @@
|
||||
/**
|
||||
* Phase 3b — Express handler that exposes a Registry over HTTP.
|
||||
*
|
||||
* Returns the Prometheus text exposition format (text/plain;
|
||||
* version=0.0.4) — same shape every node_exporter / cAdvisor / litellm
|
||||
* `/metrics` endpoint produces. Both gateway and worker mount this on
|
||||
* `/metrics`.
|
||||
*
|
||||
* Auth model (Phase 3b post-review hardening)
|
||||
* ───────────────────────────────────────────
|
||||
* The worker / gateway HTTP servers are user-facing, so /metrics MUST
|
||||
* NOT be wide open by default — labels like `team` and `key_prefix`
|
||||
* (= the first 8 chars of a stable virtual-key id) leak to anyone who
|
||||
* can reach the port. We therefore gate the handler with **two**
|
||||
* complementary mechanisms, both opt-in via config:
|
||||
*
|
||||
* 1. Bearer token (`metrics.bearer_token`): when set, requests must
|
||||
* carry `Authorization: Bearer <token>`. Wins over the IP list —
|
||||
* lets operators run Prometheus from an arbitrary subnet without
|
||||
* embedding every scraper IP in config.
|
||||
* 2. Client-IP allowlist (`metrics.allowed_hosts`): when no bearer is
|
||||
* configured, only requests whose `req.ip` is in the allowlist
|
||||
* pass. Default `['127.0.0.1', '::1', 'localhost']` so a stock
|
||||
* deploy can be scraped from localhost (sidecar / SSH tunnel) and
|
||||
* nothing else.
|
||||
*
|
||||
* To open the endpoint to the world you either set a strong bearer
|
||||
* token or explicitly broaden `allowed_hosts`. There is no "no auth"
|
||||
* code path other than the default localhost allowlist.
|
||||
*
|
||||
* Errors during render are extremely rare (would require prom-client to
|
||||
* throw during string assembly) but if they do happen we surface a
|
||||
* generic 500 so the Prometheus scrape job records the failure
|
||||
* (`up == 0` for that target) instead of silently serving an empty
|
||||
* payload — a 200 with no metrics looks like "no data" in Grafana,
|
||||
* which is worse than an honest 500.
|
||||
*/
|
||||
import type { RequestHandler } from 'express';
|
||||
import type { Registry } from 'prom-client';
|
||||
import { logger } from '../logger.js';
|
||||
|
||||
export interface MetricsAuthOptions {
|
||||
/**
|
||||
* Optional bearer token. When present, requests must carry
|
||||
* `Authorization: Bearer <token>` or get a 401. Takes precedence
|
||||
* over the IP allowlist.
|
||||
*/
|
||||
bearerToken?: string;
|
||||
/**
|
||||
* Client-IP allowlist applied when no bearer token is configured.
|
||||
* Defaults to `['127.0.0.1', '::1', 'localhost']` — localhost only.
|
||||
* Include `0.0.0.0` to disable IP checks entirely (only do this when
|
||||
* an outer reverse proxy / firewall handles access control).
|
||||
*/
|
||||
allowedHosts?: string[];
|
||||
}
|
||||
|
||||
const DEFAULT_ALLOWED_HOSTS: ReadonlyArray<string> = ['127.0.0.1', '::1', 'localhost'];
|
||||
|
||||
/**
|
||||
* Build a handler that scrapes the supplied Registry on each request.
|
||||
* The handler is async — prom-client's metrics() returns a Promise so
|
||||
* we await it before calling res.end.
|
||||
*
|
||||
* The optional `auth` argument enables Phase 3b post-review hardening:
|
||||
* bearer-token auth (preferred) or a client-IP allowlist (default
|
||||
* localhost-only). See module doc for the full model.
|
||||
*/
|
||||
export function createMetricsHandler(
|
||||
registry: Registry,
|
||||
auth?: MetricsAuthOptions,
|
||||
): RequestHandler {
|
||||
const allowed = new Set(
|
||||
(auth?.allowedHosts && auth.allowedHosts.length > 0
|
||||
? auth.allowedHosts
|
||||
: DEFAULT_ALLOWED_HOSTS) as string[],
|
||||
);
|
||||
const allowAny = allowed.has('0.0.0.0') || allowed.has('*');
|
||||
const bearerToken = auth?.bearerToken && auth.bearerToken.length > 0 ? auth.bearerToken : null;
|
||||
|
||||
return async function metricsHandler(req, res, _next): Promise<void> {
|
||||
// 1. Bearer token (if configured) takes precedence over the IP
|
||||
// allowlist. This is the path operators use when scraping from
|
||||
// an arbitrary network — they trade IP discipline for token
|
||||
// secrecy.
|
||||
if (bearerToken) {
|
||||
const authHeader = req.headers['authorization'];
|
||||
const expected = `Bearer ${bearerToken}`;
|
||||
if (typeof authHeader !== 'string' || authHeader !== expected) {
|
||||
res.set('WWW-Authenticate', 'Bearer realm="metrics"').status(401).end();
|
||||
return;
|
||||
}
|
||||
} else if (!allowAny) {
|
||||
// 2. Fall back to a client-IP allowlist. We accept the request when
|
||||
// req.ip OR the raw remote address matches an entry. IPv4-mapped
|
||||
// IPv6 (::ffff:127.0.0.1) is normalized so a localhost rule works
|
||||
// on dual-stack hosts. `0.0.0.0` / `*` in the list disables the
|
||||
// check for shaved-down configurations (e.g., reverse proxy in
|
||||
// front of the gateway).
|
||||
const candidates: string[] = [];
|
||||
if (typeof req.ip === 'string' && req.ip.length > 0) {
|
||||
candidates.push(req.ip, req.ip.replace(/^::ffff:/, ''));
|
||||
}
|
||||
const remote = req.socket?.remoteAddress;
|
||||
if (typeof remote === 'string' && remote.length > 0) {
|
||||
candidates.push(remote, remote.replace(/^::ffff:/, ''));
|
||||
}
|
||||
const allowedHit = candidates.some(c => allowed.has(c));
|
||||
if (!allowedHit) {
|
||||
// 403, not 401: there's nothing the client can do — the IP isn't
|
||||
// on the list. WWW-Authenticate would mislead a bearer client
|
||||
// into retrying with a token that wouldn't be checked.
|
||||
res.status(403).end();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const body = await registry.metrics();
|
||||
res.setHeader('Content-Type', registry.contentType);
|
||||
res.status(200).end(body);
|
||||
} catch (err) {
|
||||
// Logged at warn (not error) because a flaky default-metric
|
||||
// collector shouldn't pager the on-call. Prometheus side will
|
||||
// mark the target down and alert via its own rules.
|
||||
logger.warn(
|
||||
`[metrics-handler] failed to render metrics: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
res.status(500).setHeader('Content-Type', 'text/plain; charset=utf-8');
|
||||
res.end('# failed to render metrics\n');
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
import { describe, it, expect, beforeEach } from 'vitest';
|
||||
import {
|
||||
createGatewayRegistry,
|
||||
createWorkerRegistry,
|
||||
getDefaultGatewayRegistry,
|
||||
getDefaultWorkerRegistry,
|
||||
resetAllRegistries,
|
||||
} from './registry.js';
|
||||
|
||||
describe('metrics/registry', () => {
|
||||
beforeEach(() => {
|
||||
resetAllRegistries();
|
||||
});
|
||||
|
||||
it('createGatewayRegistry returns a fresh Registry per call', () => {
|
||||
const a = createGatewayRegistry('aao_gateway_test_a');
|
||||
const b = createGatewayRegistry('aao_gateway_test_b');
|
||||
expect(a).not.toBe(b);
|
||||
});
|
||||
|
||||
it('default singletons are stable until reset', () => {
|
||||
const first = getDefaultGatewayRegistry('aao_gateway_test_singleton');
|
||||
const second = getDefaultGatewayRegistry('aao_gateway_test_singleton');
|
||||
expect(first).toBe(second);
|
||||
resetAllRegistries();
|
||||
const third = getDefaultGatewayRegistry('aao_gateway_test_singleton');
|
||||
expect(third).not.toBe(first);
|
||||
});
|
||||
|
||||
it('gateway and worker singletons are independent', () => {
|
||||
const g = getDefaultGatewayRegistry('aao_gateway_test_gw');
|
||||
const w = getDefaultWorkerRegistry('aao_worker_test_wk');
|
||||
expect(g).not.toBe(w);
|
||||
});
|
||||
|
||||
it('default metrics are wired (process_cpu_user_seconds_total appears)', async () => {
|
||||
const reg = createWorkerRegistry('aao_worker_dm');
|
||||
// Default-metric collection registers some metrics lazily; call
|
||||
// metrics() to force gathering.
|
||||
const out = await reg.metrics();
|
||||
// Process CPU is always present on Node + Linux.
|
||||
expect(out).toContain('aao_worker_dm_process_cpu_user_seconds_total');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,97 @@
|
||||
/**
|
||||
* Phase 3b — shared Prometheus registry plumbing.
|
||||
*
|
||||
* Both gateway mode and worker mode get their own Registry instance
|
||||
* (different prefixes, different metric sets) but they share this
|
||||
* factory so the wiring stays consistent: every registry gets the
|
||||
* default `process_*` / `nodejs_*` metrics from prom-client so SREs see
|
||||
* cpu / event-loop / heap metrics on every AAO process without any
|
||||
* extra config.
|
||||
*
|
||||
* Design:
|
||||
* - `createGatewayRegistry()` / `createWorkerRegistry()` return distinct
|
||||
* Registry instances. Tests pass these directly to avoid the
|
||||
* process-singleton trap (one Vitest worker running many tests must
|
||||
* not share metric state across cases).
|
||||
* - `getDefaultGatewayRegistry()` / `getDefaultWorkerRegistry()` lazily
|
||||
* create a singleton per process; called from bootstrap / bridge.
|
||||
* - `resetAllRegistries()` is a test escape hatch. Production code never
|
||||
* calls it.
|
||||
*
|
||||
* The label cardinality budget is enforced by call-site discipline (see
|
||||
* the per-metric files). Prom-client doesn't cap labels itself, so a
|
||||
* runaway labelset would silently grow memory until the next scrape —
|
||||
* we keep label sets narrow and well-typed in the metric factory funcs.
|
||||
*/
|
||||
import { Registry, collectDefaultMetrics } from 'prom-client';
|
||||
|
||||
let defaultGatewayRegistry: Registry | null = null;
|
||||
let defaultWorkerRegistry: Registry | null = null;
|
||||
|
||||
/**
|
||||
* Stand up a fresh Registry with `process_*` / `nodejs_*` collected on
|
||||
* scrape. The `prefix` param is forwarded to collectDefaultMetrics so
|
||||
* default metric names match the gateway / worker namespace (e.g.
|
||||
* `aao_gateway_process_cpu_user_seconds_total`).
|
||||
*/
|
||||
function createRegistry(prefix: string): Registry {
|
||||
const reg = new Registry();
|
||||
// Default labels let Grafana queries select by deployment without
|
||||
// having to label every counter individually. Empty for now — operators
|
||||
// typically add `instance` / `job` via the Prometheus scrape config.
|
||||
reg.setDefaultLabels({});
|
||||
collectDefaultMetrics({
|
||||
register: reg,
|
||||
// Slightly verbose names but keeps the namespace consistent.
|
||||
prefix: `${prefix}_`,
|
||||
});
|
||||
return reg;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the gateway-side Registry. Use this from
|
||||
* gateway/bootstrap.ts. Tests create their own to stay isolated.
|
||||
*/
|
||||
export function createGatewayRegistry(prefix: string = 'aao_gateway'): Registry {
|
||||
return createRegistry(prefix);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the worker-side Registry. Use this from bridge/server.ts.
|
||||
*/
|
||||
export function createWorkerRegistry(prefix: string = 'aao_worker'): Registry {
|
||||
return createRegistry(prefix);
|
||||
}
|
||||
|
||||
/**
|
||||
* Lazily create + return the process-singleton gateway registry. Bootstrap
|
||||
* calls this once at startup; admin handlers reach in (rare) via
|
||||
* getDefaultGatewayRegistry() to read current metric values.
|
||||
*/
|
||||
export function getDefaultGatewayRegistry(prefix: string = 'aao_gateway'): Registry {
|
||||
if (!defaultGatewayRegistry) {
|
||||
defaultGatewayRegistry = createGatewayRegistry(prefix);
|
||||
}
|
||||
return defaultGatewayRegistry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Lazily create + return the process-singleton worker registry.
|
||||
*/
|
||||
export function getDefaultWorkerRegistry(prefix: string = 'aao_worker'): Registry {
|
||||
if (!defaultWorkerRegistry) {
|
||||
defaultWorkerRegistry = createWorkerRegistry(prefix);
|
||||
}
|
||||
return defaultWorkerRegistry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test-only escape hatch: drop both singletons so the next call rebuilds
|
||||
* a fresh registry. Production code MUST NOT call this — clearing a
|
||||
* live registry mid-scrape produces zeroed counters that look like
|
||||
* regressions in Grafana.
|
||||
*/
|
||||
export function resetAllRegistries(): void {
|
||||
defaultGatewayRegistry = null;
|
||||
defaultWorkerRegistry = null;
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
/**
|
||||
* Phase 3b post-review — tool_name label normalization.
|
||||
*/
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { Registry } from 'prom-client';
|
||||
import { normalizeToolNameForMetric, BUILTIN_TOOL_NAMES } from './tool-name-allowlist.js';
|
||||
import { createWorkerMetrics } from './worker-metrics.js';
|
||||
|
||||
describe('normalizeToolNameForMetric', () => {
|
||||
it('passes built-in tool names through verbatim', () => {
|
||||
expect(normalizeToolNameForMetric('Read')).toBe('Read');
|
||||
expect(normalizeToolNameForMetric('WebFetch')).toBe('WebFetch');
|
||||
expect(normalizeToolNameForMetric('SpawnSubTask')).toBe('SpawnSubTask');
|
||||
});
|
||||
|
||||
it('collapses every mcp__* name to a single mcp bucket', () => {
|
||||
expect(normalizeToolNameForMetric('mcp__alpha__foo')).toBe('mcp');
|
||||
expect(normalizeToolNameForMetric('mcp__beta__bar')).toBe('mcp');
|
||||
expect(normalizeToolNameForMetric('mcp__github__create_issue')).toBe('mcp');
|
||||
});
|
||||
|
||||
it('collapses anything else to unknown', () => {
|
||||
expect(normalizeToolNameForMetric('some_random_tool')).toBe('unknown');
|
||||
expect(normalizeToolNameForMetric('')).toBe('unknown');
|
||||
expect(normalizeToolNameForMetric('not_a_real_tool_name')).toBe('unknown');
|
||||
});
|
||||
|
||||
it('does not treat partial mcp_ prefix as MCP (must be `mcp__`)', () => {
|
||||
expect(normalizeToolNameForMetric('mcp_foo')).toBe('unknown');
|
||||
expect(normalizeToolNameForMetric('mcpwhatever')).toBe('unknown');
|
||||
});
|
||||
|
||||
it('runtime tools (transition, complete) pass through', () => {
|
||||
expect(normalizeToolNameForMetric('transition')).toBe('transition');
|
||||
expect(normalizeToolNameForMetric('complete')).toBe('complete');
|
||||
});
|
||||
|
||||
it('BUILTIN_TOOL_NAMES contains the documented set of built-ins', () => {
|
||||
// Smoke-checks: ensure we didn't accidentally drop a known tool.
|
||||
const required = ['Read', 'Write', 'Edit', 'Bash', 'Glob', 'Grep', 'WebFetch', 'BrowseWeb'];
|
||||
for (const name of required) {
|
||||
expect(BUILTIN_TOOL_NAMES.has(name)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it('label cardinality stays bounded: many mcp__* names share one bucket', async () => {
|
||||
const reg = new Registry();
|
||||
const m = createWorkerMetrics(reg, 'aao_worker_n');
|
||||
// Simulate 100 distinct mcp__* names.
|
||||
for (let i = 0; i < 100; i += 1) {
|
||||
m.toolCallsTotal
|
||||
.labels({ tool_name: normalizeToolNameForMetric(`mcp__server${i}__tool${i}`), success: 'true' })
|
||||
.inc();
|
||||
}
|
||||
const dump = await reg.metrics();
|
||||
// All 100 calls should have collapsed into the single mcp bucket.
|
||||
expect(dump).toMatch(/aao_worker_n_tool_calls_total\{tool_name="mcp",success="true"\} 100/);
|
||||
// No mcp__ literals should leak into the metric body.
|
||||
expect(dump).not.toMatch(/tool_name="mcp__/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,114 @@
|
||||
/**
|
||||
* Phase 3b post-review — collapse tool names into a bounded label set
|
||||
* before they reach `aao_worker_tool_calls_total{tool_name}`.
|
||||
*
|
||||
* Without this normalization, a piece that calls `mcp__foo__bar` or a
|
||||
* user-defined ad-hoc tool would inject a fresh `tool_name` label on
|
||||
* every distinct name — `mcp__` names are user-controllable and can
|
||||
* grow unbounded, especially through Brainstorm-generated piece
|
||||
* variations. Once the cardinality explodes we either OOM prom-client
|
||||
* or get rate-limited by the scrape side.
|
||||
*
|
||||
* Policy:
|
||||
* - Known built-in tool name → pass through verbatim.
|
||||
* - Any `mcp__*` name → collapse to single label `mcp`.
|
||||
* - Anything else → collapse to `unknown`.
|
||||
*
|
||||
* Maintenance: when a new built-in tool ships, add its name to
|
||||
* BUILTIN_TOOL_NAMES below. The set is intentionally hard-coded (not
|
||||
* auto-derived from getToolDefs) so the metric label space is stable
|
||||
* across hot-reloads — if a tool module fails to load at runtime its
|
||||
* historical labels remain visible in Grafana instead of disappearing.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Pseudo-tools the agent loop fires that aren't user-callable but
|
||||
* still show up in the metric stream (transition / complete are
|
||||
* control-flow tools the runtime injects).
|
||||
*/
|
||||
const RUNTIME_TOOLS: ReadonlyArray<string> = [
|
||||
'transition',
|
||||
'complete',
|
||||
];
|
||||
|
||||
/**
|
||||
* Built-in tools listed in `src/engine/tools/*.ts`. Keep alphabetized
|
||||
* for easy maintenance + grep.
|
||||
*/
|
||||
const BUILTIN_TOOL_NAMES_LIST: ReadonlyArray<string> = [
|
||||
// core.ts
|
||||
'Bash', 'Edit', 'Glob', 'Grep', 'Read', 'Write',
|
||||
// web.ts
|
||||
'DownloadFile', 'WebFetch', 'WebSearch',
|
||||
// image.ts
|
||||
'AnnotateImage', 'ReadImage',
|
||||
// office.ts
|
||||
'PdfToImages', 'ReadDocx', 'ReadExcel', 'ReadPdf', 'ReadPPTX',
|
||||
'SplitDocxSections', 'SplitExcelSheets',
|
||||
// data.ts
|
||||
'SQLite',
|
||||
// review.ts
|
||||
'BatchReviewTextWithLLM', 'MergeReviewedResults',
|
||||
// browser.ts
|
||||
'BrowseWeb',
|
||||
// knowledge.ts
|
||||
'IngestDocument', 'IngestStatus', 'ListDocuments', 'ListNamespaces',
|
||||
'SearchKnowledge',
|
||||
// orchestration.ts
|
||||
'SpawnSubTask',
|
||||
// x.ts
|
||||
'XPostDetail', 'XSearch', 'XUserPosts',
|
||||
// maps.ts
|
||||
'GetDirections', 'ReverseGeocode', 'SearchPlaces',
|
||||
// youtube.ts
|
||||
'GetYouTubeTranscript', 'SearchYouTube',
|
||||
// amazon.ts
|
||||
'SearchAmazon',
|
||||
// speech.ts
|
||||
'TranscribeAudio',
|
||||
// checklist.ts
|
||||
'CheckItem', 'CreateChecklist', 'GetChecklist',
|
||||
// pieces.ts
|
||||
'CreatePiece', 'GetPiece', 'ListPieces', 'UpdatePiece',
|
||||
// docs.ts
|
||||
'ReadToolDoc',
|
||||
// mission.ts
|
||||
'MissionUpdate',
|
||||
// user-folder.ts
|
||||
'ListUserAssets', 'ReadUserTemplate', 'RenderUserTemplate', 'RunUserScript',
|
||||
// brainstorm.ts
|
||||
'Brainstorm',
|
||||
// app-docs.ts
|
||||
'GetMyOrchestratorState', 'ListAppDocs', 'ReadAppDoc',
|
||||
// ssh.ts
|
||||
'SshDownload', 'SshExec', 'SshListConnections', 'SshUpload',
|
||||
// ssh-console.ts
|
||||
'SshConsoleEnsure', 'SshConsoleSendKeys', 'SshConsoleSnapshot',
|
||||
// notes.ts
|
||||
'ReadNote', 'SearchNotes', 'WriteNote',
|
||||
// dashboard.ts
|
||||
'UpdateDashboardWidget',
|
||||
// ms-learn.ts (Microsoft Learn search)
|
||||
'MsLearnFetch', 'MsLearnRead', 'MsLearnSearch', 'MsLearnSummarize',
|
||||
// slide.ts
|
||||
'CreateSlide',
|
||||
];
|
||||
|
||||
export const BUILTIN_TOOL_NAMES: ReadonlySet<string> = new Set<string>([
|
||||
...BUILTIN_TOOL_NAMES_LIST,
|
||||
...RUNTIME_TOOLS,
|
||||
]);
|
||||
|
||||
/**
|
||||
* Map an arbitrary tool name into the bounded label space.
|
||||
*
|
||||
* Examples:
|
||||
* normalizeToolNameForMetric('Read') → 'Read'
|
||||
* normalizeToolNameForMetric('mcp__alpha__x') → 'mcp'
|
||||
* normalizeToolNameForMetric('user_custom') → 'unknown'
|
||||
*/
|
||||
export function normalizeToolNameForMetric(name: string): string {
|
||||
if (BUILTIN_TOOL_NAMES.has(name)) return name;
|
||||
if (name.startsWith('mcp__')) return 'mcp';
|
||||
return 'unknown';
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
import { describe, it, expect, beforeEach } from 'vitest';
|
||||
import { Registry } from 'prom-client';
|
||||
import { createWorkerMetrics } from './worker-metrics.js';
|
||||
|
||||
describe('metrics/worker-metrics', () => {
|
||||
let reg: Registry;
|
||||
|
||||
beforeEach(() => {
|
||||
reg = new Registry();
|
||||
});
|
||||
|
||||
it('registers all 6 metrics under the supplied prefix', async () => {
|
||||
createWorkerMetrics(reg, 'aao_worker_t');
|
||||
const out = await reg.metrics();
|
||||
for (const name of [
|
||||
'aao_worker_t_jobs_total',
|
||||
'aao_worker_t_active_jobs',
|
||||
'aao_worker_t_job_duration_seconds',
|
||||
'aao_worker_t_llm_calls_total',
|
||||
'aao_worker_t_llm_call_duration_seconds',
|
||||
'aao_worker_t_tool_calls_total',
|
||||
]) {
|
||||
expect(out).toContain(name);
|
||||
}
|
||||
});
|
||||
|
||||
it('activeJobs supports inc/dec', async () => {
|
||||
const m = createWorkerMetrics(reg, 'aao_worker_g');
|
||||
m.activeJobs.inc({ piece: 'chat', profile: 'main' });
|
||||
m.activeJobs.inc({ piece: 'chat', profile: 'main' });
|
||||
m.activeJobs.dec({ piece: 'chat', profile: 'main' });
|
||||
const out = await reg.metrics();
|
||||
expect(out).toMatch(/aao_worker_g_active_jobs\{piece="chat",profile="main"\} 1/);
|
||||
});
|
||||
|
||||
it('tool calls split by success label', async () => {
|
||||
const m = createWorkerMetrics(reg, 'aao_worker_tc');
|
||||
m.toolCallsTotal.inc({ tool_name: 'Read', success: 'true' }, 5);
|
||||
m.toolCallsTotal.inc({ tool_name: 'Read', success: 'false' }, 1);
|
||||
const out = await reg.metrics();
|
||||
expect(out).toMatch(/aao_worker_tc_tool_calls_total\{tool_name="Read",success="true"\} 5/);
|
||||
expect(out).toMatch(/aao_worker_tc_tool_calls_total\{tool_name="Read",success="false"\} 1/);
|
||||
});
|
||||
|
||||
it('LLM histogram observes seconds (not ms)', async () => {
|
||||
const m = createWorkerMetrics(reg, 'aao_worker_llm');
|
||||
m.llmCallDurationSeconds.observe(
|
||||
{ worker_id: 'w1', backend_id: 'gpu-a', model: 'qwen3:8b' },
|
||||
2.5,
|
||||
);
|
||||
const out = await reg.metrics();
|
||||
expect(out).toMatch(/aao_worker_llm_llm_call_duration_seconds_sum\{[^}]+\} 2\.5/);
|
||||
});
|
||||
|
||||
it('job-duration buckets fit minute-to-hour range', async () => {
|
||||
const m = createWorkerMetrics(reg, 'aao_worker_jd');
|
||||
m.jobDurationSeconds.observe(
|
||||
{ piece: 'chat', status: 'succeeded', profile: 'main' },
|
||||
75,
|
||||
);
|
||||
const out = await reg.metrics();
|
||||
// The 120s bucket should be incremented (75 < 120)
|
||||
expect(out).toMatch(/aao_worker_jd_job_duration_seconds_bucket\{[^}]*le="120"[^}]*\} 1/);
|
||||
// The 60s bucket should NOT be incremented (75 > 60)
|
||||
expect(out).toMatch(/aao_worker_jd_job_duration_seconds_bucket\{[^}]*le="60"[^}]*\} 0/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,104 @@
|
||||
/**
|
||||
* Phase 3b — Worker-side Prometheus metrics.
|
||||
*
|
||||
* 6 metrics, prefixed `aao_worker_` by default. Worker mode emits these
|
||||
* from worker.ts (job lifecycle) and agent-loop.ts (LLM call + tool
|
||||
* call counters).
|
||||
*
|
||||
* Cardinality budget:
|
||||
* - jobsTotal: pieces (~15) × statuses (~5) × profiles (~3) = ~225
|
||||
* - llmCallsTotal: workerIds (~5) × backendIds (~10) × models (~5) =
|
||||
* ~250 — the backendId axis matters when the worker talks to a
|
||||
* proxy (LiteLLM or the AAO gateway itself).
|
||||
* - toolCallsTotal: tool_names (~40) × success (2) = ~80
|
||||
*
|
||||
* Histograms emit bucket series proportional to label cardinality ×
|
||||
* 10 default buckets; even worst-case we stay well below the informal
|
||||
* 100k Prometheus cap.
|
||||
*/
|
||||
import { Counter, Gauge, Histogram, type Registry } from 'prom-client';
|
||||
|
||||
export interface WorkerMetrics {
|
||||
/** Total jobs completed by terminal status. */
|
||||
jobsTotal: Counter<'piece' | 'status' | 'profile'>;
|
||||
/** Currently-running jobs (gauge, inc on start / dec on end). */
|
||||
activeJobs: Gauge<'piece' | 'profile'>;
|
||||
/** Total wall-clock seconds per job (start → terminal). */
|
||||
jobDurationSeconds: Histogram<'piece' | 'status' | 'profile'>;
|
||||
/** Outbound LLM calls. backend_id is the proxy-resolved physical backend or the worker_id when direct. */
|
||||
llmCallsTotal: Counter<'worker_id' | 'backend_id' | 'model'>;
|
||||
/** Wall-clock seconds per LLM call. */
|
||||
llmCallDurationSeconds: Histogram<'worker_id' | 'backend_id' | 'model'>;
|
||||
/** Tool invocations from the agent loop. success is "true" / "false". */
|
||||
toolCallsTotal: Counter<'tool_name' | 'success'>;
|
||||
}
|
||||
|
||||
export function createWorkerMetrics(
|
||||
registry: Registry,
|
||||
prefix: string = 'aao_worker',
|
||||
): WorkerMetrics {
|
||||
const jobsTotal = new Counter({
|
||||
name: `${prefix}_jobs_total`,
|
||||
help: 'Total jobs completed by piece / status / profile.',
|
||||
labelNames: ['piece', 'status', 'profile'] as const,
|
||||
registers: [registry],
|
||||
});
|
||||
|
||||
const activeJobs = new Gauge({
|
||||
name: `${prefix}_active_jobs`,
|
||||
help: 'Currently-running jobs.',
|
||||
labelNames: ['piece', 'profile'] as const,
|
||||
registers: [registry],
|
||||
});
|
||||
|
||||
const jobDurationSeconds = new Histogram({
|
||||
name: `${prefix}_job_duration_seconds`,
|
||||
help: 'Wall-clock job duration in seconds.',
|
||||
labelNames: ['piece', 'status', 'profile'] as const,
|
||||
// Override default buckets — jobs run from seconds to many minutes,
|
||||
// so the HTTP-tuned defaults (5ms..10s) bury most observations.
|
||||
buckets: [1, 5, 15, 30, 60, 120, 300, 600, 1800, 3600],
|
||||
registers: [registry],
|
||||
});
|
||||
|
||||
const llmCallsTotal = new Counter({
|
||||
name: `${prefix}_llm_calls_total`,
|
||||
help: 'Outbound LLM API calls.',
|
||||
labelNames: ['worker_id', 'backend_id', 'model'] as const,
|
||||
registers: [registry],
|
||||
});
|
||||
|
||||
const llmCallDurationSeconds = new Histogram({
|
||||
name: `${prefix}_llm_call_duration_seconds`,
|
||||
help: 'Wall-clock latency per LLM call.',
|
||||
labelNames: ['worker_id', 'backend_id', 'model'] as const,
|
||||
// LLM calls vary 100ms..120s; widen the bucket range.
|
||||
buckets: [0.1, 0.5, 1, 2, 5, 10, 30, 60, 120, 300],
|
||||
registers: [registry],
|
||||
});
|
||||
|
||||
const toolCallsTotal = new Counter({
|
||||
name: `${prefix}_tool_calls_total`,
|
||||
help: 'Tool invocations executed by the agent loop.',
|
||||
labelNames: ['tool_name', 'success'] as const,
|
||||
registers: [registry],
|
||||
});
|
||||
|
||||
return {
|
||||
jobsTotal,
|
||||
activeJobs,
|
||||
jobDurationSeconds,
|
||||
llmCallsTotal,
|
||||
llmCallDurationSeconds,
|
||||
toolCallsTotal,
|
||||
};
|
||||
}
|
||||
|
||||
/** Closed enum of job terminal statuses for the jobsTotal counter. */
|
||||
export type WorkerJobStatus =
|
||||
| 'succeeded'
|
||||
| 'failed'
|
||||
| 'aborted'
|
||||
| 'cancelled'
|
||||
| 'waiting_human'
|
||||
| 'error';
|
||||
Reference in New Issue
Block a user