Merge pull request 'feat(secrets): survive OpenBao outages and report real backend health' (#115) from feat/openbao-resilience into main

src/mcpd/tests/secret-backend-health-route.test.ts

@@ -0,0 +1,98 @@
+import { describe, it, expect, vi, afterEach } from 'vitest';
+import Fastify from 'fastify';
+import type { FastifyInstance } from 'fastify';
+import type { SecretBackend } from '@prisma/client';
+import { registerSecretBackendHealthRoutes } from '../src/routes/secret-backend-health.js';
+import { SecretBackendService } from '../src/services/secret-backend.service.js';
+import type { ISecretBackendRepository } from '../src/repositories/secret-backend.repository.js';
+import type { SecretBackendDriver } from '../src/services/secret-backends/types.js';
+
+let app: FastifyInstance;
+afterEach(async () => { await app?.close(); });
+
+function backendRow(overrides: Partial<SecretBackend> = {}): SecretBackend {
+  return {
+    id: 'b1', name: 'bao-k8s', type: 'openbao',
+    config: { url: 'http://bao.example:8200', auth: 'kubernetes', role: 'mcpctl' },
+    isDefault: true, description: '', version: 1,
+    createdAt: new Date(), updatedAt: new Date(),
+    ...overrides,
+  } as SecretBackend;
+}
+
+/**
+ * Build the route over a service whose driver is stubbed. We override
+ * `driverFor` rather than the factory so the test drives the two probes
+ * directly — the point here is the route's reporting, not driver internals.
+ */
+async function buildApp(
+  probes: Pick<SecretBackendDriver, 'healthCheck' | 'authCheck'>,
+  row: SecretBackend = backendRow(),
+): Promise<FastifyInstance> {
+  const repo = {
+    findById: vi.fn(async (id: string) => (id === row.id ? row : null)),
+  } as unknown as ISecretBackendRepository;
+  const svc = new SecretBackendService(repo, {
+    plaintext: { listAllPlaintext: async () => [] },
+    secretRefResolver: { resolve: async () => 'tok' },
+  });
+  vi.spyOn(svc, 'driverFor').mockReturnValue({ kind: 'openbao', ...probes } as SecretBackendDriver);
+  vi.spyOn(svc, 'cacheStatsFor').mockReturnValue({ entries: 3, servingStale: 0, oldestStaleSince: undefined });
+
+  const a = Fastify();
+  registerSecretBackendHealthRoutes(a, svc);
+  await a.ready();
+  return a;
+}
+
+describe('GET /api/v1/secretbackends/:id/health', () => {
+  it('reports live+ready when the backend is fully working', async () => {
+    app = await buildApp({
+      healthCheck: async () => ({ ok: true, detail: 'active' }),
+      authCheck: async () => ({ ok: true, detail: 'readable at secret/mcpctl' }),
+    });
+    const res = await app.inject({ method: 'GET', url: '/api/v1/secretbackends/b1/health' });
+    expect(res.statusCode).toBe(200);
+    expect(res.json()).toMatchObject({ backend: 'bao-k8s', live: true, ready: true });
+  });
+
+  it('reports NOT live when OpenBao is sealed — regardless of rotation state', async () => {
+    // The bug this endpoint exists for: a kubernetes-auth backend never writes
+    // tokenMeta.lastRotationError, so the old status line stayed green here.
+    app = await buildApp({
+      healthCheck: async () => ({ ok: false, detail: 'sealed' }),
+      authCheck: async () => ({ ok: true, detail: 'should not be consulted' }),
+    });
+    const body = (await app.inject({ method: 'GET', url: '/api/v1/secretbackends/b1/health' })).json();
+    expect(body.live).toBe(false);
+    expect(body.liveDetail).toBe('sealed');
+    expect(body.ready).toBe(false);
+    expect(body.readyDetail).toMatch(/not probed/);
+    expect(body.rotation.lastRotationError).toBeNull();
+  });
+
+  it('distinguishes reachable-but-unusable (revoked grants) from unreachable', async () => {
+    app = await buildApp({
+      healthCheck: async () => ({ ok: true, detail: 'active' }),
+      authCheck: async () => ({ ok: false, detail: 'OpenBao list: HTTP 403 permission denied' }),
+    });
+    const body = (await app.inject({ method: 'GET', url: '/api/v1/secretbackends/b1/health' })).json();
+    expect(body).toMatchObject({ live: true, ready: false });
+    expect(body.readyDetail).toMatch(/403/);
+  });
+
+  it('surfaces cache state so degraded serving is visible', async () => {
+    app = await buildApp({
+      healthCheck: async () => ({ ok: true }),
+      authCheck: async () => ({ ok: true }),
+    });
+    const body = (await app.inject({ method: 'GET', url: '/api/v1/secretbackends/b1/health' })).json();
+    expect(body.cache).toMatchObject({ entries: 3, servingStale: 0 });
+  });
+
+  it('404s for an unknown backend', async () => {
+    app = await buildApp({ healthCheck: async () => ({ ok: true }), authCheck: async () => ({ ok: true }) });
+    const res = await app.inject({ method: 'GET', url: '/api/v1/secretbackends/nope/health' });
+    expect(res.statusCode).toBe(404);
+  });
+});

src/mcpd/tests/secret-backend-rotator-loop.test.ts

@@ -0,0 +1,155 @@
+/**
+ * SecretBackendRotatorLoop had no coverage at all, despite being the boot-time
+ * detector added after an upstream OpenBao re-init silently broke every secret
+ * write for four days (e51b924). These pin the behaviours that matter when that
+ * recurs: the boot health check fires, it reports through the injected logger
+ * (so `mcpctl errors` sees it), and stop() genuinely stops.
+ */
+import { describe, it, expect, vi } from 'vitest';
+import type { SecretBackend } from '@prisma/client';
+import { SecretBackendRotatorLoop } from '../src/services/secret-backend-rotator-loop.js';
+import type { SecretBackendService } from '../src/services/secret-backend.service.js';
+import type { SecretBackendRotator } from '../src/services/secret-backend-rotator.service.js';
+
+function backend(overrides: Partial<SecretBackend> = {}): SecretBackend {
+  return {
+    id: 'b1', name: 'bao', type: 'openbao',
+    config: { url: 'http://bao.example:8200', rotation: { enabled: true, tokenRole: 'r', intervalHours: 24 } },
+    isDefault: true, description: '', version: 1,
+    createdAt: new Date(), updatedAt: new Date(),
+    ...overrides,
+  } as SecretBackend;
+}
+
+interface Harness {
+  loop: SecretBackendRotatorLoop;
+  rotator: { isRotatable: ReturnType<typeof vi.fn>; isOverdue: ReturnType<typeof vi.fn>; healthCheck: ReturnType<typeof vi.fn>; rotateOne: ReturnType<typeof vi.fn> };
+  logs: { info: string[]; warn: string[]; error: Array<{ obj: Record<string, unknown>; msg: string }> };
+  timers: Array<{ cb: () => void; ms: number }>;
+  cleared: number;
+}
+
+function harness(opts: {
+  rows?: SecretBackend[];
+  rotatable?: boolean;
+  overdue?: boolean;
+  health?: { ok: boolean; message?: string } | Error;
+} = {}): Harness {
+  const rows = opts.rows ?? [backend()];
+  const logs: Harness['logs'] = { info: [], warn: [], error: [] };
+  const timers: Harness['timers'] = [];
+  const state = { cleared: 0 };
+
+  const rotator = {
+    isRotatable: vi.fn(() => opts.rotatable ?? true),
+    isOverdue: vi.fn(() => opts.overdue ?? false),
+    healthCheck: vi.fn(async () => {
+      if (opts.health instanceof Error) throw opts.health;
+      return opts.health ?? { ok: true };
+    }),
+    rotateOne: vi.fn(async () => ({})),
+  };
+
+  const loop = new SecretBackendRotatorLoop({
+    backends: {
+      list: async () => rows,
+      getById: async (id: string) => rows.find((r) => r.id === id) ?? rows[0]!,
+    } as unknown as SecretBackendService,
+    rotator: rotator as unknown as SecretBackendRotator,
+    setTimeout: ((cb: () => void, ms: number) => { timers.push({ cb, ms }); return { id: timers.length } as unknown as NodeJS.Timeout; }),
+    clearTimeout: (() => { state.cleared++; }),
+    log: {
+      info: (m) => { logs.info.push(m); },
+      warn: (m) => { logs.warn.push(m); },
+      error: (obj, msg) => { logs.error.push({ obj, msg }); },
+    },
+  });
+
+  return { loop, rotator, logs, timers, get cleared() { return state.cleared; } } as Harness;
+}
+
+/** The boot health check is fire-and-forget; let its microtasks settle. */
+const settle = async (): Promise<void> => { await Promise.resolve(); await Promise.resolve(); await Promise.resolve(); };
+
+describe('SecretBackendRotatorLoop', () => {
+  it('stays idle when nothing is rotatable', async () => {
+    const h = harness({ rotatable: false });
+    await h.loop.start();
+    expect(h.logs.info.join(' ')).toMatch(/no rotatable backends/);
+    expect(h.timers).toHaveLength(0);
+    expect(h.rotator.healthCheck).not.toHaveBeenCalled();
+  });
+
+  it('runs a boot-time health check for every rotatable backend', async () => {
+    const h = harness({ rows: [backend(), backend({ id: 'b2', name: 'bao2' })] });
+    await h.loop.start();
+    await settle();
+    expect(h.rotator.healthCheck).toHaveBeenCalledTimes(2);
+  });
+
+  it('emits BACKEND_TOKEN_DEAD through the logger, not console', async () => {
+    // The regression that made `mcpctl errors` blind to it: this used to be a
+    // bare console.error, which bypasses the pino stream feeding ErrorLogBuffer.
+    const h = harness({ health: { ok: false, message: 'token rejected' } });
+    await h.loop.start();
+    await settle();
+    expect(h.logs.error).toHaveLength(1);
+    expect(h.logs.error[0]?.obj).toMatchObject({ kind: 'BACKEND_TOKEN_DEAD', backend: 'bao' });
+    expect(h.logs.error[0]?.msg).toBe('token rejected');
+  });
+
+  it('does not log a fatal when the backend is healthy', async () => {
+    const h = harness({ health: { ok: true } });
+    await h.loop.start();
+    await settle();
+    expect(h.logs.error).toHaveLength(0);
+  });
+
+  it('survives a health check that throws', async () => {
+    const h = harness({ health: new Error('network down') });
+    await expect(h.loop.start()).resolves.toBeUndefined();
+    await settle();
+    expect(h.logs.warn.join(' ')).toMatch(/health check threw: network down/);
+  });
+
+  it('rotates immediately when a backend is overdue, and still schedules', async () => {
+    const h = harness({ overdue: true });
+    await h.loop.start();
+    await settle();
+    expect(h.rotator.rotateOne).toHaveBeenCalledWith('b1');
+    expect(h.timers).toHaveLength(1);
+  });
+
+  it('does not rotate on boot when not overdue', async () => {
+    const h = harness({ overdue: false });
+    await h.loop.start();
+    await settle();
+    expect(h.rotator.rotateOne).not.toHaveBeenCalled();
+    expect(h.timers).toHaveLength(1);
+  });
+
+  it('never schedules sooner than the 60s floor, even with adversarial jitter', async () => {
+    // intervalHours tiny + default jitter would otherwise produce a negative delay.
+    const rows = [backend({ config: { url: 'u', rotation: { enabled: true, tokenRole: 'r', intervalHours: 0.0001 } } } as Partial<SecretBackend>)];
+    for (let i = 0; i < 50; i++) {
+      const h = harness({ rows });
+      await h.loop.start();
+      expect(h.timers[0]?.ms).toBeGreaterThanOrEqual(60_000);
+    }
+  });
+
+  it('stop() clears timers and suppresses further scheduling', async () => {
+    const h = harness();
+    await h.loop.start();
+    expect(h.timers).toHaveLength(1);
+
+    h.loop.stop();
+    expect(h.cleared).toBeGreaterThan(0);
+
+    // The `stopped` guard has never been exercised: a firing timer must not
+    // reschedule after stop().
+    const before = h.timers.length;
+    await h.loop.rotateNow('b1').catch(() => undefined);
+    expect(h.timers).toHaveLength(before);
+  });
+});

src/mcpd/tests/secret-backends.test.ts

@@ -1,6 +1,7 @@
 import { describe, it, expect, vi } from 'vitest';
 import { PlaintextDriver } from '../src/services/secret-backends/plaintext.js';
 import { OpenBaoDriver } from '../src/services/secret-backends/openbao.js';
+import { SecretNotFoundError, SecretBackendUnavailableError } from '../src/services/secret-backends/types.js';
 
 describe('PlaintextDriver', () => {
   const driver = new PlaintextDriver({ listAllPlaintext: async () => [{ name: 'a', data: { k: 'v' } }] });
@@ -242,3 +243,91 @@ describe('OpenBaoDriver', () => {
     });
   });
 });
+
+describe('OpenBaoDriver: resilience', () => {
+  const resolver = { resolve: vi.fn(async () => 'test-vault-token') };
+  /** No real sleeping — otherwise the backoff tests take seconds. */
+  const noSleep = async (): Promise<void> => undefined;
+
+  function driverWith(fetchFn: ReturnType<typeof vi.fn>, opts: Record<string, unknown> = {}): OpenBaoDriver {
+    return new OpenBaoDriver(
+      { url: 'http://bao.example:8200', tokenSecretRef: { name: 'bao', key: 'token' } },
+      { fetch: fetchFn as unknown as typeof fetch, secretRefResolver: resolver, sleep: noSleep, ...opts },
+    );
+  }
+
+  it('maps a 404 read to SecretNotFoundError', async () => {
+    const fetchFn = vi.fn(async () => new Response('', { status: 404 }));
+    await expect(driverWith(fetchFn).read({ name: 'gone', externalRef: '', data: {} }))
+      .rejects.toThrow(SecretNotFoundError);
+  });
+
+  it('purges the token cache and retries once on 403 — outside the retry budget', async () => {
+    // This path existed but was never covered; it is the revocation/re-init case.
+    let n = 0;
+    const fetchFn = vi.fn(async () => {
+      n++;
+      if (n === 1) return new Response('', { status: 403 });
+      return new Response(JSON.stringify({ data: { data: { token: 'ok' } } }), { status: 200 });
+    });
+    const d = driverWith(fetchFn);
+    await expect(d.read({ name: 's', externalRef: '', data: {} })).resolves.toEqual({ token: 'ok' });
+    expect(fetchFn).toHaveBeenCalledTimes(2);
+  });
+
+  it('retries a 503 (sealed) and succeeds', async () => {
+    let n = 0;
+    const fetchFn = vi.fn(async () => {
+      n++;
+      if (n < 3) return new Response('', { status: 503 });
+      return new Response(JSON.stringify({ data: { data: { token: 'ok' } } }), { status: 200 });
+    });
+    await expect(driverWith(fetchFn).read({ name: 's', externalRef: '', data: {} }))
+      .resolves.toEqual({ token: 'ok' });
+    expect(fetchFn).toHaveBeenCalledTimes(3);
+  });
+
+  it('throws SecretBackendUnavailableError once the retry budget is exhausted', async () => {
+    const fetchFn = vi.fn(async () => new Response('', { status: 503 }));
+    await expect(driverWith(fetchFn, { maxAttempts: 3 }).read({ name: 's', externalRef: '', data: {} }))
+      .rejects.toThrow(SecretBackendUnavailableError);
+    expect(fetchFn).toHaveBeenCalledTimes(3);
+  });
+
+  it('classifies a network/abort failure as SecretBackendUnavailableError', async () => {
+    const fetchFn = vi.fn(async () => { throw new DOMException('timed out', 'TimeoutError'); });
+    await expect(driverWith(fetchFn, { maxAttempts: 2 }).read({ name: 's', externalRef: '', data: {} }))
+      .rejects.toThrow(SecretBackendUnavailableError);
+    expect(fetchFn).toHaveBeenCalledTimes(2);
+  });
+
+  it('passes an AbortSignal on every request', async () => {
+    const fetchFn = vi.fn(async () => new Response(JSON.stringify({ data: { data: {} } }), { status: 200 }));
+    await driverWith(fetchFn, { timeoutMs: 1234 }).read({ name: 's', externalRef: '', data: {} });
+    const [, init] = fetchFn.mock.calls[0] as [unknown, RequestInit];
+    expect(init.signal).toBeInstanceOf(AbortSignal);
+  });
+
+  it('healthCheck is unauthenticated and maps OpenBao status codes', async () => {
+    const cases: Array<[number, boolean, string]> = [
+      [200, true, 'active'],
+      [429, true, 'standby'],
+      [501, false, 'not initialized'],
+      [503, false, 'sealed'],
+    ];
+    for (const [status, ok, detail] of cases) {
+      const fetchFn = vi.fn(async () => new Response('', { status }));
+      const result = await driverWith(fetchFn).healthCheck();
+      expect(result).toEqual({ ok, detail });
+      // The whole point of the split: no token is minted for a liveness probe.
+      const [, init] = fetchFn.mock.calls[0] as [unknown, RequestInit];
+      expect((init.headers as Record<string, string>)['X-Vault-Token']).toBeUndefined();
+    }
+  });
+
+  it('authCheck reports false when the token can no longer list', async () => {
+    const fetchFn = vi.fn(async () => new Response('', { status: 403 }));
+    const result = await driverWith(fetchFn).authCheck();
+    expect(result.ok).toBe(false);
+  });
+});

src/mcpd/tests/secret-cache.test.ts

@@ -0,0 +1,198 @@
+import { describe, it, expect, vi } from 'vitest';
+import {
+  CachingSecretBackendDriver,
+  type CachingDriverLog,
+} from '../src/services/secret-backends/caching.js';
+import {
+  SecretNotFoundError,
+  SecretBackendUnavailableError,
+  type SecretBackendDriver,
+  type SecretData,
+} from '../src/services/secret-backends/types.js';
+
+/** Minimal fake backing driver whose read() behaviour the tests drive. */
+function makeInner(overrides: Partial<SecretBackendDriver> = {}): SecretBackendDriver & {
+  read: ReturnType<typeof vi.fn>;
+  write: ReturnType<typeof vi.fn>;
+  delete: ReturnType<typeof vi.fn>;
+} {
+  return {
+    kind: 'fake',
+    read: vi.fn(async () => ({ token: 'live' } as SecretData)),
+    write: vi.fn(async () => ({ externalRef: 'ref', storedData: {} as SecretData })),
+    delete: vi.fn(async () => undefined),
+    list: vi.fn(async () => []),
+    ...overrides,
+  } as never;
+}
+
+function makeLog(): CachingDriverLog & { warns: Array<Record<string, unknown>>; infos: Array<Record<string, unknown>> } {
+  const warns: Array<Record<string, unknown>> = [];
+  const infos: Array<Record<string, unknown>> = [];
+  return { warns, infos, warn: (o) => { warns.push(o); }, info: (o) => { infos.push(o); } };
+}
+
+const REQ = { name: 'gitea-creds', externalRef: 'secret/mcpctl/gitea-creds', data: {} };
+
+describe('CachingSecretBackendDriver', () => {
+  it('serves from cache within the TTL without touching the backend', async () => {
+    const inner = makeInner();
+    let now = 1_000;
+    const d = new CachingSecretBackendDriver(inner, { ttlMs: 5_000, now: () => now });
+
+    expect(await d.read(REQ)).toEqual({ token: 'live' });
+    now += 4_999;
+    expect(await d.read(REQ)).toEqual({ token: 'live' });
+
+    expect(inner.read).toHaveBeenCalledTimes(1);
+  });
+
+  it('refetches once the TTL has elapsed', async () => {
+    const inner = makeInner();
+    let now = 1_000;
+    const d = new CachingSecretBackendDriver(inner, { ttlMs: 5_000, now: () => now });
+
+    await d.read(REQ);
+    now += 5_001;
+    await d.read(REQ);
+
+    expect(inner.read).toHaveBeenCalledTimes(2);
+  });
+
+  it('serves the stale value when the backend is unavailable', async () => {
+    const inner = makeInner();
+    let now = 1_000;
+    const log = makeLog();
+    const d = new CachingSecretBackendDriver(inner, { ttlMs: 1_000, now: () => now, log, backendName: 'bao' });
+
+    await d.read(REQ);
+    inner.read.mockRejectedValue(new SecretBackendUnavailableError('bao down'));
+    now += 10_000;
+
+    // This is the whole point: no throw, so instance.service never marks ERROR.
+    expect(await d.read(REQ)).toEqual({ token: 'live' });
+    expect(log.warns[0]?.kind).toBe('BACKEND_UNREACHABLE');
+  });
+
+  it('logs BACKEND_UNREACHABLE only on the transition, not on every stale read', async () => {
+    const inner = makeInner();
+    let now = 1_000;
+    const log = makeLog();
+    const d = new CachingSecretBackendDriver(inner, { ttlMs: 1_000, now: () => now, log });
+
+    await d.read(REQ);
+    inner.read.mockRejectedValue(new SecretBackendUnavailableError('bao down'));
+    for (let i = 0; i < 5; i++) { now += 2_000; await d.read(REQ); }
+
+    expect(log.warns.filter((w) => w.kind === 'BACKEND_UNREACHABLE')).toHaveLength(1);
+  });
+
+  it('logs BACKEND_RECOVERED once the backend answers again', async () => {
+    const inner = makeInner();
+    let now = 1_000;
+    const log = makeLog();
+    const d = new CachingSecretBackendDriver(inner, { ttlMs: 1_000, now: () => now, log });
+
+    await d.read(REQ);
+    inner.read.mockRejectedValue(new SecretBackendUnavailableError('bao down'));
+    now += 2_000;
+    await d.read(REQ);
+
+    inner.read.mockResolvedValue({ token: 'rotated' });
+    now += 2_000;
+    expect(await d.read(REQ)).toEqual({ token: 'rotated' });
+    expect(log.infos.filter((i) => i.kind === 'BACKEND_RECOVERED')).toHaveLength(1);
+  });
+
+  it('NEVER serves stale for a deleted secret — evicts and rethrows', async () => {
+    // Regression guard. Serving stale here would resurrect a revoked
+    // credential, which is strictly worse than an outage.
+    const inner = makeInner();
+    let now = 1_000;
+    const d = new CachingSecretBackendDriver(inner, { ttlMs: 1_000, now: () => now });
+
+    await d.read(REQ);
+    inner.read.mockRejectedValue(new SecretNotFoundError('gone'));
+    now += 2_000;
+
+    await expect(d.read(REQ)).rejects.toThrow(SecretNotFoundError);
+    expect(d.stats().entries).toBe(0);
+
+    // And the entry really is gone — a later unavailable error has nothing to serve.
+    inner.read.mockRejectedValue(new SecretBackendUnavailableError('bao down'));
+    await expect(d.read(REQ)).rejects.toThrow(SecretBackendUnavailableError);
+  });
+
+  it('does not serve stale for a non-transport error (e.g. revoked grants)', async () => {
+    const inner = makeInner();
+    let now = 1_000;
+    const d = new CachingSecretBackendDriver(inner, { ttlMs: 1_000, now: () => now });
+
+    await d.read(REQ);
+    inner.read.mockRejectedValue(new Error('OpenBao read: HTTP 403 permission denied'));
+    now += 2_000;
+
+    await expect(d.read(REQ)).rejects.toThrow(/403/);
+  });
+
+  it('rethrows on a cold cache even when the backend is unavailable', async () => {
+    const inner = makeInner({ read: vi.fn(async () => { throw new SecretBackendUnavailableError('bao down'); }) as never });
+    const d = new CachingSecretBackendDriver(inner);
+
+    await expect(d.read(REQ)).rejects.toThrow(SecretBackendUnavailableError);
+  });
+
+  it('write() refreshes the cache so a read-after-write does not lag', async () => {
+    const inner = makeInner();
+    const d = new CachingSecretBackendDriver(inner, { ttlMs: 60_000 });
+
+    await d.read(REQ);
+    await d.write({ name: REQ.name, data: { token: 'brand-new' } });
+
+    expect(await d.read(REQ)).toEqual({ token: 'brand-new' });
+    expect(inner.read).toHaveBeenCalledTimes(1);
+  });
+
+  it('delete() evicts', async () => {
+    const inner = makeInner();
+    const d = new CachingSecretBackendDriver(inner, { ttlMs: 60_000 });
+
+    await d.read(REQ);
+    await d.delete({ name: REQ.name, externalRef: REQ.externalRef });
+
+    expect(d.stats().entries).toBe(0);
+  });
+
+  it('bounds the map with an LRU eviction', async () => {
+    const inner = makeInner();
+    const d = new CachingSecretBackendDriver(inner, { ttlMs: 60_000, maxEntries: 2 });
+
+    await d.read({ ...REQ, name: 'a' });
+    await d.read({ ...REQ, name: 'b' });
+    await d.read({ ...REQ, name: 'a' });   // 'a' becomes most-recently-used
+    await d.read({ ...REQ, name: 'c' });   // evicts 'b'
+
+    expect(d.stats().entries).toBe(2);
+    inner.read.mockClear();
+    await d.read({ ...REQ, name: 'a' });
+    expect(inner.read).not.toHaveBeenCalled();   // 'a' survived
+    await d.read({ ...REQ, name: 'b' });
+    expect(inner.read).toHaveBeenCalledTimes(1); // 'b' was evicted
+  });
+
+  it('reports stale count and age via stats()', async () => {
+    const inner = makeInner();
+    let now = 1_000;
+    const d = new CachingSecretBackendDriver(inner, { ttlMs: 1_000, now: () => now });
+
+    await d.read({ ...REQ, name: 'a' });
+    await d.read({ ...REQ, name: 'b' });
+    expect(d.stats()).toMatchObject({ entries: 2, servingStale: 0 });
+
+    inner.read.mockRejectedValue(new SecretBackendUnavailableError('down'));
+    now += 2_000;
+    await d.read({ ...REQ, name: 'a' });
+
+    expect(d.stats()).toMatchObject({ entries: 2, servingStale: 1, oldestStaleSince: 3_000 });
+  });
+});

src/mcpd/tests/warm-secret-cache.test.ts

@@ -0,0 +1,66 @@
+import { describe, it, expect, vi } from 'vitest';
+import { warmSecretCache } from '../src/bootstrap/warm-secret-cache.js';
+import type { PrismaClient } from '@prisma/client';
+import type { SecretService } from '../src/services/secret.service.js';
+
+function prismaWith(servers: Array<{ name: string; env: unknown }>): PrismaClient {
+  return { mcpServer: { findMany: vi.fn(async () => servers) } } as unknown as PrismaClient;
+}
+const noLog = { info: (): void => undefined, warn: (): void => undefined };
+
+const envRef = (name: string, secret: string, key: string): unknown =>
+  ({ name, valueFrom: { secretRef: { name: secret, key } } });
+
+describe('warmSecretCache', () => {
+  it('resolves every distinct secret ref exactly once', async () => {
+    const prisma = prismaWith([
+      { name: 'gitea', env: [envRef('GITEA_ACCESS_TOKEN', 'gitea-creds', 'GITEA_ACCESS_TOKEN')] },
+      // Two servers sharing one secret must not cost two reads.
+      { name: 'a', env: [envRef('T', 'shared', 'TOKEN')] },
+      { name: 'b', env: [envRef('T', 'shared', 'TOKEN')] },
+    ]);
+    const resolve = vi.fn(async () => 'value');
+    const result = await warmSecretCache(prisma, { resolve } as unknown as SecretService, noLog);
+
+    expect(resolve).toHaveBeenCalledTimes(2);
+    expect(result).toEqual({ warmed: 2, failed: 0 });
+  });
+
+  it('ignores inline env values', async () => {
+    const prisma = prismaWith([{ name: 's', env: [{ name: 'PLAIN', value: 'x' }] }]);
+    const resolve = vi.fn(async () => 'v');
+    expect(await warmSecretCache(prisma, { resolve } as unknown as SecretService, noLog))
+      .toEqual({ warmed: 0, failed: 0 });
+    expect(resolve).not.toHaveBeenCalled();
+  });
+
+  it('never throws when the backend is down — startup must not block', async () => {
+    const prisma = prismaWith([
+      { name: 'a', env: [envRef('T', 's1', 'K')] },
+      { name: 'b', env: [envRef('T', 's2', 'K')] },
+    ]);
+    const resolve = vi.fn(async () => { throw new Error('bao unreachable'); });
+    await expect(warmSecretCache(prisma, { resolve } as unknown as SecretService, noLog))
+      .resolves.toEqual({ warmed: 0, failed: 2 });
+  });
+
+  it('keeps going after one bad reference', async () => {
+    const prisma = prismaWith([
+      { name: 'a', env: [envRef('T', 'missing', 'K')] },
+      { name: 'b', env: [envRef('T', 'present', 'K')] },
+    ]);
+    const resolve = vi.fn(async (n: string) => {
+      if (n === 'missing') throw new Error('no such secret');
+      return 'v';
+    });
+    expect(await warmSecretCache(prisma, { resolve } as unknown as SecretService, noLog))
+      .toEqual({ warmed: 1, failed: 1 });
+  });
+
+  it('only considers servers with replicas > 0', async () => {
+    const prisma = prismaWith([]);
+    await warmSecretCache(prisma, { resolve: vi.fn() } as unknown as SecretService, noLog);
+    const findMany = (prisma.mcpServer.findMany as unknown as ReturnType<typeof vi.fn>);
+    expect(findMany.mock.calls[0]?.[0]).toMatchObject({ where: { replicas: { gt: 0 } } });
+  });
+});