feat(errors): mcpd error-log ring buffer + 'mcpctl errors'

Operators can now see recent mcpd error/fatal logs without kubectl:
- mcpd tees level>=error pino records into an in-memory ring buffer
  (src/mcpd/src/services/error-log-buffer.ts; wired via pino.multistream in
  server.ts so stdout logging is unchanged). Captures structured errors incl.
  fatal kinds like BACKEND_TOKEN_DEAD.
- GET /api/v1/logs/errors?limit=N (RBAC: 'logs' operation).
- CLI: 'mcpctl errors [-n N]' renders TIME/LEVEL/DETAIL, most-recent-first.

Buffer unit tests (6); full suite 2223 passing. Needs a deploy to go live.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

src/cli/src/commands/errors.ts

@@ -0,0 +1,59 @@
+import { Command } from 'commander';
+import type { ApiClient } from '../api-client.js';
+
+export interface ErrorsCommandDeps {
+  client: ApiClient;
+  log: (...args: string[]) => void;
+}
+
+interface ErrorEntry {
+  time: number;
+  level: number;
+  msg?: string;
+  kind?: string;
+  reqId?: string;
+  err?: string;
+}
+
+/** pino numeric level → short label. */
+function levelLabel(level: number): string {
+  if (level >= 60) return 'FATAL';
+  if (level >= 50) return 'ERROR';
+  return String(level);
+}
+
+function fmtTime(ms: number): string {
+  if (!ms) return '-';
+  // Local HH:MM:SS — enough to correlate with "it broke just now".
+  const d = new Date(ms);
+  const p = (n: number): string => String(n).padStart(2, '0');
+  return `${p(d.getHours())}:${p(d.getMinutes())}:${p(d.getSeconds())}`;
+}
+
+export function createErrorsCommand(deps?: Partial<ErrorsCommandDeps>): Command {
+  const log = deps?.log ?? ((...args: string[]): void => { console.log(...args); });
+
+  return new Command('errors')
+    .description('Show recent mcpd error/fatal log entries')
+    .option('-n, --limit <count>', 'max entries to show (default 50)', '50')
+    .action(async (opts: { limit: string }) => {
+      const client = deps?.client;
+      if (!client) throw new Error('errors: no API client configured');
+
+      const limit = Number(opts.limit) > 0 ? Number(opts.limit) : 50;
+      const res = await client.get<{ errors: ErrorEntry[] }>(`/api/v1/logs/errors?limit=${limit}`);
+      const errors = res.errors ?? [];
+
+      if (errors.length === 0) {
+        log('No recent mcpd errors. 🎉');
+        return;
+      }
+
+      log(`TIME      LEVEL  DETAIL`);
+      for (const e of errors) {
+        const detail = [e.kind, e.msg ?? e.err].filter(Boolean).join(' — ') || '(no message)';
+        log(`${fmtTime(e.time).padEnd(9)} ${levelLabel(e.level).padEnd(6)} ${detail}`);
+      }
+      log(`\n${errors.length} entr${errors.length === 1 ? 'y' : 'ies'} (most recent first; in-memory, cleared on mcpd restart)`);
+    });
+}

src/cli/src/index.ts

@@ -26,6 +26,7 @@ import { createRotateCommand } from './commands/rotate.js';
 import { createReviewCommand } from './commands/review.js';
 import { createSkillsCommand } from './commands/skills.js';
 import { createPasswdCommand } from './commands/passwd.js';
+import { createErrorsCommand } from './commands/errors.js';
 import { ApiClient, ApiError } from './api-client.js';
 import { loadConfig } from './config/index.js';
 import { loadCredentials } from './auth/index.js';
@@ -263,6 +264,11 @@ export function createProgram(): Command {
     log: (...args) => console.log(...args),
   }));
 
+  program.addCommand(createErrorsCommand({
+    client,
+    log: (...args) => console.log(...args),
+  }));
+
   program.addCommand(createBackupCommand({
     client,
     log: (...args) => console.log(...args),

src/mcpd/package.json

@@ -27,6 +27,7 @@
     "dockerode": "^4.0.9",
     "fastify": "^5.0.0",
     "js-yaml": "^4.1.0",
+    "pino": "^10.3.1",
     "zod": "^3.24.0"
   },
   "devDependencies": {

src/mcpd/src/main.ts

@@ -55,6 +55,8 @@ import { registerPersonalityRoutes } from './routes/personalities.js';
 import { registerWebUi } from './routes/web-ui.js';
 import { bootstrapSystemProject } from './bootstrap/system-project.js';
 import { SET_OWN_PASSWORD_OPERATION } from './bootstrap/self-password-permission.js';
+import { registerLogRoutes } from './routes/logs.js';
+import { errorLogBuffer } from './services/error-log-buffer.js';
 import { bootstrapSystemSkills } from './bootstrap/system-skills.js';
 import {
   McpServerService,
@@ -131,6 +133,11 @@ function mapUrlToPermission(method: string, url: string): PermissionCheck {
 
   const segment = match[1] as string;
 
+  // Recent mcpd error logs — read-only operator view, gated by `logs`.
+  if (url.startsWith('/api/v1/logs')) {
+    return { kind: 'operation', operation: 'logs' };
+  }
+
   // Self-service password change — gated by the `set-own-password` operation
   // (a default, admin-revocable permission), NOT the broad edit:users that an
   // admin reset of another user needs. Must precede the generic users mapping.
@@ -716,6 +723,7 @@ async function main(): Promise<void> {
   });
   registerRbacRoutes(app, rbacDefinitionService);
   registerUserRoutes(app, { userService, rbacDefinitionService, prisma });
+  registerLogRoutes(app, errorLogBuffer);
   registerGroupRoutes(app, groupService);
   registerMcpTokenRoutes(app, { tokenService: mcpTokenService, projectRepo });
   registerPromptRoutes(app, promptService, projectRepo, agentRepo);

src/mcpd/src/routes/logs.ts

@@ -0,0 +1,14 @@
+import type { FastifyInstance } from 'fastify';
+import type { ErrorLogBuffer } from '../services/error-log-buffer.js';
+
+/**
+ * GET /api/v1/logs/errors?limit=N — recent mcpd error/fatal log records from
+ * the in-memory ring buffer. RBAC: the `logs` operation (see mapUrlToPermission).
+ */
+export function registerLogRoutes(app: FastifyInstance, buffer: ErrorLogBuffer): void {
+  app.get<{ Querystring: { limit?: string } }>('/api/v1/logs/errors', async (request) => {
+    const raw = Number(request.query.limit);
+    const limit = Number.isFinite(raw) && raw > 0 ? Math.min(raw, 500) : 50;
+    return { errors: buffer.recent(limit) };
+  });
+}

src/mcpd/src/server.ts

@@ -1,5 +1,6 @@
 import Fastify from 'fastify';
 import type { FastifyInstance } from 'fastify';
+import pino from 'pino';
 import type { McpdConfig } from './config/index.js';
 import { registerSecurityPlugins } from './middleware/security.js';
 import { errorHandler } from './middleware/error-handler.js';
@@ -7,6 +8,7 @@ import { registerHealthRoutes } from './routes/health.js';
 import type { HealthDeps } from './routes/health.js';
 import type { AuthDeps } from './middleware/auth.js';
 import type { AuditDeps } from './middleware/audit.js';
+import { errorLogBuffer } from './services/error-log-buffer.js';
 
 export interface ServerDeps {
   health: HealthDeps;
@@ -15,9 +17,17 @@ export interface ServerDeps {
 }
 
 export async function createServer(config: McpdConfig, deps: ServerDeps): Promise<FastifyInstance> {
+  // Tee error/fatal records into an in-memory ring buffer (for `mcpctl errors`)
+  // while keeping normal stdout logging intact. Passing a `stream` (vs a full
+  // pino instance) keeps Fastify's logger typing stable. multistream routes
+  // each record to every stream whose level <= the record's level.
   const app = Fastify({
     logger: {
       level: config.logLevel,
+      stream: pino.multistream([
+        { stream: process.stdout },
+        { level: 'error', stream: errorLogBuffer.stream() },
+      ]),
     },
   });
 

src/mcpd/src/services/error-log-buffer.ts

@@ -0,0 +1,75 @@
+import { Writable } from 'node:stream';
+
+/**
+ * In-memory ring buffer of recent error/fatal log records, so operators can
+ * see "what went wrong with mcpd" via `mcpctl errors` without shelling into
+ * the pod. Fed by a pino multistream (see server.ts) that tees level>=error
+ * records here; the rest of logging is unaffected.
+ *
+ * Deliberately tiny + dependency-free: parse the serialized JSON line, keep
+ * the fields worth showing, cap the buffer. Lost on restart (errors that
+ * outlive a restart are someone else's problem — the cause usually recurs).
+ */
+
+/** pino numeric levels: warn=40, error=50, fatal=60. */
+const ERROR_LEVEL = 50;
+
+export interface ErrorLogEntry {
+  time: number;
+  level: number;
+  msg?: string;
+  /** Structured error kind, e.g. BACKEND_TOKEN_DEAD. */
+  kind?: string;
+  reqId?: string;
+  /** Flattened error message/type when the record carried an `err` object. */
+  err?: string;
+}
+
+export class ErrorLogBuffer {
+  private buf: ErrorLogEntry[] = [];
+  constructor(private readonly capacity = 200) {}
+
+  /** Parse one serialized pino line; keep it only if level>=error. */
+  recordLine(line: string): void {
+    const trimmed = line.trim();
+    if (trimmed === '') return;
+    let o: Record<string, unknown>;
+    try {
+      o = JSON.parse(trimmed) as Record<string, unknown>;
+    } catch {
+      return; // non-JSON (e.g. a raw console line) — skip
+    }
+    const level = typeof o['level'] === 'number' ? (o['level'] as number) : 0;
+    if (level < ERROR_LEVEL) return;
+    const errObj = o['err'] as { message?: string; type?: string } | undefined;
+    const entry: ErrorLogEntry = {
+      time: typeof o['time'] === 'number' ? (o['time'] as number) : 0,
+      level,
+    };
+    if (typeof o['msg'] === 'string') entry.msg = o['msg'] as string;
+    if (typeof o['kind'] === 'string') entry.kind = o['kind'] as string;
+    if (typeof o['reqId'] === 'string') entry.reqId = o['reqId'] as string;
+    const errMsg = errObj?.message ?? errObj?.type;
+    if (errMsg !== undefined) entry.err = errMsg;
+    this.buf.push(entry);
+    if (this.buf.length > this.capacity) this.buf.shift();
+  }
+
+  /** Most recent first, capped at `limit`. */
+  recent(limit = 50): ErrorLogEntry[] {
+    return this.buf.slice(-limit).reverse();
+  }
+
+  /** A Writable for pino.multistream — receives serialized JSON log lines. */
+  stream(): Writable {
+    return new Writable({
+      write: (chunk: Buffer | string, _enc, cb) => {
+        for (const line of chunk.toString('utf-8').split('\n')) this.recordLine(line);
+        cb();
+      },
+    });
+  }
+}
+
+/** Process-wide singleton fed by the logger and read by the /logs/errors route. */
+export const errorLogBuffer = new ErrorLogBuffer();

src/mcpd/tests/error-log-buffer.test.ts

@@ -0,0 +1,54 @@
+import { describe, it, expect } from 'vitest';
+import { ErrorLogBuffer } from '../src/services/error-log-buffer.js';
+
+const line = (o: Record<string, unknown>): string => JSON.stringify(o) + '\n';
+
+describe('ErrorLogBuffer', () => {
+  it('keeps error/fatal records and drops info/warn', () => {
+    const b = new ErrorLogBuffer();
+    b.recordLine(line({ level: 30, time: 1, msg: 'info' }));
+    b.recordLine(line({ level: 40, time: 2, msg: 'warn' }));
+    b.recordLine(line({ level: 50, time: 3, msg: 'an error' }));
+    b.recordLine(line({ level: 60, time: 4, msg: 'fatal', kind: 'BACKEND_TOKEN_DEAD' }));
+    const recent = b.recent();
+    expect(recent.map((e) => e.msg)).toEqual(['fatal', 'an error']); // most-recent first
+    expect(recent[0]?.kind).toBe('BACKEND_TOKEN_DEAD');
+  });
+
+  it('flattens an err object to its message', () => {
+    const b = new ErrorLogBuffer();
+    b.recordLine(line({ level: 50, time: 1, msg: 'boom', err: { type: 'Error', message: 'OpenBao write … 403' } }));
+    expect(b.recent()[0]?.err).toBe('OpenBao write … 403');
+  });
+
+  it('ignores blank and non-JSON lines', () => {
+    const b = new ErrorLogBuffer();
+    b.recordLine('');
+    b.recordLine('not json at all');
+    b.recordLine('   ');
+    expect(b.recent()).toHaveLength(0);
+  });
+
+  it('caps at capacity (ring buffer)', () => {
+    const b = new ErrorLogBuffer(3);
+    for (let i = 1; i <= 5; i++) b.recordLine(line({ level: 50, time: i, msg: `e${i}` }));
+    const recent = b.recent(10);
+    expect(recent.map((e) => e.msg)).toEqual(['e5', 'e4', 'e3']); // oldest two evicted
+  });
+
+  it('respects the limit argument', () => {
+    const b = new ErrorLogBuffer();
+    for (let i = 1; i <= 10; i++) b.recordLine(line({ level: 50, time: i }));
+    expect(b.recent(2)).toHaveLength(2);
+  });
+
+  it('handles multiple records in one chunk via the stream', (ctx) => new Promise<void>((resolve) => {
+    const b = new ErrorLogBuffer();
+    const s = b.stream();
+    s.write(line({ level: 50, time: 1, msg: 'a' }) + line({ level: 30, time: 2 }) + line({ level: 60, time: 3, msg: 'b' }));
+    s.end(() => {
+      expect(b.recent().map((e) => e.msg)).toEqual(['b', 'a']);
+      resolve();
+    });
+  }));
+});