diff --git a/completions/mcpctl.bash b/completions/mcpctl.bash index 9c42b73..0390dfb 100644 --- a/completions/mcpctl.bash +++ b/completions/mcpctl.bash @@ -5,7 +5,7 @@ _mcpctl() { local cur prev words cword _init_completion || return - local commands="status login logout config get describe delete logs create edit apply chat chat-llm patch passwd backup approve review skills console cache provider test migrate rotate" + local commands="status login logout config get describe delete logs create edit apply chat chat-llm patch passwd errors backup approve review skills console cache provider test migrate rotate" local project_commands="get describe delete logs create edit attach-server detach-server" local global_opts="-v --version --daemon-url --direct -p --project -h --help" local resources="servers instances secrets secretbackends llms agents personalities templates projects users groups rbac prompts promptrequests serverattachments proxymodels inference-tasks all" @@ -271,6 +271,9 @@ _mcpctl() { passwd) COMPREPLY=($(compgen -W "-h --help" -- "$cur")) return ;; + errors) + COMPREPLY=($(compgen -W "-n --limit -h --help" -- "$cur")) + return ;; backup) local backup_sub=$(_mcpctl_get_subcmd $subcmd_pos) if [[ -z "$backup_sub" ]]; then diff --git a/completions/mcpctl.fish b/completions/mcpctl.fish index e9a8582..24b2b71 100644 --- a/completions/mcpctl.fish +++ b/completions/mcpctl.fish @@ -4,7 +4,7 @@ # Erase any stale completions from previous versions complete -c mcpctl -e -set -l commands status login logout config get describe delete logs create edit apply chat chat-llm patch passwd backup approve review skills console cache provider test migrate rotate +set -l commands status login logout config get describe delete logs create edit apply chat chat-llm patch passwd errors backup approve review skills console cache provider test migrate rotate set -l project_commands get describe delete logs create edit attach-server detach-server # Disable file completions by default @@ -235,6 +235,7 @@ complete -c mcpctl -n "not __mcpctl_has_project; and not __fish_seen_subcommand_ complete -c mcpctl -n "not __mcpctl_has_project; and not __fish_seen_subcommand_from $commands" -a chat-llm -d 'Stateless chat with any registered LLM (public or virtual). No threads, no tools.' complete -c mcpctl -n "not __mcpctl_has_project; and not __fish_seen_subcommand_from $commands" -a patch -d 'Patch a resource field (e.g. mcpctl patch project myproj llmProvider=none)' complete -c mcpctl -n "not __mcpctl_has_project; and not __fish_seen_subcommand_from $commands" -a passwd -d 'Change a user password (your own when called without an argument)' +complete -c mcpctl -n "not __mcpctl_has_project; and not __fish_seen_subcommand_from $commands" -a errors -d 'Show recent mcpd error/fatal log entries' complete -c mcpctl -n "not __mcpctl_has_project; and not __fish_seen_subcommand_from $commands" -a backup -d 'Git-based backup status and management' complete -c mcpctl -n "not __mcpctl_has_project; and not __fish_seen_subcommand_from $commands" -a approve -d 'Approve a pending prompt request (atomic: delete request, create prompt)' complete -c mcpctl -n "not __mcpctl_has_project; and not __fish_seen_subcommand_from $commands" -a review -d 'Triage proposed prompts and skills' @@ -592,6 +593,9 @@ complete -c mcpctl -n "__fish_seen_subcommand_from chat-llm" -l max-tokens -d 'M complete -c mcpctl -n "__fish_seen_subcommand_from chat-llm" -l no-stream -d 'Disable SSE streaming (single JSON response)' complete -c mcpctl -n "__fish_seen_subcommand_from chat-llm" -l async -d 'Enqueue as a durable inference task and print the task id (does not wait for completion). Virtual Llms only. Poll with `mcpctl get task `.' +# errors options +complete -c mcpctl -n "__fish_seen_subcommand_from errors" -s n -l limit -d 'max entries to show (default 50)' -x + # console options complete -c mcpctl -n "__fish_seen_subcommand_from console" -l stdin-mcp -d 'Run inspector as MCP server over stdin/stdout (for Claude)' complete -c mcpctl -n "__fish_seen_subcommand_from console" -l audit -d 'Browse audit events from mcpd' diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ded41a7..68b6b79 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -142,6 +142,9 @@ importers: js-yaml: specifier: ^4.1.0 version: 4.1.1 + pino: + specifier: ^10.3.1 + version: 10.3.1 zod: specifier: ^3.24.0 version: 3.25.76 diff --git a/src/cli/src/commands/errors.ts b/src/cli/src/commands/errors.ts new file mode 100644 index 0000000..3b165ff --- /dev/null +++ b/src/cli/src/commands/errors.ts @@ -0,0 +1,59 @@ +import { Command } from 'commander'; +import type { ApiClient } from '../api-client.js'; + +export interface ErrorsCommandDeps { + client: ApiClient; + log: (...args: string[]) => void; +} + +interface ErrorEntry { + time: number; + level: number; + msg?: string; + kind?: string; + reqId?: string; + err?: string; +} + +/** pino numeric level → short label. */ +function levelLabel(level: number): string { + if (level >= 60) return 'FATAL'; + if (level >= 50) return 'ERROR'; + return String(level); +} + +function fmtTime(ms: number): string { + if (!ms) return '-'; + // Local HH:MM:SS — enough to correlate with "it broke just now". + const d = new Date(ms); + const p = (n: number): string => String(n).padStart(2, '0'); + return `${p(d.getHours())}:${p(d.getMinutes())}:${p(d.getSeconds())}`; +} + +export function createErrorsCommand(deps?: Partial): Command { + const log = deps?.log ?? ((...args: string[]): void => { console.log(...args); }); + + return new Command('errors') + .description('Show recent mcpd error/fatal log entries') + .option('-n, --limit ', 'max entries to show (default 50)', '50') + .action(async (opts: { limit: string }) => { + const client = deps?.client; + if (!client) throw new Error('errors: no API client configured'); + + const limit = Number(opts.limit) > 0 ? Number(opts.limit) : 50; + const res = await client.get<{ errors: ErrorEntry[] }>(`/api/v1/logs/errors?limit=${limit}`); + const errors = res.errors ?? []; + + if (errors.length === 0) { + log('No recent mcpd errors. 🎉'); + return; + } + + log(`TIME LEVEL DETAIL`); + for (const e of errors) { + const detail = [e.kind, e.msg ?? e.err].filter(Boolean).join(' — ') || '(no message)'; + log(`${fmtTime(e.time).padEnd(9)} ${levelLabel(e.level).padEnd(6)} ${detail}`); + } + log(`\n${errors.length} entr${errors.length === 1 ? 'y' : 'ies'} (most recent first; in-memory, cleared on mcpd restart)`); + }); +} diff --git a/src/cli/src/index.ts b/src/cli/src/index.ts index 892f46f..de70732 100644 --- a/src/cli/src/index.ts +++ b/src/cli/src/index.ts @@ -26,6 +26,7 @@ import { createRotateCommand } from './commands/rotate.js'; import { createReviewCommand } from './commands/review.js'; import { createSkillsCommand } from './commands/skills.js'; import { createPasswdCommand } from './commands/passwd.js'; +import { createErrorsCommand } from './commands/errors.js'; import { ApiClient, ApiError } from './api-client.js'; import { loadConfig } from './config/index.js'; import { loadCredentials } from './auth/index.js'; @@ -263,6 +264,11 @@ export function createProgram(): Command { log: (...args) => console.log(...args), })); + program.addCommand(createErrorsCommand({ + client, + log: (...args) => console.log(...args), + })); + program.addCommand(createBackupCommand({ client, log: (...args) => console.log(...args), diff --git a/src/mcpd/package.json b/src/mcpd/package.json index 48e570a..445d391 100644 --- a/src/mcpd/package.json +++ b/src/mcpd/package.json @@ -27,6 +27,7 @@ "dockerode": "^4.0.9", "fastify": "^5.0.0", "js-yaml": "^4.1.0", + "pino": "^10.3.1", "zod": "^3.24.0" }, "devDependencies": { diff --git a/src/mcpd/src/main.ts b/src/mcpd/src/main.ts index 5754df6..cc0acdd 100644 --- a/src/mcpd/src/main.ts +++ b/src/mcpd/src/main.ts @@ -55,6 +55,8 @@ import { registerPersonalityRoutes } from './routes/personalities.js'; import { registerWebUi } from './routes/web-ui.js'; import { bootstrapSystemProject } from './bootstrap/system-project.js'; import { SET_OWN_PASSWORD_OPERATION } from './bootstrap/self-password-permission.js'; +import { registerLogRoutes } from './routes/logs.js'; +import { errorLogBuffer } from './services/error-log-buffer.js'; import { bootstrapSystemSkills } from './bootstrap/system-skills.js'; import { McpServerService, @@ -131,6 +133,11 @@ function mapUrlToPermission(method: string, url: string): PermissionCheck { const segment = match[1] as string; + // Recent mcpd error logs — read-only operator view, gated by `logs`. + if (url.startsWith('/api/v1/logs')) { + return { kind: 'operation', operation: 'logs' }; + } + // Self-service password change — gated by the `set-own-password` operation // (a default, admin-revocable permission), NOT the broad edit:users that an // admin reset of another user needs. Must precede the generic users mapping. @@ -716,6 +723,7 @@ async function main(): Promise { }); registerRbacRoutes(app, rbacDefinitionService); registerUserRoutes(app, { userService, rbacDefinitionService, prisma }); + registerLogRoutes(app, errorLogBuffer); registerGroupRoutes(app, groupService); registerMcpTokenRoutes(app, { tokenService: mcpTokenService, projectRepo }); registerPromptRoutes(app, promptService, projectRepo, agentRepo); diff --git a/src/mcpd/src/routes/logs.ts b/src/mcpd/src/routes/logs.ts new file mode 100644 index 0000000..12db061 --- /dev/null +++ b/src/mcpd/src/routes/logs.ts @@ -0,0 +1,14 @@ +import type { FastifyInstance } from 'fastify'; +import type { ErrorLogBuffer } from '../services/error-log-buffer.js'; + +/** + * GET /api/v1/logs/errors?limit=N — recent mcpd error/fatal log records from + * the in-memory ring buffer. RBAC: the `logs` operation (see mapUrlToPermission). + */ +export function registerLogRoutes(app: FastifyInstance, buffer: ErrorLogBuffer): void { + app.get<{ Querystring: { limit?: string } }>('/api/v1/logs/errors', async (request) => { + const raw = Number(request.query.limit); + const limit = Number.isFinite(raw) && raw > 0 ? Math.min(raw, 500) : 50; + return { errors: buffer.recent(limit) }; + }); +} diff --git a/src/mcpd/src/server.ts b/src/mcpd/src/server.ts index e731488..167ebc6 100644 --- a/src/mcpd/src/server.ts +++ b/src/mcpd/src/server.ts @@ -1,5 +1,6 @@ import Fastify from 'fastify'; import type { FastifyInstance } from 'fastify'; +import pino from 'pino'; import type { McpdConfig } from './config/index.js'; import { registerSecurityPlugins } from './middleware/security.js'; import { errorHandler } from './middleware/error-handler.js'; @@ -7,6 +8,7 @@ import { registerHealthRoutes } from './routes/health.js'; import type { HealthDeps } from './routes/health.js'; import type { AuthDeps } from './middleware/auth.js'; import type { AuditDeps } from './middleware/audit.js'; +import { errorLogBuffer } from './services/error-log-buffer.js'; export interface ServerDeps { health: HealthDeps; @@ -15,9 +17,17 @@ export interface ServerDeps { } export async function createServer(config: McpdConfig, deps: ServerDeps): Promise { + // Tee error/fatal records into an in-memory ring buffer (for `mcpctl errors`) + // while keeping normal stdout logging intact. Passing a `stream` (vs a full + // pino instance) keeps Fastify's logger typing stable. multistream routes + // each record to every stream whose level <= the record's level. const app = Fastify({ logger: { level: config.logLevel, + stream: pino.multistream([ + { stream: process.stdout }, + { level: 'error', stream: errorLogBuffer.stream() }, + ]), }, }); diff --git a/src/mcpd/src/services/error-log-buffer.ts b/src/mcpd/src/services/error-log-buffer.ts new file mode 100644 index 0000000..19f5ec1 --- /dev/null +++ b/src/mcpd/src/services/error-log-buffer.ts @@ -0,0 +1,75 @@ +import { Writable } from 'node:stream'; + +/** + * In-memory ring buffer of recent error/fatal log records, so operators can + * see "what went wrong with mcpd" via `mcpctl errors` without shelling into + * the pod. Fed by a pino multistream (see server.ts) that tees level>=error + * records here; the rest of logging is unaffected. + * + * Deliberately tiny + dependency-free: parse the serialized JSON line, keep + * the fields worth showing, cap the buffer. Lost on restart (errors that + * outlive a restart are someone else's problem — the cause usually recurs). + */ + +/** pino numeric levels: warn=40, error=50, fatal=60. */ +const ERROR_LEVEL = 50; + +export interface ErrorLogEntry { + time: number; + level: number; + msg?: string; + /** Structured error kind, e.g. BACKEND_TOKEN_DEAD. */ + kind?: string; + reqId?: string; + /** Flattened error message/type when the record carried an `err` object. */ + err?: string; +} + +export class ErrorLogBuffer { + private buf: ErrorLogEntry[] = []; + constructor(private readonly capacity = 200) {} + + /** Parse one serialized pino line; keep it only if level>=error. */ + recordLine(line: string): void { + const trimmed = line.trim(); + if (trimmed === '') return; + let o: Record; + try { + o = JSON.parse(trimmed) as Record; + } catch { + return; // non-JSON (e.g. a raw console line) — skip + } + const level = typeof o['level'] === 'number' ? (o['level'] as number) : 0; + if (level < ERROR_LEVEL) return; + const errObj = o['err'] as { message?: string; type?: string } | undefined; + const entry: ErrorLogEntry = { + time: typeof o['time'] === 'number' ? (o['time'] as number) : 0, + level, + }; + if (typeof o['msg'] === 'string') entry.msg = o['msg'] as string; + if (typeof o['kind'] === 'string') entry.kind = o['kind'] as string; + if (typeof o['reqId'] === 'string') entry.reqId = o['reqId'] as string; + const errMsg = errObj?.message ?? errObj?.type; + if (errMsg !== undefined) entry.err = errMsg; + this.buf.push(entry); + if (this.buf.length > this.capacity) this.buf.shift(); + } + + /** Most recent first, capped at `limit`. */ + recent(limit = 50): ErrorLogEntry[] { + return this.buf.slice(-limit).reverse(); + } + + /** A Writable for pino.multistream — receives serialized JSON log lines. */ + stream(): Writable { + return new Writable({ + write: (chunk: Buffer | string, _enc, cb) => { + for (const line of chunk.toString('utf-8').split('\n')) this.recordLine(line); + cb(); + }, + }); + } +} + +/** Process-wide singleton fed by the logger and read by the /logs/errors route. */ +export const errorLogBuffer = new ErrorLogBuffer(); diff --git a/src/mcpd/tests/error-log-buffer.test.ts b/src/mcpd/tests/error-log-buffer.test.ts new file mode 100644 index 0000000..0280b7e --- /dev/null +++ b/src/mcpd/tests/error-log-buffer.test.ts @@ -0,0 +1,54 @@ +import { describe, it, expect } from 'vitest'; +import { ErrorLogBuffer } from '../src/services/error-log-buffer.js'; + +const line = (o: Record): string => JSON.stringify(o) + '\n'; + +describe('ErrorLogBuffer', () => { + it('keeps error/fatal records and drops info/warn', () => { + const b = new ErrorLogBuffer(); + b.recordLine(line({ level: 30, time: 1, msg: 'info' })); + b.recordLine(line({ level: 40, time: 2, msg: 'warn' })); + b.recordLine(line({ level: 50, time: 3, msg: 'an error' })); + b.recordLine(line({ level: 60, time: 4, msg: 'fatal', kind: 'BACKEND_TOKEN_DEAD' })); + const recent = b.recent(); + expect(recent.map((e) => e.msg)).toEqual(['fatal', 'an error']); // most-recent first + expect(recent[0]?.kind).toBe('BACKEND_TOKEN_DEAD'); + }); + + it('flattens an err object to its message', () => { + const b = new ErrorLogBuffer(); + b.recordLine(line({ level: 50, time: 1, msg: 'boom', err: { type: 'Error', message: 'OpenBao write … 403' } })); + expect(b.recent()[0]?.err).toBe('OpenBao write … 403'); + }); + + it('ignores blank and non-JSON lines', () => { + const b = new ErrorLogBuffer(); + b.recordLine(''); + b.recordLine('not json at all'); + b.recordLine(' '); + expect(b.recent()).toHaveLength(0); + }); + + it('caps at capacity (ring buffer)', () => { + const b = new ErrorLogBuffer(3); + for (let i = 1; i <= 5; i++) b.recordLine(line({ level: 50, time: i, msg: `e${i}` })); + const recent = b.recent(10); + expect(recent.map((e) => e.msg)).toEqual(['e5', 'e4', 'e3']); // oldest two evicted + }); + + it('respects the limit argument', () => { + const b = new ErrorLogBuffer(); + for (let i = 1; i <= 10; i++) b.recordLine(line({ level: 50, time: i })); + expect(b.recent(2)).toHaveLength(2); + }); + + it('handles multiple records in one chunk via the stream', (ctx) => new Promise((resolve) => { + const b = new ErrorLogBuffer(); + const s = b.stream(); + s.write(line({ level: 50, time: 1, msg: 'a' }) + line({ level: 30, time: 2 }) + line({ level: 60, time: 3, msg: 'b' })); + s.end(() => { + expect(b.recent().map((e) => e.msg)).toEqual(['b', 'a']); + resolve(); + }); + })); +});