feat: HTTP-mode mcplocal container + mcpctl test mcp + token-auth preHandler

Delivers the final piece of the mcptoken stack: a containerized,
network-accessible mcplocal that serves Streamable-HTTP MCP to off-host
clients (the vLLM use case), authenticated by project-scoped McpTokens.

New binary (same package, new entry):
  - src/mcplocal/src/serve.ts — HTTP-only entry. Reads MCPLOCAL_MCPD_URL,
    MCPLOCAL_MCPD_TOKEN, MCPLOCAL_HTTP_HOST/PORT, MCPLOCAL_CACHE_DIR from
    env. No StdioProxyServer, no --upstream.
  - src/mcplocal/src/http/token-auth.ts — Fastify preHandler that
    validates mcpctl_pat_ bearers via mcpd's /api/v1/mcptokens/introspect.
    30s positive / 5s negative TTL. Rejects wrong-project with 403.

Shared HTTP MCP client:
  - src/shared/src/mcp-http/ — reusable McpHttpSession with initialize,
    listTools, callTool, close. Handles http+https, SSE, id correlation,
    distinct McpProtocolError / McpTransportError. Plus mcpHealthCheck
    and deriveBaseUrl helpers.

New CLI verb `mcpctl test mcp <url>`:
  - Flags: --token (also $MCPCTL_TOKEN), --tool, --args (JSON),
    --expect-tools, --timeout, -o text|json, --no-health.
  - Exit codes: 0 PASS, 1 TRANSPORT/AUTH FAIL, 2 CONTRACT FAIL.

Container + deploy:
  - deploy/Dockerfile.mcplocal (Node 20 alpine, multi-stage, pnpm
    workspace, CMD node src/mcplocal/dist/serve.js, VOLUME
    /var/lib/mcplocal/cache, HEALTHCHECK on :3200/healthz).
  - scripts/build-mcplocal.sh mirrors build-mcpd.sh.
  - fulldeploy.sh is now a 4-step pipeline that also builds + rolls out
    mcplocal (gated on `kubectl get deployment/mcplocal` so the script
    stays green before the Pulumi stack lands).

Audit + cache:
  - project-mcp-endpoint.ts passes MCPLOCAL_CACHE_DIR into FileCache at
    both construction sites and, when request.mcpToken is present, calls
    collector.setSessionMcpToken(id, ...) so audit events carry the
    tokenName/tokenSha.

Tests:
  - 9 unit cases on `mcpctl test mcp` (happy path, health miss,
    expect-tools hit/miss, transport throw, tool isError, json report,
    $MCPCTL_TOKEN env fallback, invalid --args).
  - Smoke test src/mcplocal/tests/smoke/mcptoken.smoke.test.ts —
    gated on healthz($MCPGW_URL), skipped cleanly when unreachable.
    Covers happy path, wrong-project 403, --expect-tools contract
    failure, and revocation 401 within the negative-cache window.

1773/1773 workspace tests pass. Pulumi resources (Deployment, Service,
Ingress, PVC, Secret, NetworkPolicy) still need to land in
../kubernetes-deployment before the smoke gate flips on.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

src/mcplocal/package.json

@@ -10,6 +10,7 @@
     "clean": "rimraf dist",
     "dev": "tsx watch src/index.ts",
     "start": "node dist/index.js",
+    "serve": "node dist/serve.js",
     "test": "vitest",
     "test:run": "vitest run",
     "test:smoke": "vitest run --config vitest.smoke.config.ts"

src/mcplocal/src/http/project-mcp-endpoint.ts

@@ -97,7 +97,8 @@ export function registerProjectMcpEndpoint(app: FastifyInstance, mcpdClient: Mcp
       ?? effectiveRegistry?.getActiveName()
       ?? 'none';
     const llmModel = resolvedModel ?? 'default';
-    const cache = new FileCache(`${llmProvider}--${llmModel}--${proxyModelName}`);
+    const cacheConfig = process.env.MCPLOCAL_CACHE_DIR ? { dir: process.env.MCPLOCAL_CACHE_DIR } : undefined;
+    const cache = new FileCache(`${llmProvider}--${llmModel}--${proxyModelName}`, cacheConfig);
     router.setProxyModel(proxyModelName, llmAdapter, cache);
 
     // Per-server proxymodel overrides (if mcpd provides them)
@@ -200,6 +201,17 @@ export function registerProjectMcpEndpoint(app: FastifyInstance, mcpdClient: Mcp
           void ensureUserName().then((name) => {
             if (name) collector.setSessionUserName(id, name);
           });
+
+          // HTTP-mode mcplocal: if the token-auth preHandler attached an McpToken
+          // principal to the request, tag the session so audit events carry the
+          // tokenName/tokenSha alongside (or instead of) userName.
+          const principal = request.mcpToken;
+          if (principal) {
+            collector.setSessionMcpToken(id, {
+              tokenName: principal.tokenName,
+              tokenSha: principal.tokenSha,
+            });
+          }
         }
 
         // Audit: session_bind
@@ -388,7 +400,7 @@ export function registerProjectMcpEndpoint(app: FastifyInstance, mcpdClient: Mcp
     const llmAdapter = providerRegistry
       ? new LLMProviderAdapter(providerRegistry)
       : { complete: async () => '', available: () => false };
-    const cache = new FileCache('dynamic');
+    const cache = new FileCache('dynamic', process.env.MCPLOCAL_CACHE_DIR ? { dir: process.env.MCPLOCAL_CACHE_DIR } : undefined);
 
     if (serverName && serverProxyModel) {
       entry.router.setServerProxyModel(serverName, serverProxyModel, llmAdapter, cache);

src/mcplocal/src/http/token-auth.ts

@@ -0,0 +1,114 @@
+/**
+ * Fastify preHandler that authenticates `/projects/*` and `/mcp` requests
+ * against mcpd's McpToken introspection endpoint.
+ *
+ * Flow:
+ *   1. Reject non-Bearer and non-`mcpctl_pat_` auth up front.
+ *   2. Call `GET <mcpd>/api/v1/mcptokens/introspect` with the raw bearer.
+ *   3. Cache the result (positive + negative TTLs) to avoid a round-trip per MCP call.
+ *   4. Enforce `request.params.projectName === response.projectName`.
+ *   5. Stash the principal on `request.mcpToken` for the audit collector.
+ */
+import type { FastifyRequest, FastifyReply } from 'fastify';
+import { isMcpToken, hashToken } from '@mcpctl/shared';
+
+export interface TokenAuthOptions {
+  mcpdUrl: string;
+  /** TTL for a successful introspection, ms. Default 30_000. */
+  positiveTtlMs?: number;
+  /** TTL for a failed introspection, ms. Default 5_000. */
+  negativeTtlMs?: number;
+  /** Injectable HTTP fetcher for tests. Defaults to `fetch`. */
+  fetch?: (url: string, init?: RequestInit) => Promise<Response>;
+}
+
+export interface McpTokenPrincipal {
+  tokenName: string;
+  tokenSha: string;
+  projectName: string;
+}
+
+declare module 'fastify' {
+  interface FastifyRequest {
+    /** Populated by the token-auth preHandler when the bearer was a McpToken. */
+    mcpToken?: McpTokenPrincipal;
+  }
+}
+
+interface IntrospectResponse {
+  ok: boolean;
+  tokenName?: string;
+  tokenSha?: string;
+  projectName?: string;
+  revoked?: boolean;
+  expired?: boolean;
+  error?: string;
+}
+
+interface CacheEntry {
+  result: IntrospectResponse;
+  expiresAt: number;
+}
+
+export function createTokenAuthMiddleware(opts: TokenAuthOptions) {
+  const positiveTtl = opts.positiveTtlMs ?? 30_000;
+  const negativeTtl = opts.negativeTtlMs ?? 5_000;
+  const fetchImpl = opts.fetch ?? (globalThis.fetch as typeof fetch);
+  const cache = new Map<string, CacheEntry>();
+
+  async function introspect(raw: string): Promise<IntrospectResponse> {
+    const key = hashToken(raw);
+    const now = Date.now();
+    const hit = cache.get(key);
+    if (hit && hit.expiresAt > now) return hit.result;
+
+    try {
+      const res = await fetchImpl(`${opts.mcpdUrl.replace(/\/$/, '')}/api/v1/mcptokens/introspect`, {
+        method: 'GET',
+        headers: { Authorization: `Bearer ${raw}` },
+      });
+      const body = (await res.json().catch(() => ({ ok: false, error: 'unreadable body' }))) as IntrospectResponse;
+      const result: IntrospectResponse = res.ok ? body : { ...body, ok: false };
+      cache.set(key, { result, expiresAt: now + (result.ok ? positiveTtl : negativeTtl) });
+      return result;
+    } catch (err) {
+      const result: IntrospectResponse = { ok: false, error: err instanceof Error ? err.message : String(err) };
+      cache.set(key, { result, expiresAt: now + negativeTtl });
+      return result;
+    }
+  }
+
+  return async function tokenAuth(request: FastifyRequest, reply: FastifyReply): Promise<void> {
+    const header = request.headers.authorization;
+    if (header === undefined || !header.startsWith('Bearer ')) {
+      reply.code(401).send({ error: 'Missing Authorization bearer' });
+      return;
+    }
+    const raw = header.slice(7);
+    if (!isMcpToken(raw)) {
+      reply.code(401).send({ error: 'Only mcpctl_pat_ bearers are accepted on this endpoint' });
+      return;
+    }
+
+    const introspection = await introspect(raw);
+    if (!introspection.ok) {
+      reply.code(401).send({
+        error: introspection.revoked ? 'Token revoked' : introspection.expired ? 'Token expired' : 'Invalid token',
+      });
+      return;
+    }
+
+    // Project-scope check: token.projectName must match the path param.
+    const params = request.params as { projectName?: string } | undefined;
+    if (params?.projectName !== undefined && params.projectName !== introspection.projectName) {
+      reply.code(403).send({ error: `Token is not valid for project '${params.projectName}'` });
+      return;
+    }
+
+    request.mcpToken = {
+      tokenName: introspection.tokenName!,
+      tokenSha: introspection.tokenSha!,
+      projectName: introspection.projectName!,
+    };
+  };
+}

src/mcplocal/src/serve.ts

@@ -0,0 +1,99 @@
+#!/usr/bin/env node
+/**
+ * HTTP-only entry for the containerized mcplocal (deployed behind Ingress as `mcp.ad.itaz.eu`).
+ *
+ * Differences from main.ts (the STDIO/systemd entry):
+ *   - No StdioProxyServer (there's no stdin/stdout MCP client in a pod).
+ *   - No `--upstream` flag (upstreams come from mcpd project discovery).
+ *   - Host + port from env (MCPLOCAL_HTTP_HOST / MCPLOCAL_HTTP_PORT).
+ *   - Requires MCPLOCAL_MCPD_URL to point at mcpd inside the cluster.
+ *   - Registers a token-auth preHandler on `/projects/*` and `/mcp`.
+ *   - FileCache directory honours MCPLOCAL_CACHE_DIR (wired via project-mcp-endpoint).
+ */
+import { McpRouter } from './router.js';
+import { createHttpServer } from './http/server.js';
+import { loadHttpConfig, loadLlmProviders } from './http/config.js';
+import { createProvidersFromConfig } from './llm-config.js';
+import { createSecretStore } from '@mcpctl/shared';
+import { reloadStages, startWatchers, stopWatchers } from './proxymodel/watcher.js';
+import { createTokenAuthMiddleware } from './http/token-auth.js';
+
+function requireEnv(name: string): string {
+  const value = process.env[name];
+  if (value === undefined || value === '') {
+    throw new Error(`Required env var ${name} is not set`);
+  }
+  return value;
+}
+
+export async function serve(): Promise<void> {
+  const mcpdUrl = requireEnv('MCPLOCAL_MCPD_URL');
+  const httpHost = process.env.MCPLOCAL_HTTP_HOST ?? '0.0.0.0';
+  const httpPort = Number(process.env.MCPLOCAL_HTTP_PORT ?? '3200');
+  if (!Number.isFinite(httpPort) || httpPort <= 0) {
+    throw new Error(`Invalid MCPLOCAL_HTTP_PORT: ${process.env.MCPLOCAL_HTTP_PORT}`);
+  }
+  // MCPLOCAL_CACHE_DIR is optional; FileCache reads it directly.
+  const cacheDir = process.env.MCPLOCAL_CACHE_DIR;
+
+  // loadHttpConfig reads user-level config.json; we override with env.
+  const baseConfig = loadHttpConfig();
+  const httpConfig = {
+    ...baseConfig,
+    httpHost,
+    httpPort,
+    mcpdUrl,
+  };
+
+  // LLM providers (configured via mounted ConfigMap at ~/.mcpctl/config.json or env).
+  const llmEntries = loadLlmProviders();
+  const secretStore = await createSecretStore();
+  const providerRegistry = await createProvidersFromConfig(llmEntries, secretStore);
+
+  process.stderr.write(
+    `mcplocal-serve: mcpd=${mcpdUrl} host=${httpHost} port=${httpPort} cache=${cacheDir ?? '~/.mcpctl/cache'}\n`,
+  );
+
+  const router = new McpRouter();
+
+  const httpServer = await createHttpServer(httpConfig, { router, providerRegistry });
+
+  // Auth preHandler: only protect the MCP surfaces. /health, /healthz, /proxymodels etc stay open.
+  const tokenAuth = createTokenAuthMiddleware({ mcpdUrl });
+  httpServer.addHook('preHandler', async (request, reply) => {
+    const url = request.url;
+    if (!url.startsWith('/projects/') && !url.startsWith('/mcp')) return;
+    await tokenAuth(request, reply);
+  });
+
+  await httpServer.listen({ port: httpPort, host: httpHost });
+  process.stderr.write(`mcplocal-serve listening on ${httpHost}:${httpPort}\n`);
+
+  // Hot-reload proxymodel stages from ~/.mcpctl/stages (same as main.ts).
+  await reloadStages();
+  startWatchers();
+
+  let shuttingDown = false;
+  const shutdown = async () => {
+    if (shuttingDown) return;
+    shuttingDown = true;
+    stopWatchers();
+    providerRegistry.disposeAll();
+    await httpServer.close();
+    await router.closeAll();
+    process.exit(0);
+  };
+  process.on('SIGTERM', () => void shutdown());
+  process.on('SIGINT', () => void shutdown());
+}
+
+const isMain =
+  process.argv[1]?.endsWith('serve.js') ||
+  process.argv[1]?.endsWith('serve.ts');
+
+if (isMain) {
+  serve().catch((err) => {
+    process.stderr.write(`Fatal: ${err}\n`);
+    process.exit(1);
+  });
+}

src/mcplocal/tests/smoke/mcptoken.smoke.test.ts

@@ -0,0 +1,143 @@
+/**
+ * Smoke tests: McpToken + HTTP-mode mcplocal end-to-end.
+ *
+ * Exercises the full public CLI contract:
+ *   1. `mcpctl create project` + `mcpctl create mcptoken`
+ *   2. `mcpctl test mcp <url> --token $TOK --expect-tools …` → exit 0
+ *   3. Same token against a different project → exit 1 (403)
+ *   4. Revoke the token, retry → exit 1 (401) within the negative-cache window
+ *   5. --expect-tools <nonexistent> → exit 2 (contract failure)
+ *
+ * Target endpoint: $MCPGW_URL (default https://mcp.ad.itaz.eu). The containerized
+ * mcplocal must be deployed and reachable. If the /healthz preflight fails we
+ * skip the whole suite with a clear message.
+ *
+ * Run with: pnpm test:smoke
+ */
+import { describe, it, expect, beforeAll } from 'vitest';
+import http from 'node:http';
+import https from 'node:https';
+import { execSync } from 'node:child_process';
+
+const MCPGW_URL = process.env.MCPGW_URL ?? 'https://mcp.ad.itaz.eu';
+const PROJECT_NAME = `smoke-mcptoken-${Date.now().toString(36)}`;
+const TOKEN_NAME = 'smoketok';
+const OTHER_PROJECT = 'smoke-mcptoken-other';
+
+interface CliResult { code: number; stdout: string; stderr: string }
+
+function run(args: string): CliResult {
+  try {
+    const stdout = execSync(`mcpctl ${args}`, {
+      encoding: 'utf-8',
+      timeout: 30_000,
+      stdio: ['ignore', 'pipe', 'pipe'],
+    });
+    return { code: 0, stdout: stdout.trim(), stderr: '' };
+  } catch (err) {
+    const e = err as { status?: number; stdout?: Buffer | string; stderr?: Buffer | string };
+    return {
+      code: e.status ?? 1,
+      stdout: e.stdout ? (typeof e.stdout === 'string' ? e.stdout : e.stdout.toString('utf-8')) : '',
+      stderr: e.stderr ? (typeof e.stderr === 'string' ? e.stderr : e.stderr.toString('utf-8')) : '',
+    };
+  }
+}
+
+function healthz(url: string, timeoutMs = 5000): Promise<boolean> {
+  return new Promise((resolve) => {
+    const parsed = new URL(`${url.replace(/\/$/, '')}/healthz`);
+    const driver = parsed.protocol === 'https:' ? https : http;
+    const req = driver.get(
+      {
+        hostname: parsed.hostname,
+        port: parsed.port || (parsed.protocol === 'https:' ? 443 : 80),
+        path: parsed.pathname,
+        timeout: timeoutMs,
+      },
+      (res) => {
+        resolve((res.statusCode ?? 500) < 500);
+        res.resume();
+      },
+    );
+    req.on('error', () => resolve(false));
+    req.on('timeout', () => { req.destroy(); resolve(false); });
+  });
+}
+
+let gatewayUp = false;
+let rawToken = '';
+let knownToolName: string | undefined;
+
+beforeAll(async () => {
+  gatewayUp = await healthz(MCPGW_URL);
+}, 20_000);
+
+describe.skipIf(!gatewayUp)('mcptoken smoke (MCPGW_URL=' + MCPGW_URL + ')', () => {
+  it('creates the project and a project-scoped mcptoken', () => {
+    run(`delete project ${PROJECT_NAME} --force`); // cleanup leftovers — best-effort
+    const createProj = run(`create project ${PROJECT_NAME} --force`);
+    expect(createProj.code).toBe(0);
+
+    const createTok = run(`create mcptoken ${TOKEN_NAME} --project ${PROJECT_NAME} --rbac clone`);
+    expect(createTok.code).toBe(0);
+    const match = createTok.stdout.match(/mcpctl_pat_[A-Za-z0-9]+/);
+    expect(match, 'raw token was printed to stdout').not.toBeNull();
+    rawToken = match![0];
+  });
+
+  it('passes `mcpctl test mcp` against the token\'s project endpoint', () => {
+    const result = run(`test mcp ${MCPGW_URL}/projects/${PROJECT_NAME}/mcp --token ${rawToken} -o json`);
+    expect(result.code, result.stderr || result.stdout).toBe(0);
+    const report = JSON.parse(result.stdout.slice(result.stdout.indexOf('{'))) as {
+      exitCode: number;
+      tools: string[] | null;
+      initialize: string;
+    };
+    expect(report.exitCode).toBe(0);
+    expect(report.initialize).toBe('ok');
+    expect(Array.isArray(report.tools)).toBe(true);
+    // Remember a tool name for the next negative --expect-tools assertion
+    knownToolName = report.tools?.[0];
+  });
+
+  it('fails `mcpctl test mcp` against a different project with 403', () => {
+    run(`create project ${OTHER_PROJECT} --force`);
+    const result = run(`test mcp ${MCPGW_URL}/projects/${OTHER_PROJECT}/mcp --token ${rawToken} -o json`);
+    expect(result.code).toBe(1);
+    const report = JSON.parse(result.stdout.slice(result.stdout.indexOf('{'))) as { error?: string };
+    expect(report.error ?? '').toMatch(/403|not valid for|project/i);
+  });
+
+  it('exits 2 (contract failure) when --expect-tools names a nonexistent tool', () => {
+    const result = run(`test mcp ${MCPGW_URL}/projects/${PROJECT_NAME}/mcp --token ${rawToken} --expect-tools __nonexistent_tool_xyz__`);
+    expect(result.code).toBe(2);
+  });
+
+  it('returns 401 after the token is revoked (within the negative-cache window)', async () => {
+    const del = run(`delete mcptoken ${TOKEN_NAME} --project ${PROJECT_NAME}`);
+    expect(del.code).toBe(0);
+    // Let the mcplocal negative-cache window expire. Introspection negative TTL
+    // defaults to 5s; we wait 7s to be safe.
+    await new Promise((r) => setTimeout(r, 7_000));
+    const result = run(`test mcp ${MCPGW_URL}/projects/${PROJECT_NAME}/mcp --token ${rawToken} -o json`);
+    expect(result.code).toBe(1);
+    const report = JSON.parse(result.stdout.slice(result.stdout.indexOf('{'))) as { error?: string };
+    expect(report.error ?? '').toMatch(/401|revoked|Invalid token/i);
+  }, 20_000);
+
+  it('cleans up test fixtures', () => {
+    run(`delete project ${PROJECT_NAME} --force`);
+    run(`delete project ${OTHER_PROJECT} --force`);
+    // Suppress the unused-var warning in strict setups
+    expect(knownToolName === undefined || typeof knownToolName === 'string').toBe(true);
+  });
+});
+
+describe.skipIf(gatewayUp)('mcptoken smoke (SKIPPED)', () => {
+  it('is skipped because MCPGW_URL is unreachable', () => {
+    // eslint-disable-next-line no-console
+    console.warn(`mcptoken smoke: skipped — ${MCPGW_URL}/healthz unreachable. Set MCPGW_URL to override.`);
+    expect(true).toBe(true);
+  });
+});