feat: virtual-LLM smoke test + docs (v1 Stage 6)
Some checks failed
CI/CD / typecheck (pull_request) Successful in 53s
CI/CD / test (pull_request) Successful in 1m8s
CI/CD / lint (pull_request) Successful in 2m6s
CI/CD / smoke (pull_request) Failing after 1m39s
CI/CD / build (pull_request) Successful in 2m11s
CI/CD / publish (pull_request) Has been skipped

Final stage of v1.

Smoke (mcplocal/tests/smoke/virtual-llm.smoke.test.ts):
- Spins an in-process LlmProvider that returns canned content.
- Runs the registrar against the live mcpd in fulldeploy.
- Asserts: row appears with kind=virtual / status=active, infer
  through /api/v1/llms/<name>/infer comes back through the SSE
  relay with the provider's content + finish_reason, and a 503
  appears immediately after registrar.stop() (publisher offline).
- Times out / cleanup paths idempotent so re-runs against the same
  cluster don't litter rows. The 90-s heartbeat-stale flip and 4-h
  GC are unit-tested — too slow for smoke.

Docs:
- New docs/virtual-llms.md: when to use this vs creating a regular
  Llm row, how to opt-in via publish: true, the lifecycle table,
  the inference-relay sequence, the v1 streaming caveat, the v2-v5
  roadmap, and the full /api/v1/llms/_provider-* surface.
- agents.md cross-links virtual-llms.md alongside personalities/chat.
- README's Agents section gains a "Virtual LLMs" subsection.

Workspace suite: 2043/2043 (smoke files run separately). v1 closes.

Stage roadmap (each its own future PR):
  v2 wake-on-demand · v3 virtual agents · v4 LB pool · v5 task queue

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Michal
2026-04-27 14:28:43 +01:00
parent 7e6b0cab44
commit 866f6abc88
4 changed files with 410 additions and 0 deletions

View File

@@ -0,0 +1,209 @@
/**
* Smoke tests: virtual-LLM register → infer relay → cleanup against a live
* mcpd. Uses an in-process LlmProvider (returns canned content) so we
* exercise the SSE control plane and the kind=virtual infer branch
* without depending on a real upstream model.
*
* The 90-s heartbeat-stale flip and 4-h auto-deletion are covered by unit
* tests (mcpd virtual-llm-service.test.ts); waiting > 90 s in smoke would
* triple the suite duration.
*/
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
import http from 'node:http';
import https from 'node:https';
import { mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import {
VirtualLlmRegistrar,
type RegistrarPublishedProvider,
} from '../../src/providers/registrar.js';
import type { LlmProvider, CompletionResult } from '../../src/providers/types.js';
const MCPD_URL = process.env.MCPD_URL ?? 'https://mcpctl.ad.itaz.eu';
const SUFFIX = Date.now().toString(36);
const PROVIDER_NAME = `smoke-virtual-${SUFFIX}`;
function makeFakeProvider(name: string, content: string): LlmProvider {
return {
name,
async complete(): Promise<CompletionResult> {
return {
content,
toolCalls: [],
usage: { promptTokens: 1, completionTokens: 4, totalTokens: 5 },
finishReason: 'stop',
};
},
async listModels() { return []; },
async isAvailable() { return true; },
};
}
function healthz(url: string, timeoutMs = 5000): Promise<boolean> {
return new Promise((resolve) => {
const parsed = new URL(`${url.replace(/\/$/, '')}/healthz`);
const driver = parsed.protocol === 'https:' ? https : http;
const req = driver.get(
{
hostname: parsed.hostname,
port: parsed.port || (parsed.protocol === 'https:' ? 443 : 80),
path: parsed.pathname,
timeout: timeoutMs,
},
(res) => { resolve((res.statusCode ?? 500) < 500); res.resume(); },
);
req.on('error', () => resolve(false));
req.on('timeout', () => { req.destroy(); resolve(false); });
});
}
function readToken(): string | null {
try {
const home = process.env.HOME ?? '';
const path = `${home}/.mcpctl/credentials`;
// eslint-disable-next-line @typescript-eslint/no-require-imports
const fs = require('node:fs') as typeof import('node:fs');
if (!fs.existsSync(path)) return null;
const raw = fs.readFileSync(path, 'utf-8');
const parsed = JSON.parse(raw) as { token?: string };
return parsed.token ?? null;
} catch {
return null;
}
}
interface HttpResponse { status: number; body: string }
function httpRequest(method: string, urlStr: string, body: unknown): Promise<HttpResponse> {
return new Promise((resolve, reject) => {
const tokenRaw = readToken();
const parsed = new URL(urlStr);
const driver = parsed.protocol === 'https:' ? https : http;
const headers: Record<string, string> = {
Accept: 'application/json',
...(body !== undefined ? { 'Content-Type': 'application/json' } : {}),
...(tokenRaw !== null ? { Authorization: `Bearer ${tokenRaw}` } : {}),
};
const req = driver.request({
hostname: parsed.hostname,
port: parsed.port || (parsed.protocol === 'https:' ? 443 : 80),
path: parsed.pathname + parsed.search,
method,
headers,
timeout: 30_000,
}, (res) => {
const chunks: Buffer[] = [];
res.on('data', (c: Buffer) => chunks.push(c));
res.on('end', () => {
resolve({ status: res.statusCode ?? 0, body: Buffer.concat(chunks).toString('utf-8') });
});
});
req.on('error', reject);
req.on('timeout', () => { req.destroy(); reject(new Error(`httpRequest timeout: ${method} ${urlStr}`)); });
if (body !== undefined) req.write(JSON.stringify(body));
req.end();
});
}
let mcpdUp = false;
let registrar: VirtualLlmRegistrar | null = null;
let tempDir: string;
describe('virtual-LLM smoke', () => {
beforeAll(async () => {
mcpdUp = await healthz(MCPD_URL);
if (!mcpdUp) {
// eslint-disable-next-line no-console
console.warn(`\n ○ virtual-llm smoke: skipped — ${MCPD_URL}/healthz unreachable.\n`);
return;
}
if (readToken() === null) {
mcpdUp = false;
// eslint-disable-next-line no-console
console.warn('\n ○ virtual-llm smoke: skipped — no ~/.mcpctl/credentials.\n');
return;
}
tempDir = mkdtempSync(join(tmpdir(), 'mcpctl-virtual-llm-smoke-'));
}, 20_000);
afterAll(async () => {
if (registrar !== null) registrar.stop();
if (tempDir !== undefined) rmSync(tempDir, { recursive: true, force: true });
// Best-effort cleanup of the row in case the disconnect didn't finish
// before mcpd's heartbeat watchdog ticks. Idempotent.
if (mcpdUp) {
const list = await httpRequest('GET', `${MCPD_URL}/api/v1/llms`, undefined);
if (list.status === 200) {
const rows = JSON.parse(list.body) as Array<{ id: string; name: string }>;
const row = rows.find((r) => r.name === PROVIDER_NAME);
if (row !== undefined) {
await httpRequest('DELETE', `${MCPD_URL}/api/v1/llms/${row.id}`, undefined);
}
}
}
});
it('registrar publishes the provider and mcpd lists it as kind=virtual / status=active', async () => {
if (!mcpdUp) return;
const token = readToken();
if (token === null) return;
const published: RegistrarPublishedProvider[] = [
{ provider: makeFakeProvider(PROVIDER_NAME, 'hi from smoke'), type: 'openai', model: 'fake-smoke', tier: 'fast' },
];
registrar = new VirtualLlmRegistrar({
mcpdUrl: MCPD_URL,
token,
publishedProviders: published,
sessionFilePath: join(tempDir, 'session'),
log: { info: () => {}, warn: () => {}, error: () => {} },
heartbeatIntervalMs: 60_000,
});
await registrar.start();
expect(registrar.getSessionId()).not.toBeNull();
// Give the SSE handshake + register a moment to settle on mcpd's side.
await new Promise((r) => setTimeout(r, 400));
const res = await httpRequest('GET', `${MCPD_URL}/api/v1/llms`, undefined);
expect(res.status).toBe(200);
const rows = JSON.parse(res.body) as Array<{ name: string; kind: string; status: string; type: string; model: string }>;
const row = rows.find((r) => r.name === PROVIDER_NAME);
expect(row, `${PROVIDER_NAME} must be present`).toBeDefined();
expect(row!.kind).toBe('virtual');
expect(row!.status).toBe('active');
expect(row!.type).toBe('openai');
expect(row!.model).toBe('fake-smoke');
}, 30_000);
it('mcpd routes /api/v1/llms/<virtual>/infer back through the SSE relay to the fake provider', async () => {
if (!mcpdUp) return;
const res = await httpRequest('POST', `${MCPD_URL}/api/v1/llms/${PROVIDER_NAME}/infer`, {
messages: [{ role: 'user', content: 'say something' }],
});
expect(res.status).toBe(200);
const body = JSON.parse(res.body) as {
choices?: Array<{ message?: { content?: string }; finish_reason?: string }>;
usage?: { total_tokens?: number };
};
expect(body.choices?.[0]?.message?.content).toBe('hi from smoke');
expect(body.choices?.[0]?.finish_reason).toBe('stop');
expect(body.usage?.total_tokens).toBe(5);
}, 30_000);
it('returns 503 with a clear error when the publisher disconnects mid-session', async () => {
if (!mcpdUp) return;
if (registrar !== null) {
registrar.stop();
registrar = null;
}
// Immediately after stop(), the SSE socket closes and mcpd's
// unbindSession flips the row to inactive. Inference should 503.
await new Promise((r) => setTimeout(r, 300));
const res = await httpRequest('POST', `${MCPD_URL}/api/v1/llms/${PROVIDER_NAME}/infer`, {
messages: [{ role: 'user', content: 'still there?' }],
});
expect(res.status).toBe(503);
expect(res.body).toMatch(/publisher offline|inactive/);
}, 30_000);
});