2026-04-12 22:26:26 +01:00
|
|
|
import { describe, it, expect, vi } from 'vitest';
|
|
|
|
|
import { PassThrough } from 'node:stream';
|
|
|
|
|
import { PersistentStdioClient } from '../src/services/transport/persistent-stdio.js';
|
|
|
|
|
import type { InteractiveExec, McpOrchestrator } from '../src/services/orchestrator.js';
|
|
|
|
|
|
|
|
|
|
function makeFakeExec(): {
|
|
|
|
|
iexec: InteractiveExec;
|
|
|
|
|
written: string[];
|
|
|
|
|
emit: (line: unknown) => void;
|
|
|
|
|
} {
|
|
|
|
|
const stdout = new PassThrough();
|
|
|
|
|
const written: string[] = [];
|
|
|
|
|
const iexec: InteractiveExec = {
|
|
|
|
|
stdout,
|
|
|
|
|
write(data) { written.push(data); },
|
|
|
|
|
close() { stdout.destroy(); },
|
|
|
|
|
};
|
|
|
|
|
const emit = (msg: unknown) => {
|
|
|
|
|
stdout.write(JSON.stringify(msg) + '\n');
|
|
|
|
|
};
|
|
|
|
|
return { iexec, written, emit };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function makeOrchestrator(overrides: Partial<McpOrchestrator> = {}): McpOrchestrator {
|
|
|
|
|
return {
|
|
|
|
|
pullImage: vi.fn(),
|
|
|
|
|
createContainer: vi.fn(),
|
|
|
|
|
stopContainer: vi.fn(),
|
|
|
|
|
removeContainer: vi.fn(),
|
|
|
|
|
inspectContainer: vi.fn(),
|
|
|
|
|
getContainerLogs: vi.fn(),
|
|
|
|
|
execInContainer: vi.fn(),
|
|
|
|
|
ping: vi.fn(),
|
|
|
|
|
...overrides,
|
|
|
|
|
} as McpOrchestrator;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
describe('PersistentStdioClient', () => {
|
|
|
|
|
it('exec mode calls execInteractive with the command', async () => {
|
|
|
|
|
const fake = makeFakeExec();
|
|
|
|
|
const execInteractive = vi.fn(async () => fake.iexec);
|
|
|
|
|
const orch = makeOrchestrator({ execInteractive });
|
|
|
|
|
|
|
|
|
|
const client = new PersistentStdioClient(
|
|
|
|
|
orch,
|
|
|
|
|
'container-1',
|
|
|
|
|
{ kind: 'exec', command: ['node', 'index.js'] },
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
// Drive the handshake: respond to the first init request (id=1)
|
|
|
|
|
// then to the subsequent tools/list request (id=2).
|
|
|
|
|
const sendPromise = client.send('tools/list');
|
|
|
|
|
await new Promise((r) => setTimeout(r, 10));
|
|
|
|
|
|
|
|
|
|
const init = JSON.parse(fake.written[0]!);
|
|
|
|
|
expect(init.method).toBe('initialize');
|
|
|
|
|
fake.emit({ jsonrpc: '2.0', id: init.id, result: { capabilities: {} } });
|
|
|
|
|
await new Promise((r) => setTimeout(r, 150));
|
|
|
|
|
|
|
|
|
|
// Second written msg is notifications/initialized; third is tools/list
|
|
|
|
|
const toolsReq = JSON.parse(fake.written[2]!);
|
|
|
|
|
expect(toolsReq.method).toBe('tools/list');
|
|
|
|
|
fake.emit({ jsonrpc: '2.0', id: toolsReq.id, result: { tools: [] } });
|
|
|
|
|
|
|
|
|
|
const res = await sendPromise;
|
|
|
|
|
expect(res.result).toEqual({ tools: [] });
|
|
|
|
|
expect(execInteractive).toHaveBeenCalledWith('container-1', ['node', 'index.js']);
|
|
|
|
|
client.close();
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it('attach mode calls attachInteractive and never execInteractive', async () => {
|
|
|
|
|
const fake = makeFakeExec();
|
|
|
|
|
const attachInteractive = vi.fn(async () => fake.iexec);
|
|
|
|
|
const execInteractive = vi.fn();
|
|
|
|
|
const orch = makeOrchestrator({ attachInteractive, execInteractive });
|
|
|
|
|
|
|
|
|
|
const client = new PersistentStdioClient(
|
|
|
|
|
orch,
|
|
|
|
|
'container-gitea',
|
|
|
|
|
{ kind: 'attach' },
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
const sendPromise = client.send('tools/list');
|
|
|
|
|
await new Promise((r) => setTimeout(r, 10));
|
|
|
|
|
|
|
|
|
|
const init = JSON.parse(fake.written[0]!);
|
|
|
|
|
fake.emit({ jsonrpc: '2.0', id: init.id, result: { capabilities: {} } });
|
|
|
|
|
await new Promise((r) => setTimeout(r, 150));
|
|
|
|
|
|
|
|
|
|
const req = JSON.parse(fake.written[2]!);
|
|
|
|
|
fake.emit({ jsonrpc: '2.0', id: req.id, result: { tools: [{ name: 'list_repos' }] } });
|
|
|
|
|
|
|
|
|
|
const res = await sendPromise;
|
|
|
|
|
expect((res.result as { tools: unknown[] }).tools).toHaveLength(1);
|
|
|
|
|
expect(attachInteractive).toHaveBeenCalledWith('container-gitea');
|
|
|
|
|
expect(execInteractive).not.toHaveBeenCalled();
|
|
|
|
|
client.close();
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it('attach mode throws if orchestrator does not support attach', async () => {
|
|
|
|
|
const orch = makeOrchestrator({}); // no attachInteractive
|
|
|
|
|
const client = new PersistentStdioClient(orch, 'c', { kind: 'attach' });
|
|
|
|
|
await expect(client.send('tools/list')).rejects.toThrow(/attach/i);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it('exec mode throws if orchestrator does not support execInteractive', async () => {
|
|
|
|
|
const orch = makeOrchestrator({}); // no execInteractive
|
|
|
|
|
const client = new PersistentStdioClient(orch, 'c', { kind: 'exec', command: ['x'] });
|
|
|
|
|
await expect(client.send('tools/list')).rejects.toThrow(/interactive exec/i);
|
|
|
|
|
});
|
|
|
|
|
});
|
fix(mcpd): recover k8s STDIO instances after in-place container restarts
A container restart (OOMKill, crash, transient npx failure) used to strand
the instance in ERROR forever while its pod sat 1/1 Running, because five
gaps lined up (#114):
- the exec/attach websocket died without ending the stdout PassThrough
(client-node installs no onclose), so PersistentStdioClient — whose only
death signal was stdout 'end' — kept believing it was connected and every
request rode the 120s timeout into a dead pipe;
- the stdioClients cache is keyed by pod name, which survives a restart, so
nothing ever evicted the corpse;
- syncStatus never re-inspected ERROR rows and never read restartCount, so
neither the recovery nor the in-place restart was visible;
- its ERROR writes clobbered retry metadata, making the row instantly
dueForRetry, and the retry recreated the pod under the SAME name — an
uncaught 409 that looped ERROR against a healthy pod;
- the stuck row consumed the whole replica budget, blocking a fresh-id
replacement.
The fix, layer by layer:
- ws 'close'/'error' now end stdout in both k8s interactive paths (and the
docker interactive path mirrors its own one-shot handlers), funneling into
an identity-guarded teardown in PersistentStdioClient that also listens
for stream 'close'/'error' — a late event from a previous session cannot
clobber a reconnected one;
- syncStatus re-inspects ERROR rows (pod running again → back to RUNNING
with retry metadata cleared), tracks restartCount in instance metadata to
catch restarts between polls, MERGES retry metadata instead of clobbering
it, and evicts the cached stdio client through a setter-injected hook
whenever the pipe is known-dead;
- createContainer adopts an alive pod on 409 instead of throwing (and only
replaces a genuinely dead one, waiting out the deletion grace period);
- attach mode retries once through a fresh client before surfacing an error,
so the first call after a detected death succeeds;
- the health probe evicts the cached client when failures cross the
threshold, and shutdown finally calls closeAll().
Closes #114
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JaFvfHrQyUKCGv6o3N2Wir
2026-08-15 23:32:23 +01:00
|
|
|
|
|
|
|
|
// Drive a fake session through the init handshake so send() can round-trip.
|
|
|
|
|
async function completeHandshake(fake: ReturnType<typeof makeFakeExec>): Promise<void> {
|
|
|
|
|
await new Promise((r) => setTimeout(r, 10));
|
|
|
|
|
const init = JSON.parse(fake.written[0]!);
|
|
|
|
|
fake.emit({ jsonrpc: '2.0', id: init.id, result: { capabilities: {} } });
|
|
|
|
|
await new Promise((r) => setTimeout(r, 150));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
describe('PersistentStdioClient teardown & reconnect (mcpctl#114)', () => {
|
|
|
|
|
it("stdout 'end' rejects in-flight requests and the next send() redials", async () => {
|
|
|
|
|
const fakes = [makeFakeExec(), makeFakeExec()];
|
|
|
|
|
let call = 0;
|
|
|
|
|
const execInteractive = vi.fn(async () => fakes[call++]!.iexec);
|
|
|
|
|
const client = new PersistentStdioClient(
|
|
|
|
|
makeOrchestrator({ execInteractive }),
|
|
|
|
|
'c1',
|
|
|
|
|
{ kind: 'exec', command: ['node'] },
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
const first = client.send('tools/list');
|
|
|
|
|
await completeHandshake(fakes[0]!);
|
|
|
|
|
// Kill the session while tools/list is in flight.
|
|
|
|
|
fakes[0]!.iexec.stdout.end();
|
|
|
|
|
await expect(first).rejects.toThrow('STDIO process exited');
|
|
|
|
|
expect(client.isConnected).toBe(false);
|
|
|
|
|
|
|
|
|
|
// Next send must redial through a fresh interactive session.
|
|
|
|
|
const second = client.send('tools/list');
|
|
|
|
|
await completeHandshake(fakes[1]!);
|
|
|
|
|
const req = JSON.parse(fakes[1]!.written[2]!);
|
|
|
|
|
fakes[1]!.emit({ jsonrpc: '2.0', id: req.id, result: { tools: [] } });
|
|
|
|
|
const res = await second;
|
|
|
|
|
expect(res.result).toEqual({ tools: [] });
|
|
|
|
|
expect(execInteractive).toHaveBeenCalledTimes(2);
|
|
|
|
|
client.close();
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("stdout 'error' tears down with the error message", async () => {
|
|
|
|
|
const fake = makeFakeExec();
|
|
|
|
|
const client = new PersistentStdioClient(
|
|
|
|
|
makeOrchestrator({ execInteractive: vi.fn(async () => fake.iexec) }),
|
|
|
|
|
'c1',
|
|
|
|
|
{ kind: 'exec', command: ['node'] },
|
|
|
|
|
);
|
|
|
|
|
const pending = client.send('tools/list');
|
|
|
|
|
await completeHandshake(fake);
|
|
|
|
|
fake.iexec.stdout.emit('error', new Error('socket hang up'));
|
|
|
|
|
await expect(pending).rejects.toThrow('STDIO stream error: socket hang up');
|
|
|
|
|
expect(client.isConnected).toBe(false);
|
|
|
|
|
client.close();
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("a LATE 'end' from the previous session does not clobber the reconnected one", async () => {
|
|
|
|
|
const fakes = [makeFakeExec(), makeFakeExec()];
|
|
|
|
|
let call = 0;
|
|
|
|
|
const client = new PersistentStdioClient(
|
|
|
|
|
makeOrchestrator({ attachInteractive: vi.fn(async () => fakes[call++]!.iexec) }),
|
|
|
|
|
'c1',
|
|
|
|
|
{ kind: 'attach' },
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
const first = client.send('tools/list');
|
|
|
|
|
await completeHandshake(fakes[0]!);
|
|
|
|
|
fakes[0]!.iexec.stdout.end();
|
|
|
|
|
await expect(first).rejects.toThrow('STDIO process exited');
|
|
|
|
|
|
|
|
|
|
// Reconnect on a fresh session.
|
|
|
|
|
const second = client.send('tools/list');
|
|
|
|
|
await completeHandshake(fakes[1]!);
|
|
|
|
|
|
|
|
|
|
// The old session's stream fires a stale 'close' AFTER the reconnect —
|
|
|
|
|
// the identity guard must ignore it.
|
|
|
|
|
fakes[0]!.iexec.stdout.emit('close');
|
|
|
|
|
expect(client.isConnected).toBe(true);
|
|
|
|
|
|
|
|
|
|
const req = JSON.parse(fakes[1]!.written[2]!);
|
|
|
|
|
fakes[1]!.emit({ jsonrpc: '2.0', id: req.id, result: { tools: [{ name: 't' }] } });
|
|
|
|
|
const res = await second;
|
|
|
|
|
expect((res.result as { tools: unknown[] }).tools).toHaveLength(1);
|
|
|
|
|
client.close();
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it('attach mode also redials after teardown', async () => {
|
|
|
|
|
const fakes = [makeFakeExec(), makeFakeExec()];
|
|
|
|
|
let call = 0;
|
|
|
|
|
const attachInteractive = vi.fn(async () => fakes[call++]!.iexec);
|
|
|
|
|
const client = new PersistentStdioClient(
|
|
|
|
|
makeOrchestrator({ attachInteractive }),
|
|
|
|
|
'c-gitea',
|
|
|
|
|
{ kind: 'attach' },
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
const first = client.send('tools/list');
|
|
|
|
|
await completeHandshake(fakes[0]!);
|
|
|
|
|
fakes[0]!.iexec.stdout.emit('close');
|
|
|
|
|
await expect(first).rejects.toThrow('STDIO stream closed');
|
|
|
|
|
|
|
|
|
|
const second = client.send('tools/list');
|
|
|
|
|
await completeHandshake(fakes[1]!);
|
|
|
|
|
const req = JSON.parse(fakes[1]!.written[2]!);
|
|
|
|
|
fakes[1]!.emit({ jsonrpc: '2.0', id: req.id, result: { tools: [] } });
|
|
|
|
|
await expect(second).resolves.toMatchObject({ result: { tools: [] } });
|
|
|
|
|
expect(attachInteractive).toHaveBeenCalledTimes(2);
|
|
|
|
|
client.close();
|
|
|
|
|
});
|
|
|
|
|
});
|