feat(mcpd+db): Llm.poolName + chat dispatcher pool failover (v4 Stage 1)

Adds LB-pool-by-shared-name without introducing a new resource. The
existing `Llm.name` stays globally unique; a new optional `poolName`
column declares membership in a pool. Multiple Llms sharing a non-null
`poolName` stack into one load-balanced pool that the chat dispatcher
expands at request time.

Effective pool key = `poolName ?? name`. Solo rows (poolName=null) are
addressable as a "pool of 1" via their own name, so existing single-Llm
agents and YAMLs keep working unchanged. A solo row whose name happens
to match an explicit poolName joins the same pool — by design — so an
operator can transparently promote an existing Llm to pool seed.

Dispatcher (chat.service): prepareContext now resolves a randomly-
shuffled list of viable pool candidates (status != inactive) once per
turn. runOneInference and streamInference iterate the list on
transport-level failure (network, virtual publisher disconnect) until
one succeeds or the list is exhausted. Streaming failover only covers
"failed before first chunk" — once we've yielded text, we're committed
to that backend. Auth/4xx errors surfaced as result.status are NOT
retried; siblings with the same key/model would fail identically.

When the agent's pinned Llm is itself inactive but a sibling pool
member is up, dispatch transparently uses the sibling — that's the
whole point. When every member is inactive, prepareContext throws a
clear "No active Llm in pool '<key>' (pinned: <name>)" error rather
than letting the dispatcher's "exhausted" branch surface it.

Tests:
- 5 new chat-service tests for pool dispatch / failover / pinned-down /
  all-inactive (chat-service.test.ts).
- 7 new db schema tests for the column, the unique-name invariant, the
  fallback-to-name semantics, and the solo-name-joins-explicit-pool
  edge case (llm-pool-schema.test.ts).
- mcpd 865/865 (was 860; +5), db pool-schema 7/7, no regressions.

Stage 2 (next): HTTP route /api/v1/llms/<name>/members + aggregate pool
stats on the existing single-Llm route, CLI POOL column + describe
block + --pool-name flag, yaml round-trip.
This commit is contained in:
Michal
2026-04-27 22:02:41 +01:00
parent f5bdeea8e7
commit 7949e1393d
9 changed files with 586 additions and 71 deletions

View File

@@ -71,17 +71,25 @@ function mockAgents(): AgentService {
}
function mockLlmsVirtual(): LlmService {
const baseRow = (name: string): Record<string, unknown> => ({
id: 'llm-1', name, type: 'openai', model: 'fake',
url: '', tier: 'fast', description: '',
apiKeySecretId: null, apiKeySecretKey: null,
extraConfig: {},
poolName: null,
kind: 'virtual',
providerSessionId: null,
status: 'active',
lastHeartbeatAt: NOW,
inactiveSince: null,
version: 1, createdAt: NOW, updatedAt: NOW,
});
return {
getByName: vi.fn(async (name: string) => ({
id: 'llm-1', name, type: 'openai', model: 'fake',
url: '', tier: 'fast', description: '',
apiKeyRef: null, extraConfig: {},
kind: 'virtual',
status: 'active',
lastHeartbeatAt: NOW,
inactiveSince: null,
version: 1, createdAt: NOW, updatedAt: NOW,
...baseRow(name),
apiKeyRef: null,
})),
findByPoolName: vi.fn(async (poolName: string) => [baseRow(poolName)]),
resolveApiKey: vi.fn(async () => ''),
} as unknown as LlmService;
}

View File

@@ -119,17 +119,30 @@ function mockAgents(opts: { defaultPersonality?: { id: string; name: string } |
}
function mockLlms(opts: { kind?: 'public' | 'virtual' } = {}): LlmService {
// v4: prepareContext now resolves a pool by effective key. For unit
// tests that pass a single agent.llm we return that one row twice —
// once for getByName (LlmView shape) and once for findByPoolName (raw
// Llm shape with the same name) so the dispatcher's poolCandidates
// ends up with exactly one member, matching pre-v4 behavior.
const baseRow = (name: string): Record<string, unknown> => ({
id: 'llm-1', name, type: 'openai', model: 'qwen3-thinking',
url: '', tier: 'fast', description: '',
apiKeySecretId: null, apiKeySecretKey: null,
extraConfig: {},
poolName: null,
kind: opts.kind ?? 'public',
providerSessionId: null,
status: 'active',
lastHeartbeatAt: null,
inactiveSince: null,
version: 1, createdAt: NOW, updatedAt: NOW,
});
return {
getByName: vi.fn(async (name: string) => ({
id: 'llm-1', name, type: 'openai', model: 'qwen3-thinking',
url: '', tier: 'fast', description: '',
apiKeyRef: null, extraConfig: {},
kind: opts.kind ?? 'public',
status: 'active',
lastHeartbeatAt: null,
inactiveSince: null,
version: 1, createdAt: NOW, updatedAt: NOW,
...baseRow(name),
apiKeyRef: null,
})),
findByPoolName: vi.fn(async (poolName: string) => [baseRow(poolName)]),
resolveApiKey: vi.fn(async () => 'fake-key'),
} as unknown as LlmService;
}
@@ -604,6 +617,176 @@ describe('ChatService', () => {
.toEqual(['thinking via litellm...']);
});
// ── v4: LB pool by shared name ──
// Helper: build a multi-member mock LlmService where all members share an
// effective pool key. `nameToFail` lets a test mark specific names as
// throwing on dispatch, exercising failover.
function mockLlmsPool(opts: {
pinnedName: string;
poolName: string | null;
members: Array<{ name: string; status?: 'active' | 'inactive'; kind?: 'public' | 'virtual' }>;
}): LlmService {
const baseRow = (m: { name: string; status?: 'active' | 'inactive'; kind?: 'public' | 'virtual' }): Record<string, unknown> => ({
id: `id-${m.name}`,
name: m.name,
type: 'openai',
model: 'qwen3-thinking',
url: '',
tier: 'fast',
description: '',
apiKeySecretId: null,
apiKeySecretKey: null,
extraConfig: {},
poolName: opts.poolName,
kind: m.kind ?? 'public',
providerSessionId: null,
status: m.status ?? 'active',
lastHeartbeatAt: null,
inactiveSince: null,
version: 1,
createdAt: NOW,
updatedAt: NOW,
});
return {
getByName: vi.fn(async (name: string) => {
const m = opts.members.find((x) => x.name === name) ?? opts.members[0]!;
return { ...baseRow(m), apiKeyRef: null };
}),
findByPoolName: vi.fn(async () => opts.members.map(baseRow)),
resolveApiKey: vi.fn(async () => 'fake-key'),
} as unknown as LlmService;
}
it('chat dispatches to a pool member and persists the reply (pool size N, primary returns)', async () => {
// Three healthy members; the (random) primary wins on first try.
// Adapter returns the same canned reply regardless of which member
// got picked because in this test we don't differentiate by name —
// we just assert that dispatch goes through and the agent gets a
// reply. Per-member assertion is covered by the failover test below.
const chatRepo = mockChatRepo();
const adapter = scriptedAdapter([chatCompletion('hello from pool')]);
const svc = new ChatService(
mockAgents(),
mockLlmsPool({
pinnedName: 'qwen-prod-1',
poolName: 'qwen-pool',
members: [
{ name: 'qwen-prod-1' },
{ name: 'qwen-prod-2' },
{ name: 'qwen-prod-3' },
],
}),
adapterRegistry(adapter),
chatRepo, mockPromptRepo(), mockTools(),
);
const result = await svc.chat({ agentName: 'reviewer', userMessage: 'hi', ownerId: 'owner-1' });
expect(result.assistant).toBe('hello from pool');
expect(adapter.infer).toHaveBeenCalledTimes(1);
});
it('chat fails over to the next pool member when the first throws on dispatch', async () => {
// 3 members; first 2 throw, 3rd succeeds. Verify exactly 3 dispatches
// and the final reply propagates.
const chatRepo = mockChatRepo();
let call = 0;
const adapter: LlmAdapter = {
kind: 'flaky',
infer: vi.fn(async () => {
call += 1;
if (call < 3) throw new Error(`transport-error-${String(call)}`);
return chatCompletion('survived to the third try').body !== undefined
? { status: 200, body: chatCompletion('survived to the third try').body }
: (() => { throw new Error('unreachable'); })();
}),
stream: async function*() { yield { data: '[DONE]', done: true }; },
};
const svc = new ChatService(
mockAgents(),
mockLlmsPool({
pinnedName: 'qwen-prod-1',
poolName: 'qwen-pool',
members: [
{ name: 'qwen-prod-1' },
{ name: 'qwen-prod-2' },
{ name: 'qwen-prod-3' },
],
}),
adapterRegistry(adapter),
chatRepo, mockPromptRepo(), mockTools(),
);
const result = await svc.chat({ agentName: 'reviewer', userMessage: 'hi', ownerId: 'owner-1' });
expect(result.assistant).toBe('survived to the third try');
expect(adapter.infer).toHaveBeenCalledTimes(3);
});
it('chat throws when every pool member throws (exhausted)', async () => {
const chatRepo = mockChatRepo();
const adapter: LlmAdapter = {
kind: 'all-broken',
infer: vi.fn(async () => { throw new Error('transport-down'); }),
stream: async function*() { yield { data: '[DONE]', done: true }; },
};
const svc = new ChatService(
mockAgents(),
mockLlmsPool({
pinnedName: 'qwen-prod-1',
poolName: 'qwen-pool',
members: [{ name: 'qwen-prod-1' }, { name: 'qwen-prod-2' }],
}),
adapterRegistry(adapter),
chatRepo, mockPromptRepo(), mockTools(),
);
await expect(svc.chat({ agentName: 'reviewer', userMessage: 'hi', ownerId: 'owner-1' }))
.rejects.toThrow(/transport-down/);
expect(adapter.infer).toHaveBeenCalledTimes(2);
});
it('chat refuses with 404 when every pool member is inactive', async () => {
const chatRepo = mockChatRepo();
const adapter = scriptedAdapter([chatCompletion('should never run')]);
const svc = new ChatService(
mockAgents(),
mockLlmsPool({
pinnedName: 'qwen-prod-1',
poolName: 'qwen-pool',
members: [
{ name: 'qwen-prod-1', status: 'inactive' },
{ name: 'qwen-prod-2', status: 'inactive' },
],
}),
adapterRegistry(adapter),
chatRepo, mockPromptRepo(), mockTools(),
);
await expect(svc.chat({ agentName: 'reviewer', userMessage: 'hi', ownerId: 'owner-1' }))
.rejects.toThrow(/No active Llm in pool/);
expect(adapter.infer).not.toHaveBeenCalled();
});
it('chat picks a healthy sibling when the pinned Llm is itself inactive', async () => {
// The agent pins to qwen-prod-1 which is inactive. qwen-prod-2 is
// active. Pool dispatch must skip the dead pinned row and use the
// sibling instead — that's the whole point of v4.
const chatRepo = mockChatRepo();
const adapter = scriptedAdapter([chatCompletion('via sibling')]);
const svc = new ChatService(
mockAgents(),
mockLlmsPool({
pinnedName: 'qwen-prod-1',
poolName: 'qwen-pool',
members: [
{ name: 'qwen-prod-1', status: 'inactive' },
{ name: 'qwen-prod-2', status: 'active' },
],
}),
adapterRegistry(adapter),
chatRepo, mockPromptRepo(), mockTools(),
);
const result = await svc.chat({ agentName: 'reviewer', userMessage: 'hi', ownerId: 'owner-1' });
expect(result.assistant).toBe('via sibling');
expect(adapter.infer).toHaveBeenCalledTimes(1);
});
// Regression: per-agent maxIterations override + clamp.
// Found by /gstack-review on 2026-04-25.
// Without the clamp, a hostile agent definition with `extras.maxIterations:1000000`