feat(mcpd+db): Llm.poolName + chat dispatcher pool failover (v4 Stage 1)
Adds LB-pool-by-shared-name without introducing a new resource. The existing `Llm.name` stays globally unique; a new optional `poolName` column declares membership in a pool. Multiple Llms sharing a non-null `poolName` stack into one load-balanced pool that the chat dispatcher expands at request time. Effective pool key = `poolName ?? name`. Solo rows (poolName=null) are addressable as a "pool of 1" via their own name, so existing single-Llm agents and YAMLs keep working unchanged. A solo row whose name happens to match an explicit poolName joins the same pool — by design — so an operator can transparently promote an existing Llm to pool seed. Dispatcher (chat.service): prepareContext now resolves a randomly- shuffled list of viable pool candidates (status != inactive) once per turn. runOneInference and streamInference iterate the list on transport-level failure (network, virtual publisher disconnect) until one succeeds or the list is exhausted. Streaming failover only covers "failed before first chunk" — once we've yielded text, we're committed to that backend. Auth/4xx errors surfaced as result.status are NOT retried; siblings with the same key/model would fail identically. When the agent's pinned Llm is itself inactive but a sibling pool member is up, dispatch transparently uses the sibling — that's the whole point. When every member is inactive, prepareContext throws a clear "No active Llm in pool '<key>' (pinned: <name>)" error rather than letting the dispatcher's "exhausted" branch surface it. Tests: - 5 new chat-service tests for pool dispatch / failover / pinned-down / all-inactive (chat-service.test.ts). - 7 new db schema tests for the column, the unique-name invariant, the fallback-to-name semantics, and the solo-name-joins-explicit-pool edge case (llm-pool-schema.test.ts). - mcpd 865/865 (was 860; +5), db pool-schema 7/7, no regressions. Stage 2 (next): HTTP route /api/v1/llms/<name>/members + aggregate pool stats on the existing single-Llm route, CLI POOL column + describe block + --pool-name flag, yaml round-trip.
This commit is contained in:
@@ -71,17 +71,25 @@ function mockAgents(): AgentService {
|
||||
}
|
||||
|
||||
function mockLlmsVirtual(): LlmService {
|
||||
const baseRow = (name: string): Record<string, unknown> => ({
|
||||
id: 'llm-1', name, type: 'openai', model: 'fake',
|
||||
url: '', tier: 'fast', description: '',
|
||||
apiKeySecretId: null, apiKeySecretKey: null,
|
||||
extraConfig: {},
|
||||
poolName: null,
|
||||
kind: 'virtual',
|
||||
providerSessionId: null,
|
||||
status: 'active',
|
||||
lastHeartbeatAt: NOW,
|
||||
inactiveSince: null,
|
||||
version: 1, createdAt: NOW, updatedAt: NOW,
|
||||
});
|
||||
return {
|
||||
getByName: vi.fn(async (name: string) => ({
|
||||
id: 'llm-1', name, type: 'openai', model: 'fake',
|
||||
url: '', tier: 'fast', description: '',
|
||||
apiKeyRef: null, extraConfig: {},
|
||||
kind: 'virtual',
|
||||
status: 'active',
|
||||
lastHeartbeatAt: NOW,
|
||||
inactiveSince: null,
|
||||
version: 1, createdAt: NOW, updatedAt: NOW,
|
||||
...baseRow(name),
|
||||
apiKeyRef: null,
|
||||
})),
|
||||
findByPoolName: vi.fn(async (poolName: string) => [baseRow(poolName)]),
|
||||
resolveApiKey: vi.fn(async () => ''),
|
||||
} as unknown as LlmService;
|
||||
}
|
||||
|
||||
@@ -119,17 +119,30 @@ function mockAgents(opts: { defaultPersonality?: { id: string; name: string } |
|
||||
}
|
||||
|
||||
function mockLlms(opts: { kind?: 'public' | 'virtual' } = {}): LlmService {
|
||||
// v4: prepareContext now resolves a pool by effective key. For unit
|
||||
// tests that pass a single agent.llm we return that one row twice —
|
||||
// once for getByName (LlmView shape) and once for findByPoolName (raw
|
||||
// Llm shape with the same name) so the dispatcher's poolCandidates
|
||||
// ends up with exactly one member, matching pre-v4 behavior.
|
||||
const baseRow = (name: string): Record<string, unknown> => ({
|
||||
id: 'llm-1', name, type: 'openai', model: 'qwen3-thinking',
|
||||
url: '', tier: 'fast', description: '',
|
||||
apiKeySecretId: null, apiKeySecretKey: null,
|
||||
extraConfig: {},
|
||||
poolName: null,
|
||||
kind: opts.kind ?? 'public',
|
||||
providerSessionId: null,
|
||||
status: 'active',
|
||||
lastHeartbeatAt: null,
|
||||
inactiveSince: null,
|
||||
version: 1, createdAt: NOW, updatedAt: NOW,
|
||||
});
|
||||
return {
|
||||
getByName: vi.fn(async (name: string) => ({
|
||||
id: 'llm-1', name, type: 'openai', model: 'qwen3-thinking',
|
||||
url: '', tier: 'fast', description: '',
|
||||
apiKeyRef: null, extraConfig: {},
|
||||
kind: opts.kind ?? 'public',
|
||||
status: 'active',
|
||||
lastHeartbeatAt: null,
|
||||
inactiveSince: null,
|
||||
version: 1, createdAt: NOW, updatedAt: NOW,
|
||||
...baseRow(name),
|
||||
apiKeyRef: null,
|
||||
})),
|
||||
findByPoolName: vi.fn(async (poolName: string) => [baseRow(poolName)]),
|
||||
resolveApiKey: vi.fn(async () => 'fake-key'),
|
||||
} as unknown as LlmService;
|
||||
}
|
||||
@@ -604,6 +617,176 @@ describe('ChatService', () => {
|
||||
.toEqual(['thinking via litellm...']);
|
||||
});
|
||||
|
||||
// ── v4: LB pool by shared name ──
|
||||
|
||||
// Helper: build a multi-member mock LlmService where all members share an
|
||||
// effective pool key. `nameToFail` lets a test mark specific names as
|
||||
// throwing on dispatch, exercising failover.
|
||||
function mockLlmsPool(opts: {
|
||||
pinnedName: string;
|
||||
poolName: string | null;
|
||||
members: Array<{ name: string; status?: 'active' | 'inactive'; kind?: 'public' | 'virtual' }>;
|
||||
}): LlmService {
|
||||
const baseRow = (m: { name: string; status?: 'active' | 'inactive'; kind?: 'public' | 'virtual' }): Record<string, unknown> => ({
|
||||
id: `id-${m.name}`,
|
||||
name: m.name,
|
||||
type: 'openai',
|
||||
model: 'qwen3-thinking',
|
||||
url: '',
|
||||
tier: 'fast',
|
||||
description: '',
|
||||
apiKeySecretId: null,
|
||||
apiKeySecretKey: null,
|
||||
extraConfig: {},
|
||||
poolName: opts.poolName,
|
||||
kind: m.kind ?? 'public',
|
||||
providerSessionId: null,
|
||||
status: m.status ?? 'active',
|
||||
lastHeartbeatAt: null,
|
||||
inactiveSince: null,
|
||||
version: 1,
|
||||
createdAt: NOW,
|
||||
updatedAt: NOW,
|
||||
});
|
||||
return {
|
||||
getByName: vi.fn(async (name: string) => {
|
||||
const m = opts.members.find((x) => x.name === name) ?? opts.members[0]!;
|
||||
return { ...baseRow(m), apiKeyRef: null };
|
||||
}),
|
||||
findByPoolName: vi.fn(async () => opts.members.map(baseRow)),
|
||||
resolveApiKey: vi.fn(async () => 'fake-key'),
|
||||
} as unknown as LlmService;
|
||||
}
|
||||
|
||||
it('chat dispatches to a pool member and persists the reply (pool size N, primary returns)', async () => {
|
||||
// Three healthy members; the (random) primary wins on first try.
|
||||
// Adapter returns the same canned reply regardless of which member
|
||||
// got picked because in this test we don't differentiate by name —
|
||||
// we just assert that dispatch goes through and the agent gets a
|
||||
// reply. Per-member assertion is covered by the failover test below.
|
||||
const chatRepo = mockChatRepo();
|
||||
const adapter = scriptedAdapter([chatCompletion('hello from pool')]);
|
||||
const svc = new ChatService(
|
||||
mockAgents(),
|
||||
mockLlmsPool({
|
||||
pinnedName: 'qwen-prod-1',
|
||||
poolName: 'qwen-pool',
|
||||
members: [
|
||||
{ name: 'qwen-prod-1' },
|
||||
{ name: 'qwen-prod-2' },
|
||||
{ name: 'qwen-prod-3' },
|
||||
],
|
||||
}),
|
||||
adapterRegistry(adapter),
|
||||
chatRepo, mockPromptRepo(), mockTools(),
|
||||
);
|
||||
const result = await svc.chat({ agentName: 'reviewer', userMessage: 'hi', ownerId: 'owner-1' });
|
||||
expect(result.assistant).toBe('hello from pool');
|
||||
expect(adapter.infer).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('chat fails over to the next pool member when the first throws on dispatch', async () => {
|
||||
// 3 members; first 2 throw, 3rd succeeds. Verify exactly 3 dispatches
|
||||
// and the final reply propagates.
|
||||
const chatRepo = mockChatRepo();
|
||||
let call = 0;
|
||||
const adapter: LlmAdapter = {
|
||||
kind: 'flaky',
|
||||
infer: vi.fn(async () => {
|
||||
call += 1;
|
||||
if (call < 3) throw new Error(`transport-error-${String(call)}`);
|
||||
return chatCompletion('survived to the third try').body !== undefined
|
||||
? { status: 200, body: chatCompletion('survived to the third try').body }
|
||||
: (() => { throw new Error('unreachable'); })();
|
||||
}),
|
||||
stream: async function*() { yield { data: '[DONE]', done: true }; },
|
||||
};
|
||||
const svc = new ChatService(
|
||||
mockAgents(),
|
||||
mockLlmsPool({
|
||||
pinnedName: 'qwen-prod-1',
|
||||
poolName: 'qwen-pool',
|
||||
members: [
|
||||
{ name: 'qwen-prod-1' },
|
||||
{ name: 'qwen-prod-2' },
|
||||
{ name: 'qwen-prod-3' },
|
||||
],
|
||||
}),
|
||||
adapterRegistry(adapter),
|
||||
chatRepo, mockPromptRepo(), mockTools(),
|
||||
);
|
||||
const result = await svc.chat({ agentName: 'reviewer', userMessage: 'hi', ownerId: 'owner-1' });
|
||||
expect(result.assistant).toBe('survived to the third try');
|
||||
expect(adapter.infer).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it('chat throws when every pool member throws (exhausted)', async () => {
|
||||
const chatRepo = mockChatRepo();
|
||||
const adapter: LlmAdapter = {
|
||||
kind: 'all-broken',
|
||||
infer: vi.fn(async () => { throw new Error('transport-down'); }),
|
||||
stream: async function*() { yield { data: '[DONE]', done: true }; },
|
||||
};
|
||||
const svc = new ChatService(
|
||||
mockAgents(),
|
||||
mockLlmsPool({
|
||||
pinnedName: 'qwen-prod-1',
|
||||
poolName: 'qwen-pool',
|
||||
members: [{ name: 'qwen-prod-1' }, { name: 'qwen-prod-2' }],
|
||||
}),
|
||||
adapterRegistry(adapter),
|
||||
chatRepo, mockPromptRepo(), mockTools(),
|
||||
);
|
||||
await expect(svc.chat({ agentName: 'reviewer', userMessage: 'hi', ownerId: 'owner-1' }))
|
||||
.rejects.toThrow(/transport-down/);
|
||||
expect(adapter.infer).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('chat refuses with 404 when every pool member is inactive', async () => {
|
||||
const chatRepo = mockChatRepo();
|
||||
const adapter = scriptedAdapter([chatCompletion('should never run')]);
|
||||
const svc = new ChatService(
|
||||
mockAgents(),
|
||||
mockLlmsPool({
|
||||
pinnedName: 'qwen-prod-1',
|
||||
poolName: 'qwen-pool',
|
||||
members: [
|
||||
{ name: 'qwen-prod-1', status: 'inactive' },
|
||||
{ name: 'qwen-prod-2', status: 'inactive' },
|
||||
],
|
||||
}),
|
||||
adapterRegistry(adapter),
|
||||
chatRepo, mockPromptRepo(), mockTools(),
|
||||
);
|
||||
await expect(svc.chat({ agentName: 'reviewer', userMessage: 'hi', ownerId: 'owner-1' }))
|
||||
.rejects.toThrow(/No active Llm in pool/);
|
||||
expect(adapter.infer).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('chat picks a healthy sibling when the pinned Llm is itself inactive', async () => {
|
||||
// The agent pins to qwen-prod-1 which is inactive. qwen-prod-2 is
|
||||
// active. Pool dispatch must skip the dead pinned row and use the
|
||||
// sibling instead — that's the whole point of v4.
|
||||
const chatRepo = mockChatRepo();
|
||||
const adapter = scriptedAdapter([chatCompletion('via sibling')]);
|
||||
const svc = new ChatService(
|
||||
mockAgents(),
|
||||
mockLlmsPool({
|
||||
pinnedName: 'qwen-prod-1',
|
||||
poolName: 'qwen-pool',
|
||||
members: [
|
||||
{ name: 'qwen-prod-1', status: 'inactive' },
|
||||
{ name: 'qwen-prod-2', status: 'active' },
|
||||
],
|
||||
}),
|
||||
adapterRegistry(adapter),
|
||||
chatRepo, mockPromptRepo(), mockTools(),
|
||||
);
|
||||
const result = await svc.chat({ agentName: 'reviewer', userMessage: 'hi', ownerId: 'owner-1' });
|
||||
expect(result.assistant).toBe('via sibling');
|
||||
expect(adapter.infer).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
// Regression: per-agent maxIterations override + clamp.
|
||||
// Found by /gstack-review on 2026-04-25.
|
||||
// Without the clamp, a hostile agent definition with `extras.maxIterations:1000000`
|
||||
|
||||
Reference in New Issue
Block a user