feat(chat): LLM failover chain + show which model answered
Some checks failed
CI/CD / typecheck (pull_request) Successful in 1m4s
CI/CD / lint (pull_request) Successful in 2m14s
CI/CD / test (pull_request) Successful in 1m18s
CI/CD / build (pull_request) Successful in 2m30s
CI/CD / smoke (pull_request) Failing after 3m18s
CI/CD / publish (pull_request) Has been skipped
Some checks failed
CI/CD / typecheck (pull_request) Successful in 1m4s
CI/CD / lint (pull_request) Successful in 2m14s
CI/CD / test (pull_request) Successful in 1m18s
CI/CD / build (pull_request) Successful in 2m30s
CI/CD / smoke (pull_request) Failing after 3m18s
CI/CD / publish (pull_request) Has been skipped
Chat is LLM-essential but not model-specific — instead of failing when the
pinned model is down/drifted, it now fails over across an ordered chain and
reports which model actually answered.
- Ordered fallback: an Llm declares `extraConfig.fallbacks: string[]`; the
dispatcher builds primary-pool → fallback-pool(s) candidates and tries them
in order (resolveCandidatesWithFallbacks).
- Fail over on real failures, not just transport: runOneInference now advances
on a non-2xx status (e.g. a 400 from a drifted model) or an empty/invalid
completion, not only thrown transport errors. Streaming fails over
pre-first-chunk (already threw on 4xx).
- Transparency: ChatResult + the SSE `final` frame carry {llm, model,
failedOver}; the CLI prints `model: <llm> (<model>)` each turn and
`⚠ failed over → answered by …` when a fallback was used.
- Exhaustion names the last model + upstream body (not "no choice").
Tests: 3 failover unit tests (primary 400 → fallback answers + model reported;
primary answers → failedOver=false; all fail → clear aggregated error).
mcpd 948 + CLI 508 green; tsc + lint clean. docs/reliability.md updated.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1080,3 +1080,64 @@ function mockPersonalityRepo(
|
||||
findBinding: vi.fn(async () => null),
|
||||
};
|
||||
}
|
||||
|
||||
// ── Failover chain + "which model answered" reporting ──
|
||||
describe('ChatService — failover chain', () => {
|
||||
const NOW2 = new Date();
|
||||
function llmRow(name: string, model: string, fallbacks: string[] = []): Record<string, unknown> {
|
||||
return {
|
||||
id: `id-${name}`, name, type: 'openai', model, url: '', tier: 'fast', description: '',
|
||||
apiKeySecretId: null, apiKeySecretKey: null,
|
||||
extraConfig: fallbacks.length > 0 ? { fallbacks } : {},
|
||||
poolName: null, kind: 'public', providerSessionId: null, status: 'active',
|
||||
lastHeartbeatAt: null, inactiveSince: null, version: 1, createdAt: NOW2, updatedAt: NOW2,
|
||||
};
|
||||
}
|
||||
function failoverLlms(): LlmService {
|
||||
const rows: Record<string, Record<string, unknown>> = {
|
||||
'primary-llm': llmRow('primary-llm', 'primary-model', ['fallback-llm']),
|
||||
'fallback-llm': llmRow('fallback-llm', 'fallback-model'),
|
||||
};
|
||||
return {
|
||||
getByName: vi.fn(async (name: string) => ({ ...rows[name], apiKeyRef: null })),
|
||||
findByPoolName: vi.fn(async (pool: string) => [rows[pool]]),
|
||||
resolveApiKey: vi.fn(async () => 'k'),
|
||||
} as unknown as LlmService;
|
||||
}
|
||||
function agentPinnedTo(llmName: string): AgentService {
|
||||
return {
|
||||
getByName: vi.fn(async (name: string) => ({
|
||||
id: `agent-${name}`, name, description: '', systemPrompt: 'sys',
|
||||
llm: { id: 'x', name: llmName }, project: null, defaultPersonality: null,
|
||||
proxyModelName: null, defaultParams: {}, extras: {}, ownerId: 'o', version: 1, createdAt: NOW2, updatedAt: NOW2,
|
||||
})),
|
||||
} as unknown as AgentService;
|
||||
}
|
||||
const status400 = (): NonStreamingResult => ({ status: 400, body: { error: 'model not served' } });
|
||||
|
||||
it('fails over to the fallback Llm on a primary 400 and reports the answering model', async () => {
|
||||
const adapter = scriptedAdapter([status400(), chatCompletion('fallback answer')]);
|
||||
const svc = new ChatService(agentPinnedTo('primary-llm'), failoverLlms(), adapterRegistry(adapter), mockChatRepo(), mockPromptRepo(), mockTools());
|
||||
const res = await svc.chat({ agentName: 'a', userMessage: 'hi', ownerId: 'o' });
|
||||
expect(res.assistant).toBe('fallback answer');
|
||||
expect(res.failedOver).toBe(true);
|
||||
expect(res.llm).toBe('fallback-llm');
|
||||
expect(res.model).toBe('fallback-model');
|
||||
});
|
||||
|
||||
it('reports the primary model + failedOver=false when the primary answers', async () => {
|
||||
const adapter = scriptedAdapter([chatCompletion('primary answer')]);
|
||||
const svc = new ChatService(agentPinnedTo('primary-llm'), failoverLlms(), adapterRegistry(adapter), mockChatRepo(), mockPromptRepo(), mockTools());
|
||||
const res = await svc.chat({ agentName: 'a', userMessage: 'hi', ownerId: 'o' });
|
||||
expect(res.assistant).toBe('primary answer');
|
||||
expect(res.failedOver).toBe(false);
|
||||
expect(res.llm).toBe('primary-llm');
|
||||
expect(res.model).toBe('primary-model');
|
||||
});
|
||||
|
||||
it('throws a clear aggregated error naming the model when every candidate fails', async () => {
|
||||
const adapter = scriptedAdapter([status400(), status400()]);
|
||||
const svc = new ChatService(agentPinnedTo('primary-llm'), failoverLlms(), adapterRegistry(adapter), mockChatRepo(), mockPromptRepo(), mockTools());
|
||||
await expect(svc.chat({ agentName: 'a', userMessage: 'hi', ownerId: 'o' })).rejects.toThrow(/HTTP 400/);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user