feat: LLM provider failover in proxymodel adapter

LLMProviderAdapter now tries all registered providers before giving up:
  1. Named provider (if specified)
  2. All 'fast' tier providers in order
  3. All 'heavy' tier providers in order
  4. Legacy active provider

Previously, if the first provider (e.g., vllm-local) failed, the adapter
threw immediately even though Anthropic and Gemini were available. Now it
logs the failure and tries the next candidate.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Michal
2026-03-03 22:04:58 +00:00
parent 03827f11e4
commit 4cfdd805d8
2 changed files with 163 additions and 20 deletions

View File

@@ -17,6 +17,15 @@ function mockProvider(name: string, response = 'mock response'): LlmProvider {
};
}
function failingProvider(name: string, error = 'connection refused'): LlmProvider {
return {
name,
complete: vi.fn().mockRejectedValue(new Error(error)),
listModels: vi.fn().mockResolvedValue([]),
isAvailable: vi.fn().mockResolvedValue(true),
};
}
describe('LLMProviderAdapter', () => {
it('available() returns true when a provider is registered', () => {
const registry = new ProviderRegistry();
@@ -45,7 +54,6 @@ describe('LLMProviderAdapter', () => {
expect(result).toBe('mock response');
expect(provider.complete).toHaveBeenCalledWith({
messages: [{ role: 'user', content: 'summarize this' }],
maxTokens: undefined,
temperature: 0,
});
});
@@ -75,4 +83,87 @@ describe('LLMProviderAdapter', () => {
await expect(adapter.complete('test')).rejects.toThrow('No LLM provider available');
});
// --- Failover tests ---
it('falls back to next provider in same tier on failure', async () => {
const failing = failingProvider('vllm-local', 'vLLM startup timed out');
const working = mockProvider('anthropic', 'anthropic response');
const registry = new ProviderRegistry();
registry.register(failing);
registry.register(working);
registry.assignTier('vllm-local', 'fast');
registry.assignTier('anthropic', 'fast');
const adapter = new LLMProviderAdapter(registry);
const result = await adapter.complete('test');
expect(result).toBe('anthropic response');
expect(failing.complete).toHaveBeenCalledOnce();
expect(working.complete).toHaveBeenCalledOnce();
});
it('falls back cross-tier when all fast providers fail', async () => {
const fastFail = failingProvider('vllm-local');
const heavy = mockProvider('gemini', 'gemini response');
const registry = new ProviderRegistry();
registry.register(fastFail);
registry.register(heavy);
registry.assignTier('vllm-local', 'fast');
registry.assignTier('gemini', 'heavy');
const adapter = new LLMProviderAdapter(registry);
const result = await adapter.complete('test');
expect(result).toBe('gemini response');
expect(fastFail.complete).toHaveBeenCalledOnce();
expect(heavy.complete).toHaveBeenCalledOnce();
});
it('throws last error when all providers fail', async () => {
const fail1 = failingProvider('vllm', 'vLLM down');
const fail2 = failingProvider('anthropic', 'rate limited');
const registry = new ProviderRegistry();
registry.register(fail1);
registry.register(fail2);
registry.assignTier('vllm', 'fast');
registry.assignTier('anthropic', 'heavy');
const adapter = new LLMProviderAdapter(registry);
await expect(adapter.complete('test')).rejects.toThrow('rate limited');
expect(fail1.complete).toHaveBeenCalledOnce();
expect(fail2.complete).toHaveBeenCalledOnce();
});
it('does not retry provider that already succeeded', async () => {
const fast = mockProvider('fast-provider', 'fast result');
const heavy = mockProvider('heavy-provider', 'heavy result');
const registry = new ProviderRegistry();
registry.register(fast);
registry.register(heavy);
registry.assignTier('fast-provider', 'fast');
registry.assignTier('heavy-provider', 'heavy');
const adapter = new LLMProviderAdapter(registry);
const result = await adapter.complete('test');
expect(result).toBe('fast result');
expect(fast.complete).toHaveBeenCalledOnce();
expect(heavy.complete).not.toHaveBeenCalled();
});
it('prefers named provider but falls back on failure', async () => {
const named = failingProvider('preferred', 'preferred down');
const fallback = mockProvider('fallback', 'fallback response');
const registry = new ProviderRegistry();
registry.register(named);
registry.register(fallback);
registry.assignTier('preferred', 'fast');
registry.assignTier('fallback', 'fast');
const adapter = new LLMProviderAdapter(registry, 'preferred');
const result = await adapter.complete('test');
expect(result).toBe('fallback response');
});
});