feat: LLM provider failover in proxymodel adapter
LLMProviderAdapter now tries all registered providers before giving up: 1. Named provider (if specified) 2. All 'fast' tier providers in order 3. All 'heavy' tier providers in order 4. Legacy active provider Previously, if the first provider (e.g., vllm-local) failed, the adapter threw immediately even though Anthropic and Gemini were available. Now it logs the failure and tries the next candidate. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -17,6 +17,15 @@ function mockProvider(name: string, response = 'mock response'): LlmProvider {
|
||||
};
|
||||
}
|
||||
|
||||
function failingProvider(name: string, error = 'connection refused'): LlmProvider {
|
||||
return {
|
||||
name,
|
||||
complete: vi.fn().mockRejectedValue(new Error(error)),
|
||||
listModels: vi.fn().mockResolvedValue([]),
|
||||
isAvailable: vi.fn().mockResolvedValue(true),
|
||||
};
|
||||
}
|
||||
|
||||
describe('LLMProviderAdapter', () => {
|
||||
it('available() returns true when a provider is registered', () => {
|
||||
const registry = new ProviderRegistry();
|
||||
@@ -45,7 +54,6 @@ describe('LLMProviderAdapter', () => {
|
||||
expect(result).toBe('mock response');
|
||||
expect(provider.complete).toHaveBeenCalledWith({
|
||||
messages: [{ role: 'user', content: 'summarize this' }],
|
||||
maxTokens: undefined,
|
||||
temperature: 0,
|
||||
});
|
||||
});
|
||||
@@ -75,4 +83,87 @@ describe('LLMProviderAdapter', () => {
|
||||
|
||||
await expect(adapter.complete('test')).rejects.toThrow('No LLM provider available');
|
||||
});
|
||||
|
||||
// --- Failover tests ---
|
||||
|
||||
it('falls back to next provider in same tier on failure', async () => {
|
||||
const failing = failingProvider('vllm-local', 'vLLM startup timed out');
|
||||
const working = mockProvider('anthropic', 'anthropic response');
|
||||
const registry = new ProviderRegistry();
|
||||
registry.register(failing);
|
||||
registry.register(working);
|
||||
registry.assignTier('vllm-local', 'fast');
|
||||
registry.assignTier('anthropic', 'fast');
|
||||
|
||||
const adapter = new LLMProviderAdapter(registry);
|
||||
const result = await adapter.complete('test');
|
||||
|
||||
expect(result).toBe('anthropic response');
|
||||
expect(failing.complete).toHaveBeenCalledOnce();
|
||||
expect(working.complete).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('falls back cross-tier when all fast providers fail', async () => {
|
||||
const fastFail = failingProvider('vllm-local');
|
||||
const heavy = mockProvider('gemini', 'gemini response');
|
||||
const registry = new ProviderRegistry();
|
||||
registry.register(fastFail);
|
||||
registry.register(heavy);
|
||||
registry.assignTier('vllm-local', 'fast');
|
||||
registry.assignTier('gemini', 'heavy');
|
||||
|
||||
const adapter = new LLMProviderAdapter(registry);
|
||||
const result = await adapter.complete('test');
|
||||
|
||||
expect(result).toBe('gemini response');
|
||||
expect(fastFail.complete).toHaveBeenCalledOnce();
|
||||
expect(heavy.complete).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('throws last error when all providers fail', async () => {
|
||||
const fail1 = failingProvider('vllm', 'vLLM down');
|
||||
const fail2 = failingProvider('anthropic', 'rate limited');
|
||||
const registry = new ProviderRegistry();
|
||||
registry.register(fail1);
|
||||
registry.register(fail2);
|
||||
registry.assignTier('vllm', 'fast');
|
||||
registry.assignTier('anthropic', 'heavy');
|
||||
|
||||
const adapter = new LLMProviderAdapter(registry);
|
||||
await expect(adapter.complete('test')).rejects.toThrow('rate limited');
|
||||
expect(fail1.complete).toHaveBeenCalledOnce();
|
||||
expect(fail2.complete).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('does not retry provider that already succeeded', async () => {
|
||||
const fast = mockProvider('fast-provider', 'fast result');
|
||||
const heavy = mockProvider('heavy-provider', 'heavy result');
|
||||
const registry = new ProviderRegistry();
|
||||
registry.register(fast);
|
||||
registry.register(heavy);
|
||||
registry.assignTier('fast-provider', 'fast');
|
||||
registry.assignTier('heavy-provider', 'heavy');
|
||||
|
||||
const adapter = new LLMProviderAdapter(registry);
|
||||
const result = await adapter.complete('test');
|
||||
|
||||
expect(result).toBe('fast result');
|
||||
expect(fast.complete).toHaveBeenCalledOnce();
|
||||
expect(heavy.complete).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('prefers named provider but falls back on failure', async () => {
|
||||
const named = failingProvider('preferred', 'preferred down');
|
||||
const fallback = mockProvider('fallback', 'fallback response');
|
||||
const registry = new ProviderRegistry();
|
||||
registry.register(named);
|
||||
registry.register(fallback);
|
||||
registry.assignTier('preferred', 'fast');
|
||||
registry.assignTier('fallback', 'fast');
|
||||
|
||||
const adapter = new LLMProviderAdapter(registry, 'preferred');
|
||||
const result = await adapter.complete('test');
|
||||
|
||||
expect(result).toBe('fallback response');
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user