feat(chat): LLM failover chain + show which model answered
Some checks failed
CI/CD / typecheck (pull_request) Successful in 1m4s
CI/CD / lint (pull_request) Successful in 2m14s
CI/CD / test (pull_request) Successful in 1m18s
CI/CD / build (pull_request) Successful in 2m30s
CI/CD / smoke (pull_request) Failing after 3m18s
CI/CD / publish (pull_request) Has been skipped

Chat is LLM-essential but not model-specific — instead of failing when the
pinned model is down/drifted, it now fails over across an ordered chain and
reports which model actually answered.

- Ordered fallback: an Llm declares `extraConfig.fallbacks: string[]`; the
  dispatcher builds primary-pool → fallback-pool(s) candidates and tries them
  in order (resolveCandidatesWithFallbacks).
- Fail over on real failures, not just transport: runOneInference now advances
  on a non-2xx status (e.g. a 400 from a drifted model) or an empty/invalid
  completion, not only thrown transport errors. Streaming fails over
  pre-first-chunk (already threw on 4xx).
- Transparency: ChatResult + the SSE `final` frame carry {llm, model,
  failedOver}; the CLI prints `model: <llm> (<model>)` each turn and
  `⚠ failed over → answered by …` when a fallback was used.
- Exhaustion names the last model + upstream body (not "no choice").

Tests: 3 failover unit tests (primary 400 → fallback answers + model reported;
primary answers → failedOver=false; all fail → clear aggregated error).
mcpd 948 + CLI 508 green; tsc + lint clean. docs/reliability.md updated.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Michal
2026-07-21 23:39:56 +01:00
parent 3465596635
commit 18524c1c57
4 changed files with 194 additions and 24 deletions

View File

@@ -198,7 +198,8 @@ async function runOneShot(
// `thread: <id>` — single space after the colon, matching the streaming
// path (line 160 below) so any tooling/regex that watches one form picks
// up the other too.
process.stderr.write(styleStats(`(${String(words)}w · ${(words / sec).toFixed(1)} w/s · ${sec.toFixed(1)}s)`) + ` thread: ${res.threadId}\n`);
const answered = formatAnswered(res.llm, res.model, res.failedOver);
process.stderr.write(styleStats(`(${String(words)}w · ${(words / sec).toFixed(1)} w/s · ${sec.toFixed(1)}s)`) + ` thread: ${res.threadId}${answered !== '' ? ` ${answered}` : ''}\n`);
return;
}
const bar = installStatusBar();
@@ -258,6 +259,8 @@ async function runRepl(
const res = await chatRequestNonStream(deps, subject, body);
threadId = res.threadId;
process.stdout.write(`${res.assistant}\n`);
const answered = formatAnswered(res.llm, res.model, res.failedOver);
if (answered !== '') process.stderr.write(`${styleStats(`(${answered})`)}\n`);
} else {
threadId = await streamOnce(deps, subject, line, threadId, overrides, bar);
process.stdout.write('\n');
@@ -455,7 +458,7 @@ async function chatRequestNonStream(
deps: ChatCommandDeps,
subject: ChatSubject,
body: Record<string, unknown>,
): Promise<{ assistant: string; threadId: string; turnIndex: number }> {
): Promise<{ assistant: string; threadId: string; turnIndex: number; llm?: string; model?: string; failedOver?: boolean }> {
const url = new URL(`${deps.baseUrl}/api/v1/${subject.path}/chat`);
const payload = JSON.stringify(body);
return new Promise((resolve, reject) => {
@@ -482,7 +485,7 @@ async function chatRequestNonStream(
return;
}
try {
resolve(JSON.parse(raw) as { assistant: string; threadId: string; turnIndex: number });
resolve(JSON.parse(raw) as { assistant: string; threadId: string; turnIndex: number; llm?: string; model?: string; failedOver?: boolean });
} catch (err) {
reject(new Error(`malformed response: ${(err as Error).message}`));
}
@@ -550,6 +553,7 @@ async function streamOnce(
}
let buf = '';
let resolvedThread = threadId ?? '';
let answered = '';
res.setEncoding('utf-8');
res.on('data', (chunk: string) => {
buf += chunk;
@@ -590,6 +594,7 @@ async function streamOnce(
break;
case 'final':
if (evt.threadId !== undefined) resolvedThread = evt.threadId;
answered = formatAnswered(evt.llm, evt.model, evt.failedOver);
break;
case 'error':
process.stderr.write(`\n[error: ${evt.message ?? ''}]\n`);
@@ -606,10 +611,11 @@ async function streamOnce(
const final = formatStats(stats, false);
// Inline final-stats footer (permanent record): goes through the
// scroll region, so a copy-pasted transcript captures it.
if (final !== '' && STDERR_IS_TTY) {
process.stderr.write(`\n${styleStats(`(${final})`)}`);
} else if (final !== '') {
process.stderr.write(`\n(${final})`);
const footer = [final, answered].filter((s) => s !== '').join(' · ');
if (footer !== '' && STDERR_IS_TTY) {
process.stderr.write(`\n${styleStats(`(${footer})`)}`);
} else if (footer !== '') {
process.stderr.write(`\n(${footer})`);
}
// Live status bar: pin the final value so it stays visible between
// turns (answers "how fast was the last one?").
@@ -637,6 +643,16 @@ interface ChatStreamFrame {
threadId?: string;
turnIndex?: number;
message?: string;
llm?: string;
model?: string;
failedOver?: boolean;
}
/** One-line "which model answered" footer, dim; loud when it failed over. */
function formatAnswered(llm?: string, model?: string, failedOver?: boolean): string {
if (llm === undefined && model === undefined) return '';
const label = model !== undefined && model !== '' ? `${llm ?? '?'} (${model})` : (llm ?? '?');
return failedOver === true ? `⚠ failed over → answered by ${label}` : `model: ${label}`;
}
// ANSI codes for the reasoning sidebar. Dim + italic visually separates