feat(chat): LLM failover chain + show which model answered
Some checks failed
CI/CD / typecheck (pull_request) Successful in 1m4s
CI/CD / lint (pull_request) Successful in 2m14s
CI/CD / test (pull_request) Successful in 1m18s
CI/CD / build (pull_request) Successful in 2m30s
CI/CD / smoke (pull_request) Failing after 3m18s
CI/CD / publish (pull_request) Has been skipped
Some checks failed
CI/CD / typecheck (pull_request) Successful in 1m4s
CI/CD / lint (pull_request) Successful in 2m14s
CI/CD / test (pull_request) Successful in 1m18s
CI/CD / build (pull_request) Successful in 2m30s
CI/CD / smoke (pull_request) Failing after 3m18s
CI/CD / publish (pull_request) Has been skipped
Chat is LLM-essential but not model-specific — instead of failing when the
pinned model is down/drifted, it now fails over across an ordered chain and
reports which model actually answered.
- Ordered fallback: an Llm declares `extraConfig.fallbacks: string[]`; the
dispatcher builds primary-pool → fallback-pool(s) candidates and tries them
in order (resolveCandidatesWithFallbacks).
- Fail over on real failures, not just transport: runOneInference now advances
on a non-2xx status (e.g. a 400 from a drifted model) or an empty/invalid
completion, not only thrown transport errors. Streaming fails over
pre-first-chunk (already threw on 4xx).
- Transparency: ChatResult + the SSE `final` frame carry {llm, model,
failedOver}; the CLI prints `model: <llm> (<model>)` each turn and
`⚠ failed over → answered by …` when a fallback was used.
- Exhaustion names the last model + upstream body (not "no choice").
Tests: 3 failover unit tests (primary 400 → fallback answers + model reported;
primary answers → failedOver=false; all fail → clear aggregated error).
mcpd 948 + CLI 508 green; tsc + lint clean. docs/reliability.md updated.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -198,7 +198,8 @@ async function runOneShot(
|
||||
// `thread: <id>` — single space after the colon, matching the streaming
|
||||
// path (line 160 below) so any tooling/regex that watches one form picks
|
||||
// up the other too.
|
||||
process.stderr.write(styleStats(`(${String(words)}w · ${(words / sec).toFixed(1)} w/s · ${sec.toFixed(1)}s)`) + ` thread: ${res.threadId}\n`);
|
||||
const answered = formatAnswered(res.llm, res.model, res.failedOver);
|
||||
process.stderr.write(styleStats(`(${String(words)}w · ${(words / sec).toFixed(1)} w/s · ${sec.toFixed(1)}s)`) + ` thread: ${res.threadId}${answered !== '' ? ` ${answered}` : ''}\n`);
|
||||
return;
|
||||
}
|
||||
const bar = installStatusBar();
|
||||
@@ -258,6 +259,8 @@ async function runRepl(
|
||||
const res = await chatRequestNonStream(deps, subject, body);
|
||||
threadId = res.threadId;
|
||||
process.stdout.write(`${res.assistant}\n`);
|
||||
const answered = formatAnswered(res.llm, res.model, res.failedOver);
|
||||
if (answered !== '') process.stderr.write(`${styleStats(`(${answered})`)}\n`);
|
||||
} else {
|
||||
threadId = await streamOnce(deps, subject, line, threadId, overrides, bar);
|
||||
process.stdout.write('\n');
|
||||
@@ -455,7 +458,7 @@ async function chatRequestNonStream(
|
||||
deps: ChatCommandDeps,
|
||||
subject: ChatSubject,
|
||||
body: Record<string, unknown>,
|
||||
): Promise<{ assistant: string; threadId: string; turnIndex: number }> {
|
||||
): Promise<{ assistant: string; threadId: string; turnIndex: number; llm?: string; model?: string; failedOver?: boolean }> {
|
||||
const url = new URL(`${deps.baseUrl}/api/v1/${subject.path}/chat`);
|
||||
const payload = JSON.stringify(body);
|
||||
return new Promise((resolve, reject) => {
|
||||
@@ -482,7 +485,7 @@ async function chatRequestNonStream(
|
||||
return;
|
||||
}
|
||||
try {
|
||||
resolve(JSON.parse(raw) as { assistant: string; threadId: string; turnIndex: number });
|
||||
resolve(JSON.parse(raw) as { assistant: string; threadId: string; turnIndex: number; llm?: string; model?: string; failedOver?: boolean });
|
||||
} catch (err) {
|
||||
reject(new Error(`malformed response: ${(err as Error).message}`));
|
||||
}
|
||||
@@ -550,6 +553,7 @@ async function streamOnce(
|
||||
}
|
||||
let buf = '';
|
||||
let resolvedThread = threadId ?? '';
|
||||
let answered = '';
|
||||
res.setEncoding('utf-8');
|
||||
res.on('data', (chunk: string) => {
|
||||
buf += chunk;
|
||||
@@ -590,6 +594,7 @@ async function streamOnce(
|
||||
break;
|
||||
case 'final':
|
||||
if (evt.threadId !== undefined) resolvedThread = evt.threadId;
|
||||
answered = formatAnswered(evt.llm, evt.model, evt.failedOver);
|
||||
break;
|
||||
case 'error':
|
||||
process.stderr.write(`\n[error: ${evt.message ?? ''}]\n`);
|
||||
@@ -606,10 +611,11 @@ async function streamOnce(
|
||||
const final = formatStats(stats, false);
|
||||
// Inline final-stats footer (permanent record): goes through the
|
||||
// scroll region, so a copy-pasted transcript captures it.
|
||||
if (final !== '' && STDERR_IS_TTY) {
|
||||
process.stderr.write(`\n${styleStats(`(${final})`)}`);
|
||||
} else if (final !== '') {
|
||||
process.stderr.write(`\n(${final})`);
|
||||
const footer = [final, answered].filter((s) => s !== '').join(' · ');
|
||||
if (footer !== '' && STDERR_IS_TTY) {
|
||||
process.stderr.write(`\n${styleStats(`(${footer})`)}`);
|
||||
} else if (footer !== '') {
|
||||
process.stderr.write(`\n(${footer})`);
|
||||
}
|
||||
// Live status bar: pin the final value so it stays visible between
|
||||
// turns (answers "how fast was the last one?").
|
||||
@@ -637,6 +643,16 @@ interface ChatStreamFrame {
|
||||
threadId?: string;
|
||||
turnIndex?: number;
|
||||
message?: string;
|
||||
llm?: string;
|
||||
model?: string;
|
||||
failedOver?: boolean;
|
||||
}
|
||||
|
||||
/** One-line "which model answered" footer, dim; loud when it failed over. */
|
||||
function formatAnswered(llm?: string, model?: string, failedOver?: boolean): string {
|
||||
if (llm === undefined && model === undefined) return '';
|
||||
const label = model !== undefined && model !== '' ? `${llm ?? '?'} (${model})` : (llm ?? '?');
|
||||
return failedOver === true ? `⚠ failed over → answered by ${label}` : `model: ${label}`;
|
||||
}
|
||||
|
||||
// ANSI codes for the reasoning sidebar. Dim + italic visually separates
|
||||
|
||||
Reference in New Issue
Block a user