feat: automatic reconciliation loop for MCP server instances
mcpd now runs a periodic reconcileAll() every 30s that: - Detects crashed/missing containers (syncStatus) - Cleans up ERROR instances - Creates replacement pods to match desired replica count This replaces the old syncStatus-only timer. Servers migrated from another deployment or recovering from node failures will automatically get their instances recreated. 6 new tests for reconcileAll covering: missing instances, skip replicas=0, already-at-count, ERROR cleanup, multi-server, error isolation. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -107,6 +107,49 @@ export class InstanceService {
|
||||
return this.instanceRepo.findAll(serverId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconcile ALL servers — the operator loop.
|
||||
*
|
||||
* For every server with replicas > 0, ensures the correct number of
|
||||
* healthy instances exist. Cleans up ERROR instances and starts
|
||||
* replacements. This is the core self-healing mechanism.
|
||||
*/
|
||||
async reconcileAll(): Promise<{ reconciled: number; errors: string[] }> {
|
||||
await this.syncStatus();
|
||||
|
||||
const servers = await this.serverRepo.findAll();
|
||||
let reconciled = 0;
|
||||
const errors: string[] = [];
|
||||
|
||||
for (const server of servers) {
|
||||
if (server.replicas <= 0) continue;
|
||||
|
||||
try {
|
||||
const instances = await this.instanceRepo.findAll(server.id);
|
||||
const active = instances.filter((i) => i.status === 'RUNNING' || i.status === 'STARTING');
|
||||
const errored = instances.filter((i) => i.status === 'ERROR');
|
||||
|
||||
// Clean up ERROR instances so they don't accumulate
|
||||
for (const inst of errored) {
|
||||
await this.removeOne(inst);
|
||||
}
|
||||
|
||||
// Scale up if needed
|
||||
const toStart = server.replicas - active.length;
|
||||
if (toStart > 0) {
|
||||
for (let i = 0; i < toStart; i++) {
|
||||
await this.startOne(server.id);
|
||||
}
|
||||
reconciled++;
|
||||
}
|
||||
} catch (err) {
|
||||
errors.push(`${server.name}: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
return { reconciled, errors };
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove an instance (stop container + delete DB record).
|
||||
* Does NOT reconcile — caller should reconcile after if needed.
|
||||
|
||||
Reference in New Issue
Block a user