feat: automatic reconciliation loop for MCP server instances

mcpd now runs a periodic reconcileAll() every 30s that:
- Detects crashed/missing containers (syncStatus)
- Cleans up ERROR instances
- Creates replacement pods to match desired replica count

This replaces the old syncStatus-only timer. Servers migrated
from another deployment or recovering from node failures will
automatically get their instances recreated.

6 new tests for reconcileAll covering: missing instances, skip
replicas=0, already-at-count, ERROR cleanup, multi-server,
error isolation.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Michal
2026-04-08 19:00:19 +01:00
parent 14be2fa18e
commit d293df738a
3 changed files with 153 additions and 7 deletions

View File

@@ -107,6 +107,49 @@ export class InstanceService {
return this.instanceRepo.findAll(serverId);
}
/**
* Reconcile ALL servers — the operator loop.
*
* For every server with replicas > 0, ensures the correct number of
* healthy instances exist. Cleans up ERROR instances and starts
* replacements. This is the core self-healing mechanism.
*/
async reconcileAll(): Promise<{ reconciled: number; errors: string[] }> {
await this.syncStatus();
const servers = await this.serverRepo.findAll();
let reconciled = 0;
const errors: string[] = [];
for (const server of servers) {
if (server.replicas <= 0) continue;
try {
const instances = await this.instanceRepo.findAll(server.id);
const active = instances.filter((i) => i.status === 'RUNNING' || i.status === 'STARTING');
const errored = instances.filter((i) => i.status === 'ERROR');
// Clean up ERROR instances so they don't accumulate
for (const inst of errored) {
await this.removeOne(inst);
}
// Scale up if needed
const toStart = server.replicas - active.length;
if (toStart > 0) {
for (let i = 0; i < toStart; i++) {
await this.startOne(server.id);
}
reconciled++;
}
} catch (err) {
errors.push(`${server.name}: ${err instanceof Error ? err.message : String(err)}`);
}
}
return { reconciled, errors };
}
/**
* Remove an instance (stop container + delete DB record).
* Does NOT reconcile — caller should reconcile after if needed.