From 0c1e18cee1bebe3eebb012d6b2a416efdfb257bc Mon Sep 17 00:00:00 2001
From: Michal <michal@itaz.eu>
Date: Sun, 29 Mar 2026 02:34:26 +0100
Subject: [PATCH] feat: persist machine state to CockroachDB on
 bastion-state-sync

When bastion syncs state, labd now upserts discovered and installed
machines into the Server table. /api/machines merges live bastion
state with DB records, so machines survive pod restarts.

Discovered machines get status=discovered with hardware labels.
Installed machines get status=online with hostname, role, IP.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 bastion/src/labd/src/main.ts            |  1 +
 bastion/src/labd/src/routes/bastions.ts | 49 ++++++++++++++++++++++++-
 bastion/src/labd/src/server.ts          | 47 ++++++++++++++++++++++++
 3 files changed, 95 insertions(+), 2 deletions(-)

diff --git a/bastion/src/labd/src/main.ts b/bastion/src/labd/src/main.ts
index 17110d9..1c365ef 100644
--- a/bastion/src/labd/src/main.ts
+++ b/bastion/src/labd/src/main.ts
@@ -34,6 +34,7 @@ async function main(): Promise<void> {
       server: {
         findMany: () => dbError(),
         findUnique: () => dbError(),
+        upsert: () => dbError(),
       },
       joinToken: {
         findUnique: () => dbError(),
diff --git a/bastion/src/labd/src/routes/bastions.ts b/bastion/src/labd/src/routes/bastions.ts
index 8ed15ec..a1c0af8 100644
--- a/bastion/src/labd/src/routes/bastions.ts
+++ b/bastion/src/labd/src/routes/bastions.ts
@@ -80,9 +80,54 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
     });
   });
 
-  // Aggregated machines from all connected bastions
+  // Aggregated machines from all connected bastions + DB fallback
   app.get("/api/machines", async () => {
-    return bastionRegistry.getAggregatedState();
+    const live = bastionRegistry.getAggregatedState();
+
+    // Merge DB records for machines not currently in any bastion's live state
+    try {
+      const dbServers = (await db.server.findMany({})) as Array<{
+        mac: string | null; hostname: string; role: string; ip: string | null;
+        status: string; labels: Record<string, unknown>;
+      }>;
+      for (const s of dbServers) {
+        if (!s.mac) continue;
+        const mac = s.mac.toLowerCase();
+        // Only add from DB if not already in live state
+        if (!(mac in live.discovered) && !(mac in live.install_queue) && !(mac in live.installed)) {
+          if (s.status === "discovered") {
+            live.discovered[mac] = {
+              mac,
+              product: String(s.labels?.product ?? "unknown"),
+              board: "unknown",
+              serial: "unknown",
+              manufacturer: String(s.labels?.manufacturer ?? "unknown"),
+              cpu_model: String(s.labels?.cpu ?? "unknown"),
+              cpu_cores: Number(s.labels?.cores ?? 0),
+              memory_gb: Number(s.labels?.memory_gb ?? 0),
+              arch: String(s.labels?.arch ?? "unknown"),
+              disks: [],
+              nics: [],
+              first_seen: "",
+              last_seen: "",
+              bastionId: "db",
+            };
+          } else if (s.status === "online" || s.status === "offline") {
+            live.installed[mac] = {
+              hostname: s.hostname,
+              role: s.role,
+              ip: s.ip ?? "",
+              installed_at: "",
+              bastionId: "db",
+            };
+          }
+        }
+      }
+    } catch {
+      // DB unavailable — return live state only
+    }
+
+    return live;
   });
 
   // Queue install — route to correct bastion by MAC
diff --git a/bastion/src/labd/src/server.ts b/bastion/src/labd/src/server.ts
index b1bd8f4..4881962 100644
--- a/bastion/src/labd/src/server.ts
+++ b/bastion/src/labd/src/server.ts
@@ -19,6 +19,7 @@ export interface DbClient {
   server: {
     findMany: (...args: unknown[]) => Promise<unknown[]>;
     findUnique: (...args: unknown[]) => Promise<unknown>;
+    upsert: (...args: unknown[]) => Promise<unknown>;
   };
   joinToken: {
     findUnique: (...args: unknown[]) => Promise<unknown>;
@@ -175,6 +176,52 @@ export async function createApp(_config: LabdConfig, db: DbClient): Promise<{
               if (bastionId) {
                 bastionRegistry.updateState(bastionId, msg.state);
                 logger.info(`Bastion ${bastionId.slice(0, 8)} state sync: ${Object.keys(msg.state.discovered).length} discovered, ${Object.keys(msg.state.installed).length} installed`);
+
+                // Persist machines to DB
+                void (async () => {
+                  try {
+                    // Upsert discovered machines
+                    for (const [mac, hw] of Object.entries(msg.state.discovered)) {
+                      await db.server.upsert({
+                        where: { mac },
+                        create: {
+                          hostname: hw.product ?? mac,
+                          mac,
+                          role: "unknown",
+                          status: "discovered",
+                          labels: { cpu: hw.cpu_model, cores: hw.cpu_cores, memory_gb: hw.memory_gb, arch: hw.arch, product: hw.product, manufacturer: hw.manufacturer },
+                        },
+                        update: {
+                          status: "discovered",
+                          lastHeartbeat: new Date(),
+                          labels: { cpu: hw.cpu_model, cores: hw.cpu_cores, memory_gb: hw.memory_gb, arch: hw.arch, product: hw.product, manufacturer: hw.manufacturer },
+                        },
+                      });
+                    }
+                    // Upsert installed machines
+                    for (const [mac, info] of Object.entries(msg.state.installed)) {
+                      await db.server.upsert({
+                        where: { mac },
+                        create: {
+                          hostname: info.hostname,
+                          mac,
+                          role: info.role ?? "worker",
+                          ip: info.ip,
+                          status: "online",
+                        },
+                        update: {
+                          hostname: info.hostname,
+                          role: info.role ?? "worker",
+                          ip: info.ip,
+                          status: "online",
+                          lastHeartbeat: new Date(),
+                        },
+                      });
+                    }
+                  } catch (err) {
+                    logger.warn(`Failed to persist machines to DB: ${err instanceof Error ? err.message : String(err)}`);
+                  }
+                })();
               }
               break;
             }