Merge branch 'feat/unifi-export-and-vyos-dhcp': USG to VyOS migration
Some checks failed
Some checks failed
Carries the UniFi export tooling, the generated VyOS DHCP/DNS config, the reversible cutover switch, the health-checked WAN failover, and the labctl side of applying a Pulumi-rendered bundle at install time. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01DMVzWZgiKW2wquf5z8S1yH
This commit is contained in:
Michal
@@ -52,6 +52,134 @@ function normalizeDiskPath(value: string | undefined): string {
|
||||
return raw.startsWith("/dev/") ? raw : `/dev/${raw}`;
|
||||
}
|
||||
|
||||
/** Sentinel marking a value that lives in Pulumi config, not in the bundle. */
|
||||
const SECRET_PREFIX = "@secret:";
|
||||
|
||||
/**
|
||||
* Enable the VyOS HTTP API so the router is manageable the moment it boots.
|
||||
*
|
||||
* This belongs at install time rather than in the Pulumi model: the model is
|
||||
* applied THROUGH this API, so a router that lacks it cannot be brought under
|
||||
* management without a hand-run change on a live firewall. It is also why the
|
||||
* model excludes `service https` outright -- a provider able to rewrite its own
|
||||
* transport can lock itself out permanently.
|
||||
*
|
||||
* `listen-address` is always set. Leaving it unbound would expose a
|
||||
* config-write endpoint on every segment the router touches, the WAN included.
|
||||
*/
|
||||
function apiSets(apiKey: string, listenAddress: string): VyosSetOp[] {
|
||||
const sets: VyosSetOp[] = [
|
||||
{ path: ["service", "https", "api", "keys", "id", "pulumi", "key"], value: apiKey },
|
||||
{ path: ["service", "https", "api", "rest"] },
|
||||
];
|
||||
if (listenAddress !== "") {
|
||||
sets.push({ path: ["service", "https", "listen-address"], value: listenAddress });
|
||||
}
|
||||
return sets;
|
||||
}
|
||||
|
||||
/** Tag nodes introduced by the API config, needed by the installer's ConfigTree. */
|
||||
const API_TAGS: string[][] = [["service", "https", "api", "keys", "id"]];
|
||||
|
||||
/**
|
||||
* The address to bind the API to: an explicit choice, else the management
|
||||
* address with its prefix length stripped. Under DHCP there is no address to
|
||||
* bind at build time, so the caller must pass one or the listener stays unbound
|
||||
* and the API is not enabled at all.
|
||||
*/
|
||||
function apiListenAddress(spec: VyosInstallSpec, mgmtAddress: string): string {
|
||||
if (spec.apiListenAddress !== undefined && spec.apiListenAddress !== "") {
|
||||
return spec.apiListenAddress;
|
||||
}
|
||||
return mgmtAddress.includes("/") ? (mgmtAddress.split("/")[0] ?? "") : "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Use a Pulumi-rendered bundle as the router's config verbatim.
|
||||
*
|
||||
* Secret-valued nodes are dropped rather than installed with their sentinel
|
||||
* text: writing `@secret:pppoePassword` into config.boot would look configured
|
||||
* while being wrong, which is worse than being absent. The router comes up
|
||||
* without those values and the first `pulumi up` fills them in.
|
||||
*
|
||||
* `system host-name` is forced to the hostname the install was asked for. The
|
||||
* bundle carries the name of whichever router it was exported from, and
|
||||
* installing vyos001's hostname onto vyos002 would collide on the network.
|
||||
*/
|
||||
function buildFromBundle(
|
||||
params: { hostname: string; defaultPassword: string; disk?: string | undefined },
|
||||
spec: VyosInstallSpec,
|
||||
bundle: NonNullable<VyosInstallSpec["bundle"]>,
|
||||
mgmtAddress: string,
|
||||
): VyosConfigSpec {
|
||||
const sets: VyosSetOp[] = [];
|
||||
const dropped: string[] = [];
|
||||
for (const op of bundle.sets) {
|
||||
if (op.value !== undefined && op.value.startsWith(SECRET_PREFIX)) {
|
||||
dropped.push(op.path.join(" "));
|
||||
continue;
|
||||
}
|
||||
if (op.path.length === 2 && op.path[0] === "system" && op.path[1] === "host-name") {
|
||||
continue;
|
||||
}
|
||||
sets.push({
|
||||
path: op.path,
|
||||
...(op.value === undefined ? {} : { value: op.value }),
|
||||
...(op.replace === undefined ? {} : { replace: op.replace }),
|
||||
});
|
||||
}
|
||||
sets.unshift({ path: ["system", "host-name"], value: params.hostname });
|
||||
|
||||
if (dropped.length > 0) {
|
||||
console.warn(
|
||||
`vyos ${params.hostname}: ${dropped.length} secret-valued node(s) left unset by the ` +
|
||||
`bundle; run \`pulumi up\` to supply them: ${dropped.join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
const tags = [...bundle.tags];
|
||||
const api = enableApi(spec, params.hostname, mgmtAddress);
|
||||
if (api.length > 0) {
|
||||
sets.push(...api);
|
||||
tags.push(...API_TAGS);
|
||||
}
|
||||
|
||||
return {
|
||||
hostname: params.hostname,
|
||||
imageName: "",
|
||||
password: spec.password ?? params.defaultPassword,
|
||||
console: "K",
|
||||
disk: normalizeDiskPath(params.disk),
|
||||
reportAddress: mgmtAddress.includes("/") ? (mgmtAddress.split("/")[0] ?? "") : "",
|
||||
raid: false,
|
||||
freshConfig: spec.freshConfig ?? false,
|
||||
sets,
|
||||
tags,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The API config for this install, or nothing when it cannot be enabled safely.
|
||||
*
|
||||
* Refusing to enable it unbound is deliberate. Under DHCP there is no address
|
||||
* known at build time, and the alternative -- binding to every interface --
|
||||
* would publish a config-write endpoint on the WAN. Better to leave the router
|
||||
* SSH-only and say so than to open it everywhere.
|
||||
*/
|
||||
function enableApi(spec: VyosInstallSpec, hostname: string, mgmtAddress: string): VyosSetOp[] {
|
||||
if (spec.apiKey === undefined || spec.apiKey === "") return [];
|
||||
const listen = apiListenAddress(spec, mgmtAddress);
|
||||
if (listen === "") {
|
||||
console.warn(
|
||||
`vyos ${hostname}: --vyos-api-key given but no address to bind to ` +
|
||||
`(management is "${mgmtAddress}"). Pass --vyos-api-listen <addr>; the HTTP API ` +
|
||||
`has NOT been enabled, so Pulumi cannot manage this router yet.`,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
return apiSets(spec.apiKey, listen);
|
||||
}
|
||||
|
||||
export function buildVyosConfigSpec(params: {
|
||||
hostname: string;
|
||||
spec?: VyosInstallSpec | undefined;
|
||||
@@ -62,6 +190,14 @@ export function buildVyosConfigSpec(params: {
|
||||
const spec = params.spec ?? {};
|
||||
const mgmt = spec.mgmtInterface ?? "eth0";
|
||||
const mgmtAddress = spec.mgmtAddress ?? "dhcp";
|
||||
|
||||
// A rendered bundle replaces the derived config entirely. Deriving a second
|
||||
// opinion alongside it is the drift the bundle exists to prevent: Pulumi and
|
||||
// labctl would each believe they knew the router's config, and the box would
|
||||
// end up with whichever ran last.
|
||||
if (spec.bundle !== undefined) {
|
||||
return buildFromBundle(params, spec, spec.bundle, mgmtAddress);
|
||||
}
|
||||
const bondMembers = spec.bondMembers ?? [];
|
||||
const vlans = spec.vlans ?? [];
|
||||
|
||||
@@ -192,6 +328,14 @@ export function buildVyosConfigSpec(params: {
|
||||
});
|
||||
}
|
||||
|
||||
// Enabled here too, not just for bundle installs: every VyOS this bastion
|
||||
// provisions should be manageable from first boot.
|
||||
const api = enableApi(spec, params.hostname, mgmtAddress);
|
||||
if (api.length > 0) {
|
||||
sets.push(...api);
|
||||
tags.push(...API_TAGS);
|
||||
}
|
||||
|
||||
return {
|
||||
hostname: params.hostname,
|
||||
imageName: "",
|
||||
|
||||
155
bastion/src/bastion/tests/vyos-bundle.test.ts
Normal file
155
bastion/src/bastion/tests/vyos-bundle.test.ts
Normal file
@@ -0,0 +1,155 @@
|
||||
import { describe, it, expect, vi } from "vitest";
|
||||
import type { VyosBundle } from "@lab/shared";
|
||||
import { buildVyosConfigSpec } from "../src/templates/vyos-config-spec.js";
|
||||
|
||||
/**
|
||||
* A bundle is what makes "one config, two apply paths" true rather than
|
||||
* aspirational: `pulumi up` POSTs the subtree model to a running router, labctl
|
||||
* writes the same model into config.boot during a PXE install. These tests pin
|
||||
* the properties that keep the two honest.
|
||||
*/
|
||||
const bundle: VyosBundle = {
|
||||
sets: [
|
||||
{ path: ["system", "host-name"], value: "vyos001" },
|
||||
{ path: ["interfaces", "bonding", "bond0", "address"], value: "192.168.1.252/24" },
|
||||
{ path: ["interfaces", "bonding", "bond0", "member", "interface"], value: "eth1", replace: false },
|
||||
{ path: ["interfaces", "bonding", "bond0", "vif", "53", "disable"] },
|
||||
{ path: ["interfaces", "pppoe", "pppoe0", "authentication", "password"], value: "@secret:pppoePassword" },
|
||||
{ path: ["interfaces", "pppoe", "pppoe0", "mtu"], value: "1492" },
|
||||
],
|
||||
tags: [["interfaces", "bonding", "bond0"], ["interfaces", "ethernet"]],
|
||||
};
|
||||
|
||||
const build = (hostname: string, extra: Record<string, unknown> = {}) =>
|
||||
buildVyosConfigSpec({
|
||||
hostname,
|
||||
spec: { bundle, ...extra },
|
||||
defaultPassword: "changeme",
|
||||
});
|
||||
|
||||
describe("vyos config spec from a Pulumi bundle", () => {
|
||||
it("applies non-secret nodes verbatim, preserving valuelessness and replace:false", () => {
|
||||
const spec = build("vyos001");
|
||||
|
||||
expect(spec.sets).toContainEqual({
|
||||
path: ["interfaces", "bonding", "bond0", "address"],
|
||||
value: "192.168.1.252/24",
|
||||
});
|
||||
// A multi-value node must keep replace:false or the second bond member
|
||||
// overwrites the first.
|
||||
expect(spec.sets).toContainEqual({
|
||||
path: ["interfaces", "bonding", "bond0", "member", "interface"],
|
||||
value: "eth1",
|
||||
replace: false,
|
||||
});
|
||||
// A valueless node must not acquire a value on the way through.
|
||||
expect(spec.sets).toContainEqual({
|
||||
path: ["interfaces", "bonding", "bond0", "vif", "53", "disable"],
|
||||
});
|
||||
expect(spec.tags).toEqual(bundle.tags);
|
||||
});
|
||||
|
||||
it("drops secret-valued nodes instead of installing the sentinel text", () => {
|
||||
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
const spec = build("vyos001");
|
||||
|
||||
const values = spec.sets.map((s) => s.value ?? "");
|
||||
expect(values.some((v) => v.startsWith("@secret:"))).toBe(false);
|
||||
expect(spec.sets.some((s) => s.path.includes("authentication"))).toBe(false);
|
||||
// Silently dropping the WAN credential would leave someone debugging a dead
|
||||
// PPPoE link, so it has to be said out loud.
|
||||
expect(warn).toHaveBeenCalledWith(expect.stringContaining("pulumi up"));
|
||||
warn.mockRestore();
|
||||
});
|
||||
|
||||
it("forces the hostname the install was asked for, not the bundle's", () => {
|
||||
// The bundle is exported from one router and reused for its peer; taking the
|
||||
// hostname from it would put two vyos001s on the network.
|
||||
const spec = build("vyos002");
|
||||
const hostnames = spec.sets.filter(
|
||||
(s) => s.path.length === 2 && s.path[0] === "system" && s.path[1] === "host-name",
|
||||
);
|
||||
expect(hostnames).toEqual([{ path: ["system", "host-name"], value: "vyos002" }]);
|
||||
});
|
||||
|
||||
it("still honours installer inputs, which are not router config", () => {
|
||||
const spec = buildVyosConfigSpec({
|
||||
hostname: "vyos001",
|
||||
spec: { bundle, password: "s3cret", freshConfig: true },
|
||||
defaultPassword: "changeme",
|
||||
disk: "nvme0n1",
|
||||
});
|
||||
expect(spec.password).toBe("s3cret");
|
||||
expect(spec.freshConfig).toBe(true);
|
||||
expect(spec.disk).toBe("/dev/nvme0n1");
|
||||
});
|
||||
|
||||
it("enables the HTTP API at install so Pulumi can manage the router from first boot", () => {
|
||||
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
const spec = buildVyosConfigSpec({
|
||||
hostname: "vyos001",
|
||||
spec: { bundle, apiKey: "k3y", apiListenAddress: "10.0.1.252" },
|
||||
defaultPassword: "changeme",
|
||||
});
|
||||
|
||||
expect(spec.sets).toContainEqual({
|
||||
path: ["service", "https", "api", "keys", "id", "pulumi", "key"],
|
||||
value: "k3y",
|
||||
});
|
||||
expect(spec.sets).toContainEqual({ path: ["service", "https", "api", "rest"] });
|
||||
expect(spec.sets).toContainEqual({
|
||||
path: ["service", "https", "listen-address"],
|
||||
value: "10.0.1.252",
|
||||
});
|
||||
// The key id is a tag node; without this the installer's ConfigTree rejects it.
|
||||
expect(spec.tags).toContainEqual(["service", "https", "api", "keys", "id"]);
|
||||
warn.mockRestore();
|
||||
});
|
||||
|
||||
it("binds the API to the static management address when none is given", () => {
|
||||
const spec = buildVyosConfigSpec({
|
||||
hostname: "vyos001",
|
||||
spec: { apiKey: "k3y", mgmtAddress: "192.168.1.252/24" },
|
||||
defaultPassword: "changeme",
|
||||
});
|
||||
expect(spec.sets).toContainEqual({
|
||||
path: ["service", "https", "listen-address"],
|
||||
value: "192.168.1.252",
|
||||
});
|
||||
});
|
||||
|
||||
it("refuses to enable the API unbound rather than exposing it on the WAN", () => {
|
||||
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
// Management is DHCP, so there is no address to bind at build time. Binding
|
||||
// to everything would put a config-write endpoint on the WAN.
|
||||
const spec = buildVyosConfigSpec({
|
||||
hostname: "vyos001",
|
||||
spec: { apiKey: "k3y", mgmtAddress: "dhcp" },
|
||||
defaultPassword: "changeme",
|
||||
});
|
||||
expect(spec.sets.some((s) => s.path[0] === "service" && s.path[1] === "https")).toBe(false);
|
||||
expect(warn).toHaveBeenCalledWith(expect.stringContaining("has NOT been enabled"));
|
||||
warn.mockRestore();
|
||||
});
|
||||
|
||||
it("does not enable the API when no key is supplied", () => {
|
||||
const spec = buildVyosConfigSpec({
|
||||
hostname: "vyos001",
|
||||
spec: { mgmtAddress: "192.168.1.252/24" },
|
||||
defaultPassword: "changeme",
|
||||
});
|
||||
expect(spec.sets.some((s) => s.path[0] === "service" && s.path[1] === "https")).toBe(false);
|
||||
});
|
||||
|
||||
it("ignores the derived path entirely when a bundle is present", () => {
|
||||
// Belt and braces: even if topology flags reach this far (the CLI rejects
|
||||
// them), the bundle must win rather than merge.
|
||||
const spec = buildVyosConfigSpec({
|
||||
hostname: "vyos001",
|
||||
spec: { bundle, bondMembers: ["eth2", "eth3"], vlans: [{ id: 99, address: "10.9.9.1/24" }] },
|
||||
defaultPassword: "changeme",
|
||||
});
|
||||
expect(spec.sets.some((s) => s.path.includes("99"))).toBe(false);
|
||||
expect(spec.sets.filter((s) => s.value === "eth2" || s.value === "eth3")).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -1,11 +1,42 @@
|
||||
// CLI command: provision install
|
||||
// Queue a discovered machine for OS installation via labd.
|
||||
|
||||
import { readFileSync } from "node:fs";
|
||||
import { Command, Option, InvalidArgumentError } from "commander";
|
||||
import { isValidOsId, SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY } from "@lab/shared";
|
||||
import type { VyosInstallSpec, VyosVlanSpec } from "@lab/shared";
|
||||
import type { VyosBundle, VyosInstallSpec, VyosVlanSpec } from "@lab/shared";
|
||||
import { getLabdClient } from "../api/config.js";
|
||||
|
||||
/**
|
||||
* Load one router's config out of a Pulumi-rendered bundle.
|
||||
*
|
||||
* The bundle is produced by `kubernetes-deployment` (npm run vyos:bundle) and
|
||||
* holds every router it manages, keyed by name. Selecting by hostname here is
|
||||
* what keeps bring-up and `pulumi up` describing the same box: labctl replays
|
||||
* the declared config rather than deriving its own.
|
||||
*/
|
||||
export function loadVyosBundle(path: string, hostname: string): VyosBundle {
|
||||
let parsed: { version?: number; routers?: Record<string, VyosBundle> };
|
||||
try {
|
||||
parsed = JSON.parse(readFileSync(path, "utf8"));
|
||||
} catch (e) {
|
||||
throw new InvalidArgumentError(`Cannot read VyOS bundle ${path}: ${(e as Error).message}`);
|
||||
}
|
||||
if (parsed.version !== 1) {
|
||||
throw new InvalidArgumentError(
|
||||
`VyOS bundle ${path} has version ${parsed.version ?? "<none>"}; this labctl understands 1`,
|
||||
);
|
||||
}
|
||||
const router = parsed.routers?.[hostname];
|
||||
if (router === undefined) {
|
||||
const known = Object.keys(parsed.routers ?? {}).join(", ") || "<none>";
|
||||
throw new InvalidArgumentError(
|
||||
`VyOS bundle ${path} has no entry for "${hostname}" (has: ${known})`,
|
||||
);
|
||||
}
|
||||
return router;
|
||||
}
|
||||
|
||||
/** Parse a repeated --vlan flag: "<id>:<cidr>[:<description>]". */
|
||||
export function parseVlan(value: string, previous: VyosVlanSpec[] = []): VyosVlanSpec[] {
|
||||
const parts = value.split(":");
|
||||
@@ -88,6 +119,20 @@ export function registerInstallCommand(parent: Command): void {
|
||||
.option("--vyos-password <password>", "VyOS: password for the 'vyos' user")
|
||||
.option("--vyos-hwid <iface=mac>", "VyOS: pin an interface name to a MAC via hw-id (repeatable)", parseHwId)
|
||||
.option("--vyos-fresh-config", "VyOS: on reinstall, overwrite the preserved config with the generated one")
|
||||
.option(
|
||||
"--vyos-bundle <path>",
|
||||
"VyOS: apply a Pulumi-rendered bundle verbatim (kubernetes-deployment/infra/vyos/vyos-bundle.json). " +
|
||||
"Replaces the derived --vyos-bond/--vlan/... config; secret values are left unset for `pulumi up`.",
|
||||
)
|
||||
.option(
|
||||
"--vyos-api-key <key>",
|
||||
"VyOS: enable the HTTP API with this key so Pulumi can manage the router from first boot",
|
||||
)
|
||||
.option(
|
||||
"--vyos-api-listen <addr>",
|
||||
"VyOS: address the HTTP API binds to (default: the static management address). " +
|
||||
"Required when management is DHCP; the API is never bound to all interfaces.",
|
||||
)
|
||||
.action(async (mac: string, hostname: string, opts: {
|
||||
role: string;
|
||||
os: string;
|
||||
@@ -104,6 +149,9 @@ export function registerInstallCommand(parent: Command): void {
|
||||
vyosPassword?: string;
|
||||
vyosHwid?: Record<string, string>;
|
||||
vyosFreshConfig?: boolean;
|
||||
vyosBundle?: string;
|
||||
vyosApiKey?: string;
|
||||
vyosApiListen?: string;
|
||||
}) => {
|
||||
if (!isValidOsId(opts.os)) {
|
||||
console.error(`Unknown OS: ${opts.os}. Supported: ${SUPPORTED_OS.join(", ")}`);
|
||||
@@ -159,9 +207,31 @@ export function registerInstallCommand(parent: Command): void {
|
||||
...(opts.vyosMgmtVlan !== undefined && opts.vyosMgmtVlan !== ""
|
||||
? { mgmtVlan: parseVlan(opts.vyosMgmtVlan)[0] as VyosVlanSpec } : {}),
|
||||
...(opts.vyosFreshConfig === true ? { freshConfig: true } : {}),
|
||||
...(opts.vyosBundle !== undefined && opts.vyosBundle !== ""
|
||||
? { bundle: loadVyosBundle(opts.vyosBundle, hostname) } : {}),
|
||||
...(opts.vyosApiKey !== undefined && opts.vyosApiKey !== ""
|
||||
? { apiKey: opts.vyosApiKey } : {}),
|
||||
...(opts.vyosApiListen !== undefined && opts.vyosApiListen !== ""
|
||||
? { apiListenAddress: opts.vyosApiListen } : {}),
|
||||
};
|
||||
const hasVyosOptions = Object.keys(vyos).length > 0;
|
||||
|
||||
// A bundle already describes the whole router. Accepting derived topology
|
||||
// flags alongside it would silently discard them (the bundle wins in
|
||||
// buildVyosConfigSpec), so say so rather than appear to honour both.
|
||||
if (vyos.bundle !== undefined) {
|
||||
const derived = ["mgmtInterface", "mgmtAddress", "bondMembers", "bondAddress",
|
||||
"bondVrrp", "vrrpPriority", "vlans", "mgmtVlan"] as const;
|
||||
const conflicting = derived.filter((k) => vyos[k] !== undefined);
|
||||
if (conflicting.length > 0) {
|
||||
console.error(
|
||||
`--vyos-bundle describes the whole router; these would be ignored: ${conflicting.join(", ")}`,
|
||||
);
|
||||
console.error("Remove them, or change the bundle in kubernetes-deployment and re-render.");
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
if (hasVyosOptions && !opts.os.startsWith("vyos")) {
|
||||
console.error(`VyOS options require --os vyos-rolling (got --os ${opts.os})`);
|
||||
process.exit(1);
|
||||
|
||||
@@ -10,6 +10,8 @@ export type {
|
||||
BastionConfig,
|
||||
VyosVlanSpec,
|
||||
VyosInstallSpec,
|
||||
VyosBundle,
|
||||
VyosBundleSetOp,
|
||||
} from "./types/index.js";
|
||||
|
||||
export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId } from "./types/index.js";
|
||||
|
||||
@@ -9,6 +9,8 @@ export type {
|
||||
BastionState,
|
||||
VyosVlanSpec,
|
||||
VyosInstallSpec,
|
||||
VyosBundle,
|
||||
VyosBundleSetOp,
|
||||
} from "./state.js";
|
||||
|
||||
export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId } from "./state.js";
|
||||
|
||||
@@ -88,6 +88,33 @@ export interface VyosVlanSpec {
|
||||
vrrp?: string;
|
||||
}
|
||||
|
||||
/** One config node in a rendered bundle. Mirrors VyosSetOp on the bastion side. */
|
||||
export interface VyosBundleSetOp {
|
||||
path: string[];
|
||||
value?: string;
|
||||
/** false appends to a multi-value node (e.g. bond members) instead of replacing. */
|
||||
replace?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* A router's complete desired config, rendered from the Pulumi model.
|
||||
*
|
||||
* Produced by `kubernetes-deployment/scripts/vyos-render-bundle.ts` from the
|
||||
* same subtree model `pulumi up` applies. The point is that labctl never
|
||||
* authors VyOS config: bring-up replays what Pulumi already declares, so a
|
||||
* freshly installed router and a `pulumi up` cannot disagree.
|
||||
*
|
||||
* Secret values arrive as `@secret:<key>` sentinels and are DROPPED at install
|
||||
* time -- the bundle is committed to git and must stay safe to read. The router
|
||||
* comes up on the LAN without its PPPoE credential; the first `pulumi up`
|
||||
* supplies it. That handoff is deliberate.
|
||||
*/
|
||||
export interface VyosBundle {
|
||||
sets: VyosBundleSetOp[];
|
||||
/** Paths that are VyOS tag nodes — the installer's ConfigTree needs them marked. */
|
||||
tags: string[][];
|
||||
}
|
||||
|
||||
/**
|
||||
* VyOS-specific install parameters. Rendered into the config.boot that the
|
||||
* installer adopts, so the router comes up already configured.
|
||||
@@ -96,6 +123,31 @@ export interface VyosVlanSpec {
|
||||
* PXE cannot run over LACP, so the install-time NIC has to stay unbonded.
|
||||
*/
|
||||
export interface VyosInstallSpec {
|
||||
/**
|
||||
* A complete rendered config for this router. When present it REPLACES the
|
||||
* derived interface/VLAN/VRRP config below -- the bundle already describes
|
||||
* all of it, and deriving a second opinion is exactly the drift this exists
|
||||
* to prevent. The remaining install parameters (password, disk, console) are
|
||||
* still honoured because they are installer inputs, not router config.
|
||||
*/
|
||||
bundle?: VyosBundle;
|
||||
/**
|
||||
* Key for the VyOS HTTP API, enabled at install so the router is manageable
|
||||
* from the moment it boots.
|
||||
*
|
||||
* Without this the box comes up reachable only over SSH, and enabling the API
|
||||
* later is a hand-run config change on a live firewall -- which is exactly the
|
||||
* gap that left vyos001/vyos002 unmanageable by Pulumi after their cutover.
|
||||
* The API is deliberately NOT part of the Pulumi model: a provider that
|
||||
* manages its own transport can revoke its own access.
|
||||
*/
|
||||
apiKey?: string;
|
||||
/**
|
||||
* Address the API listens on. Defaults to the management address when static.
|
||||
* Never left unbound: an unrestricted listener puts a config-write endpoint on
|
||||
* every segment the router touches, including the WAN.
|
||||
*/
|
||||
apiListenAddress?: string;
|
||||
/** Interfaces aggregated into bond0 with LACP (802.3ad). Omit for no bond. */
|
||||
bondMembers?: string[];
|
||||
/** CIDR address on bond0 itself — the switch trunk's native/untagged VLAN. */
|
||||
|
||||
@@ -15,10 +15,27 @@ Each VLAN is its own isolated libvirt network with one tiny Alpine VM on it.
|
||||
| 2 | k8s | 172.31.2.0/24 | 172.31.2.10 | 192.168.8.0/23 |
|
||||
| 3 | kvm | 172.31.3.0/24 | 172.31.3.10 | 192.168.3.0/24 |
|
||||
| 9 | private | 172.31.9.0/24 | 172.31.9.10 | 10.0.9.0/23 |
|
||||
| 10 | lot | 172.31.10.0/24 | 172.31.10.10 | 10.0.0.0/23 |
|
||||
| 10 | lot | **172.31.10.0/23** | 172.31.10.10 | 10.0.0.0/23 |
|
||||
| 200 | roomates | 172.31.200.0/24 | 172.31.200.10 | 192.168.2.0/24 |
|
||||
|
||||
The sim subnet always encodes the VLAN id: `172.31.<vlan>.0/24`.
|
||||
The sim subnet encodes the VLAN id: `172.31.<vlan>.0/24`, with one exception.
|
||||
**VLAN 10 is a `/23`** because every UniFi DHCP reservation lives in LoT and LoT
|
||||
spans `10.0.0.x` *and* `10.0.1.x`, which a `/24` cannot hold. The mapping stays
|
||||
readable — `10.0.0.46 → 172.31.10.46`, `10.0.1.67 → 172.31.11.67`.
|
||||
|
||||
LoT's host leg is `.3`, not `.2`, because `10.0.0.2` is a real reservation
|
||||
(Hubitat) that maps onto `172.31.10.2`. `.3` is unreserved and sits below the
|
||||
DHCP pool, so it can never be handed out.
|
||||
|
||||
`vlans.conf` therefore takes two optional trailing fields:
|
||||
|
||||
```
|
||||
vlan_id:name:sim_prefix:real_subnet[:masklen][:host_octet]
|
||||
```
|
||||
|
||||
defaulting to `24` and `2`. k8s and Private are also `/23` in production but
|
||||
hold no reservations, so they keep their `/24` and their DHCP range is clamped
|
||||
— reported at generation time, never silently.
|
||||
|
||||
Address plan, identical on every VLAN:
|
||||
|
||||
@@ -69,6 +86,54 @@ sudo virsh console labsim-2-k8s # root / labsim
|
||||
./monitoring-up.sh # topology page + Prometheus + Grafana
|
||||
```
|
||||
|
||||
## Testing the DHCP migration
|
||||
|
||||
`./labsim-dhcp-test.sh` boots throwaway VMs whose MACs are **real production
|
||||
MACs** and checks each gets the address UniFi reserved for it. MACs are the one
|
||||
piece of production config that transplants verbatim, which is what makes this a
|
||||
test rather than a rehearsal. It is safe because `ovs-labsim` has no physical
|
||||
NIC — verified with `ovs-vsctl show` — so a production MAC cannot reach the real
|
||||
LAN.
|
||||
|
||||
Apply the config first, from `../migration`:
|
||||
|
||||
```bash
|
||||
python3 unifi-to-vyos.py --mode sim -o /tmp/sim.conf # 6 subnets, 31 mappings
|
||||
# load onto labsim-vyos, then:
|
||||
./labsim-dhcp-test.sh
|
||||
```
|
||||
|
||||
**Result on VyOS 2026.08 (kea): all four cases pass.** The one that mattered:
|
||||
most UniFi reservations sit *inside* the DHCP pool, and **kea honours in-pool
|
||||
host reservations** — `printer1` received `172.31.10.46` from within the
|
||||
`.10.11–.11.254` pool. That was the open question blocking the cutover.
|
||||
|
||||
### The lease database will lie to you
|
||||
|
||||
The script wipes `/config/dhcp/dhcp4-leases.csv*` before every run, and both
|
||||
halves of that matter:
|
||||
|
||||
- **Stale leases defeat reservations.** Re-running against yesterday's leases,
|
||||
kea handed dynamic addresses to three devices that have reservations. The
|
||||
reservation was present and correct in `/run/kea/kea-dhcp4.conf` the whole
|
||||
time. Kea saw the reserved address as already leased to "another client" —
|
||||
same MAC, but a different client-id from the earlier boot — and allocated
|
||||
elsewhere. The cutover itself starts with an empty lease database, so this is
|
||||
a *testing* artifact, but it is worth knowing that a reservation is not an
|
||||
unconditional guarantee once leases exist.
|
||||
- **The `*` is load-bearing.** Kea's memfile backend keeps lease-file-cleanup
|
||||
rotations (`dhcp4-leases.csv.2`) and restores from them on start, so
|
||||
truncating only the primary file changes nothing.
|
||||
|
||||
Both of those first appeared as a *passing* test. The verdict logic now refuses
|
||||
to score a MAC with more than one lease, because taking the first match had
|
||||
reported an hours-old lease as the current answer and turned three failures
|
||||
into apparent passes.
|
||||
|
||||
Still open: whether kea will hand a *reserved* address to a *different* client
|
||||
while the reserved device is offline. The negative case here only proves an
|
||||
unreserved MAC gets an unreserved address.
|
||||
|
||||
- **http://localhost:9101/** — live mesh: a node per VLAN, the router in the
|
||||
middle, one line per pair coloured green/red with the ICMP RTT on it. Hover a
|
||||
line for per-direction detail. Refreshes every 5s. This is the one to watch
|
||||
|
||||
109
labsim/console-apply.py
Executable file
109
labsim/console-apply.py
Executable file
@@ -0,0 +1,109 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Apply VyOS config to a labsim VM over its serial console.
|
||||
|
||||
Needed because a freshly installed VyOS comes up holding the same addresses as
|
||||
its peer, so there is a window where it cannot safely be reached over the
|
||||
network at all. The console does not care.
|
||||
|
||||
./console-apply.py --vm labsim-vyos2 --config r2.conf
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import time
|
||||
|
||||
import pexpect
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--vm", required=True)
|
||||
ap.add_argument("--config", required=True)
|
||||
ap.add_argument("--user", default="vyos")
|
||||
ap.add_argument("--password", default="vyos")
|
||||
args = ap.parse_args()
|
||||
|
||||
cmds = [l.rstrip() for l in open(args.config)
|
||||
if l.strip() and not l.lstrip().startswith("#")]
|
||||
print(f"{len(cmds)} commands to apply to {args.vm}", file=sys.stderr)
|
||||
|
||||
c = pexpect.spawn(f"virsh --connect qemu:///system console {args.vm}",
|
||||
timeout=90, encoding="utf-8")
|
||||
c.logfile_read = None
|
||||
c.sendline("")
|
||||
time.sleep(2)
|
||||
c.sendline("")
|
||||
|
||||
# Log in. A freshly booted box may still be starting services, so allow a
|
||||
# generous window and re-prod the console rather than failing on the first
|
||||
# miss.
|
||||
#
|
||||
# `# ` matters as much as `$ `: a previous run that died mid-config leaves
|
||||
# the console sitting in configuration mode, and waiting only for the
|
||||
# operational prompt then hangs forever against a perfectly healthy VM.
|
||||
in_config = False
|
||||
for _ in range(40):
|
||||
i = c.expect([r"login:", r"\$ ", r"# ", pexpect.TIMEOUT], timeout=15)
|
||||
if i == 0:
|
||||
c.sendline(args.user)
|
||||
c.expect("Password:", timeout=30)
|
||||
c.sendline(args.password)
|
||||
c.expect([r"\$ ", r"# "], timeout=60)
|
||||
break
|
||||
if i == 1:
|
||||
break
|
||||
if i == 2:
|
||||
in_config = True
|
||||
break
|
||||
c.sendline("")
|
||||
else:
|
||||
print("never reached a prompt", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if in_config:
|
||||
# Drop whatever the previous run left half-built rather than committing
|
||||
# a candidate nobody has seen.
|
||||
print("console was left in config mode; discarding stale candidate",
|
||||
file=sys.stderr)
|
||||
c.sendline("discard")
|
||||
c.expect(r"# ", timeout=60)
|
||||
else:
|
||||
c.sendline("configure")
|
||||
c.expect(r"# ", timeout=60)
|
||||
|
||||
for cmd in cmds:
|
||||
c.sendline(cmd)
|
||||
c.expect(r"# ", timeout=60)
|
||||
out = c.before or ""
|
||||
if "Set failed" in out or "not valid" in out or "Invalid" in out:
|
||||
print(f"FAILED: {cmd}\n {out.strip()[:200]}", file=sys.stderr)
|
||||
|
||||
print("committing...", file=sys.stderr)
|
||||
c.sendline("commit")
|
||||
c.expect(r"# ", timeout=300)
|
||||
commit_out = c.before or ""
|
||||
c.sendline("save")
|
||||
c.expect(r"# ", timeout=120)
|
||||
# Accept either prompt on the way out. Insisting on `$ ` here hangs against
|
||||
# a healthy box -- and worse, leaves the console parked in config mode, so
|
||||
# the NEXT run finds a `# ` it was not expecting either. One strict expect
|
||||
# turned into two failures.
|
||||
c.sendline("exit")
|
||||
c.expect([r"\$ ", r"# ", pexpect.TIMEOUT], timeout=60)
|
||||
c.sendline("exit")
|
||||
c.close(force=True)
|
||||
|
||||
bad = [l for l in commit_out.splitlines()
|
||||
if "failed" in l.lower() or "error" in l.lower()]
|
||||
if bad:
|
||||
print("commit reported:", file=sys.stderr)
|
||||
for l in bad[:10]:
|
||||
print(f" {l.strip()}", file=sys.stderr)
|
||||
return 1
|
||||
print("committed and saved", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
219
labsim/labsim-dhcp-test.sh
Executable file
219
labsim/labsim-dhcp-test.sh
Executable file
@@ -0,0 +1,219 @@
|
||||
#!/bin/bash
|
||||
# Prove that VyOS hands each device the address UniFi reserved for it.
|
||||
#
|
||||
# The question this answers is narrow and important: 30 of the 31 UniFi
|
||||
# reservations sit INSIDE the DHCP pool (LoT's pool is 10.0.0.11-10.0.1.254 and
|
||||
# only 10.0.0.2 falls outside it). UniFi's dhcpd tolerates that. VyOS uses kea,
|
||||
# and whether kea honours in-pool host reservations decides whether the cutover
|
||||
# silently renumbers 30 devices. That is not something to predict.
|
||||
#
|
||||
# Method: boot throwaway VMs whose MAC is a REAL production MAC, on the sim
|
||||
# VLAN, and check the address they are given. MACs are the one piece of
|
||||
# production config that transplants verbatim -- the subnet is rewritten, the
|
||||
# MAC is not -- which is what makes this a real test rather than a rehearsal.
|
||||
#
|
||||
# Safe: the ovs-labsim bridge contains only internal ports and VM taps, with no
|
||||
# physical NIC, so a production MAC here cannot reach or confuse the real LAN.
|
||||
# Verified with `ovs-vsctl show` before this script was written.
|
||||
#
|
||||
# ./labsim-dhcp-test.sh run the standard cases
|
||||
# ./labsim-dhcp-test.sh --keep leave the VMs up for inspection
|
||||
# ./labsim-dhcp-test.sh --clean just remove any leftover test VMs
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "$SCRIPT_DIR/lib.sh"
|
||||
|
||||
ROUTER_IP="${ROUTER_IP:-172.31.1.1}"
|
||||
ROUTER_PW="${ROUTER_PW:-vyos}"
|
||||
TEST_VLAN="${TEST_VLAN:-10}"
|
||||
BOOT_WAIT="${BOOT_WAIT:-150}"
|
||||
TAG="labsim-dhcptest"
|
||||
|
||||
# mac|expected|why. "POOL" means: must get an address from the pool and must
|
||||
# NOT get any reserved address -- the negative case that stops a pass from
|
||||
# meaning merely "DHCP works".
|
||||
CASES=(
|
||||
"f8:0d:ac:90:65:c6|172.31.10.46|printer1 - reservation inside the pool"
|
||||
"1c:69:20:7f:bc:77|172.31.11.67|sonoff-matter - in-pool AND across the /23 boundary"
|
||||
"34:e1:d1:80:29:ce|172.31.10.2|Hubitat - the one reservation OUTSIDE the pool"
|
||||
"52:54:00:ab:cd:ef|POOL|unreserved MAC - must get a pool address, not a reserved one"
|
||||
)
|
||||
|
||||
vm_of() { echo "${TAG}-$(echo "$1" | tr -d ':')"; }
|
||||
|
||||
cleanup_vms() {
|
||||
local n=0
|
||||
while read -r vm; do
|
||||
[ -z "$vm" ] && continue
|
||||
virsh_q destroy "$vm" >/dev/null 2>&1
|
||||
virsh_q undefine "$vm" --remove-all-storage >/dev/null 2>&1
|
||||
n=$((n + 1))
|
||||
done < <(virsh_q list --all --name 2>/dev/null | grep "^${TAG}-" || true)
|
||||
[ "$n" -gt 0 ] && log "removed $n test VM(s)"
|
||||
sudo rm -f "$IMG_DIR/${TAG}-"*.qcow2 "$IMG_DIR/${TAG}-"*-seed.iso 2>/dev/null
|
||||
return 0
|
||||
}
|
||||
|
||||
# A seed that asks for DHCP instead of taking a static address. Alpine's
|
||||
# cloud-init ignores network-config here (verified previously and documented in
|
||||
# README), so /etc/network/interfaces is what actually takes effect.
|
||||
build_dhcp_seed() {
|
||||
local iso="$1" vm="$2" pubkey="$3"
|
||||
local tmp; tmp="$(mktemp -d)"
|
||||
cat > "$tmp/meta-data" <<EOF
|
||||
instance-id: $vm
|
||||
local-hostname: $vm
|
||||
EOF
|
||||
cat > "$tmp/user-data" <<EOF
|
||||
#cloud-config
|
||||
hostname: $vm
|
||||
users:
|
||||
- name: alpine
|
||||
shell: /bin/ash
|
||||
lock_passwd: false
|
||||
plain_text_passwd: labsim
|
||||
ssh_authorized_keys:
|
||||
- $pubkey
|
||||
ssh_authorized_keys:
|
||||
- $pubkey
|
||||
disable_root: false
|
||||
chpasswd:
|
||||
list: |
|
||||
root:labsim
|
||||
expire: false
|
||||
write_files:
|
||||
- path: /etc/network/interfaces
|
||||
content: |
|
||||
auto lo
|
||||
iface lo inet loopback
|
||||
auto eth0
|
||||
iface eth0 inet dhcp
|
||||
runcmd:
|
||||
- [ sh, -c, "ifdown eth0 2>/dev/null; ifup eth0 || udhcpc -i eth0 -q || true" ]
|
||||
EOF
|
||||
python3 - "$tmp/user-data" <<'PY' || die "generated user-data is not valid YAML"
|
||||
import sys, yaml
|
||||
yaml.safe_load(open(sys.argv[1]).read().split("#cloud-config",1)[1])
|
||||
PY
|
||||
sudo genisoimage -quiet -output "$iso" -volid cidata -joliet -rock \
|
||||
"$tmp/user-data" "$tmp/meta-data" >/dev/null 2>&1 || die "seed build failed"
|
||||
rm -rf "$tmp"
|
||||
}
|
||||
|
||||
router() {
|
||||
timeout 30 sshpass -p "$ROUTER_PW" ssh -o StrictHostKeyChecking=no \
|
||||
-o BatchMode=no -o ConnectTimeout=8 "vyos@$ROUTER_IP" "$@" 2>/dev/null
|
||||
}
|
||||
|
||||
# --- argument handling ----------------------------------------------------
|
||||
KEEP=0
|
||||
case "${1:-}" in
|
||||
--clean) cleanup_vms; exit 0 ;;
|
||||
--keep) KEEP=1 ;;
|
||||
"") ;;
|
||||
*) die "usage: $0 [--keep|--clean]" ;;
|
||||
esac
|
||||
|
||||
command -v sshpass >/dev/null || die "sshpass required"
|
||||
require_tools
|
||||
[ -f "$BASE_IMAGE" ] || die "base image missing: $BASE_IMAGE (run labsim-up.sh first)"
|
||||
|
||||
log "checking the router is serving DHCP..."
|
||||
subnets=$(router '/opt/vyatta/bin/vyatta-op-cmd-wrapper show configuration commands | grep -c subnet-id')
|
||||
maps=$(router '/opt/vyatta/bin/vyatta-op-cmd-wrapper show configuration commands | grep -c "static-mapping .* mac"')
|
||||
log " router has ${subnets:-0} subnets and ${maps:-0} static-mappings"
|
||||
[ "${maps:-0}" -gt 0 ] || die "router has no static-mappings -- apply the generated config first"
|
||||
|
||||
cleanup_vms
|
||||
|
||||
# Flush the lease database first. This is not tidiness -- it is the condition
|
||||
# the cutover actually runs under, because kea does not inherit UniFi's leases
|
||||
# and starts empty. It also makes the test deterministic: with stale leases
|
||||
# present, kea saw the reserved address as held by "another client" (the same
|
||||
# MAC but a different client-id from a previous boot) and allocated a dynamic
|
||||
# address instead, which produced three misleading results before this existed.
|
||||
log "flushing the router's lease database (cutover starts with an empty one)"
|
||||
# Every dhcp4-leases.csv* must go, not just the main file: kea's memfile
|
||||
# backend keeps lease-file-cleanup rotations (.1/.2) and restores from them on
|
||||
# start, so truncating only the primary leaves the old leases intact.
|
||||
router 'sudo systemctl stop isc-kea-dhcp4-server;
|
||||
sudo sh -c "rm -f /config/dhcp/dhcp4-leases.csv*";
|
||||
sudo systemctl start isc-kea-dhcp4-server' >/dev/null
|
||||
sleep 5
|
||||
remaining="$(router '/opt/vyatta/bin/vyatta-op-cmd-wrapper show dhcp server leases' | sed -n '3,$p' | grep -c .)"
|
||||
[ "${remaining:-0}" -eq 0 ] || warn "lease table still has ${remaining} row(s) after flush"
|
||||
|
||||
SSH_PUB="$(find_ssh_pubkey)"
|
||||
sudo mkdir -p "$IMG_DIR"
|
||||
|
||||
# --- boot one VM per case -------------------------------------------------
|
||||
for c in "${CASES[@]}"; do
|
||||
IFS='|' read -r mac expected why <<<"$c"
|
||||
vm="$(vm_of "$mac")"
|
||||
disk="$IMG_DIR/${vm}.qcow2"; seed="$IMG_DIR/${vm}-seed.iso"
|
||||
log "booting $vm mac=$mac ($why)"
|
||||
sudo qemu-img create -q -f qcow2 -F qcow2 -b "$BASE_IMAGE" "$disk" "$VM_DISK" >/dev/null
|
||||
build_dhcp_seed "$seed" "$vm" "$SSH_PUB"
|
||||
sudo virt-install --connect "$LIBVIRT_URI" --name "$vm" \
|
||||
--memory "$VM_MEM" --vcpus "$VM_CPUS" \
|
||||
--disk "path=$disk,format=qcow2,bus=virtio" \
|
||||
--disk "path=$seed,device=cdrom,readonly=on" \
|
||||
--network "network=labsim-ovs,portgroup=vlan${TEST_VLAN},model=virtio,mac=$mac" \
|
||||
--os-variant alpinelinux3.18 --graphics none --noautoconsole --import >/dev/null \
|
||||
|| die "virt-install failed for $vm"
|
||||
done
|
||||
|
||||
log "waiting ${BOOT_WAIT}s for boot + DHCP..."
|
||||
sleep "$BOOT_WAIT"
|
||||
|
||||
# --- verdict --------------------------------------------------------------
|
||||
# The lease table on the router is the authority: it says what the server
|
||||
# decided, independent of whether the guest brought the interface up cleanly.
|
||||
leases="$(router '/opt/vyatta/bin/vyatta-op-cmd-wrapper show dhcp server leases')"
|
||||
echo
|
||||
echo "=== router lease table ==="
|
||||
echo "$leases"
|
||||
echo
|
||||
|
||||
reserved_ips="$(cd "$SCRIPT_DIR/../migration" && python3 unifi-to-vyos.py --mode sim 2>/dev/null \
|
||||
| awk '/static-mapping .* ip-address/ {print $NF}')"
|
||||
|
||||
pass=0; fail=0
|
||||
printf '%-19s %-16s %-16s %s\n' "MAC" "EXPECTED" "GOT" "RESULT"
|
||||
for c in "${CASES[@]}"; do
|
||||
IFS='|' read -r mac expected why <<<"$c"
|
||||
# Never guess which lease is "the" lease. Taking the first match is how an
|
||||
# hours-old lease was once reported as the current answer, turning three
|
||||
# failures into apparent passes.
|
||||
matches="$(echo "$leases" | awk -v m="$mac" 'tolower($2) == tolower(m) {print $1}')"
|
||||
n_match="$(echo "$matches" | grep -c . )"
|
||||
if [ "$n_match" -gt 1 ]; then
|
||||
got="AMBIGUOUS($(echo "$matches" | tr '\n' ',' | sed 's/,$//'))"
|
||||
else
|
||||
got="${matches:-<none>}"
|
||||
fi
|
||||
if [ "${got#AMBIGUOUS}" != "$got" ]; then
|
||||
# More than one lease for this MAC means the flush did not take. Any
|
||||
# verdict from here is a guess, so refuse to give one.
|
||||
result="FAIL (multiple leases -- flush did not take)"
|
||||
elif [ "$expected" = "POOL" ]; then
|
||||
if [ "$got" = "<none>" ]; then
|
||||
result="FAIL (no lease at all)"
|
||||
elif echo "$reserved_ips" | grep -qx "$got"; then
|
||||
result="FAIL (got a RESERVED address)"
|
||||
else
|
||||
result="pass"
|
||||
fi
|
||||
else
|
||||
[ "$got" = "$expected" ] && result="pass" || result="FAIL"
|
||||
fi
|
||||
[ "$result" = "pass" ] && pass=$((pass + 1)) || fail=$((fail + 1))
|
||||
printf '%-19s %-16s %-16s %s\n' "$mac" "$expected" "$got" "$result"
|
||||
printf ' %s\n' "$why"
|
||||
done
|
||||
|
||||
echo
|
||||
log "$pass passed, $fail failed"
|
||||
[ "$KEEP" -eq 1 ] && log "VMs left running (--keep). Remove with: $0 --clean" || cleanup_vms
|
||||
[ "$fail" -eq 0 ] || exit 1
|
||||
@@ -78,9 +78,15 @@ def load_vlans() -> list[dict]:
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
vid, name, prefix, real = line.split(":", 3)
|
||||
# masklen and host_octet are optional trailing fields; VLAN 10 sets
|
||||
# both because it must be a /23 (see vlans.conf).
|
||||
parts = line.split(":")
|
||||
vid, name, prefix, real = parts[0], parts[1], parts[2], parts[3]
|
||||
masklen = int(parts[4]) if len(parts) > 4 and parts[4] else 24
|
||||
host = parts[5] if len(parts) > 5 and parts[5] else "2"
|
||||
vlans.append({"vid": vid, "name": name, "ip": f"{prefix}.10",
|
||||
"label": f"{vid}:{name}", "real": real})
|
||||
"label": f"{vid}:{name}", "real": real,
|
||||
"masklen": masklen, "host_ip": f"{prefix}.{host}"})
|
||||
return vlans
|
||||
|
||||
|
||||
|
||||
@@ -25,7 +25,8 @@ ovs_up
|
||||
|
||||
# --- VMs ------------------------------------------------------------------
|
||||
for entry in "${SELECTED[@]}"; do
|
||||
IFS=: read -r vid name prefix real <<<"$entry"
|
||||
parse_vlan_entry "$entry"
|
||||
vid="$V_VID"; name="$V_NAME"; prefix="$V_PREFIX"; real="$V_REAL"
|
||||
vm="$(vm_name "$vid" "$name")"
|
||||
ip="${prefix}.10"
|
||||
|
||||
@@ -48,7 +49,7 @@ for entry in "${SELECTED[@]}"; do
|
||||
# Copy-on-write overlay: each VM costs a few MB, not 176.
|
||||
sudo qemu-img create -q -f qcow2 -F qcow2 -b "$BASE_IMAGE" "$disk" "$VM_DISK" >/dev/null
|
||||
|
||||
build_seed "$seed" "$vm" "$vid" "$name" "$prefix" "$ip" "$real" "$SSH_PUB"
|
||||
build_seed "$seed" "$vm" "$vid" "$name" "$prefix" "$ip" "$real" "$SSH_PUB" "$V_MASK"
|
||||
|
||||
sudo virt-install \
|
||||
--connect "$LIBVIRT_URI" \
|
||||
|
||||
@@ -58,9 +58,32 @@ selected_vlans() {
|
||||
[ ${#SELECTED[@]} -gt 0 ] || die "no VLANs selected (checked $CONF)"
|
||||
}
|
||||
|
||||
# Split one vlans.conf line, applying defaults for the two optional trailing
|
||||
# fields. Sets V_VID V_NAME V_PREFIX V_REAL V_MASK V_HOST.
|
||||
parse_vlan_entry() {
|
||||
IFS=: read -r V_VID V_NAME V_PREFIX V_REAL V_MASK V_HOST <<<"$1"
|
||||
V_MASK="${V_MASK:-24}"
|
||||
V_HOST="${V_HOST:-2}"
|
||||
}
|
||||
|
||||
# Dotted netmask for a prefix length — cloud-init's network-config v1 wants the
|
||||
# dotted form, not a /len. /24 -> 255.255.255.0, /23 -> 255.255.254.0.
|
||||
netmask_for() {
|
||||
local len="$1" i bits out=()
|
||||
for i in 0 1 2 3; do
|
||||
bits=$(( len - i * 8 ))
|
||||
(( bits > 8 )) && bits=8
|
||||
(( bits < 0 )) && bits=0
|
||||
out+=( $(( 256 - 2 ** (8 - bits) )) )
|
||||
done
|
||||
local IFS=.; echo "${out[*]}"
|
||||
}
|
||||
|
||||
# cloud-init NoCloud seed: static addressing + SSH key + hello-world HTTP.
|
||||
build_seed() {
|
||||
local iso="$1" vm="$2" vid="$3" name="$4" prefix="$5" ip="$6" real="$7" pubkey="$8"
|
||||
local masklen="${9:-24}"
|
||||
local netmask; netmask="$(netmask_for "$masklen")"
|
||||
local tmp; tmp="$(mktemp -d)"
|
||||
|
||||
cat > "$tmp/meta-data" <<EOF
|
||||
@@ -85,7 +108,7 @@ config:
|
||||
subnets:
|
||||
- type: static
|
||||
address: $ip
|
||||
netmask: 255.255.255.0
|
||||
netmask: $netmask
|
||||
# Default route via the router under test. Without this the VMs can
|
||||
# reach their own /24 and their gateway, but nothing beyond it — which
|
||||
# looks exactly like "the router is broken" in the matrix.
|
||||
@@ -122,14 +145,14 @@ write_files:
|
||||
auto eth0
|
||||
iface eth0 inet static
|
||||
address $ip
|
||||
netmask 255.255.255.0
|
||||
netmask $netmask
|
||||
post-up ip route add default via ${prefix}.1 || true
|
||||
- path: /var/www/index.html
|
||||
content: |
|
||||
<html><body>
|
||||
<h1>labsim vlan $vid — $name</h1>
|
||||
<p>host: $vm</p>
|
||||
<p>address: $ip/24</p>
|
||||
<p>address: $ip/$masklen</p>
|
||||
<p>gateway under test: ${prefix}.1</p>
|
||||
<p>mirrors production: $real</p>
|
||||
</body></html>
|
||||
|
||||
@@ -41,12 +41,20 @@ ovs_up() {
|
||||
# default route (.1 is), so inter-VLAN tests exercise the router, not the
|
||||
# host's routing table.
|
||||
for entry in "${SELECTED[@]}"; do
|
||||
IFS=: read -r vid _name prefix _real <<<"$entry"
|
||||
local port="hostv${vid}"
|
||||
ovs --may-exist add-port "$OVS_BR" "$port" tag="$vid" \
|
||||
parse_vlan_entry "$entry"
|
||||
local port="hostv${V_VID}"
|
||||
ovs --may-exist add-port "$OVS_BR" "$port" tag="$V_VID" \
|
||||
-- set interface "$port" type=internal
|
||||
sudo ip link set "$port" up 2>/dev/null || true
|
||||
sudo ip addr replace "${prefix}.2/24" dev "$port"
|
||||
# Drop any address from a previous mask/octet so a changed vlans.conf does
|
||||
# not leave a stale second address on the port.
|
||||
sudo ip -4 addr flush dev "$port" 2>/dev/null || true
|
||||
# host_octet 0 means "no host leg": the WAN transport VLANs belong to the
|
||||
# fake ISPs, and giving the host an address there would misrepresent the
|
||||
# segment -- the whole point is that VyOS reaches an ISP, not the host.
|
||||
if [ "$V_HOST" != "0" ]; then
|
||||
sudo ip addr replace "${V_PREFIX}.${V_HOST}/${V_MASK}" dev "$port"
|
||||
fi
|
||||
done
|
||||
|
||||
ovs_define_libvirt_net
|
||||
@@ -114,9 +122,19 @@ ovs_bond_router() {
|
||||
local count; count="$(echo "$taps" | grep -c .)"
|
||||
[ "$count" -eq 2 ] || { warn "router $vm has $count tap(s), expected 2 — skipping bond"; return 1; }
|
||||
|
||||
# Already bonded? (idempotent re-runs)
|
||||
# Already bonded? Re-runs must still reconcile the VLAN list: adding a VLAN to
|
||||
# vlans.conf and finding the bond unchanged is exactly how a VLAN silently
|
||||
# fails to reach a router -- interface present, tag missing, frames dropped by
|
||||
# the switch. Returning early here once cost real debugging time.
|
||||
if ovs list-ports "$OVS_BR" 2>/dev/null | grep -qx "$LAG_NAME"; then
|
||||
log "LACP bond $LAG_NAME already present"
|
||||
local want; want="$(vlan_id_list | tr ',' '\n' | grep -vx 1 | paste -sd, -)"
|
||||
local have; have="$(ovs get port "$LAG_NAME" trunks 2>/dev/null | tr -d '[] ')"
|
||||
if [ "$want" != "$have" ]; then
|
||||
log "bond $LAG_NAME trunk drift: [$have] -> [$want]; updating"
|
||||
ovs set port "$LAG_NAME" trunks="$want"
|
||||
else
|
||||
log "LACP bond $LAG_NAME already present, trunk correct"
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
|
||||
|
||||
109
labsim/sim-ha-config.py
Executable file
109
labsim/sim-ha-config.py
Executable file
@@ -0,0 +1,109 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate the HA config for the labsim VyOS pair.
|
||||
|
||||
Exists to answer one question that cannot be answered on a single router, and
|
||||
that would otherwise only be discovered at cutover: with kea HA active-passive,
|
||||
does exactly ONE box answer a DHCP request?
|
||||
|
||||
Mirrors the production shape so the answer transfers:
|
||||
|
||||
router1 172.31.<v>.252 priority 200 DHCP HA primary
|
||||
router2 172.31.<v>.253 priority 100 DHCP HA secondary
|
||||
VIP 172.31.<v>.1 (what clients use as their gateway)
|
||||
|
||||
Note the sim's LoT VLAN is a /23 like production, so the VIP prefix differs
|
||||
there -- getting that wrong produces a config that commits and then behaves
|
||||
subtly wrongly, which is worse than a failure.
|
||||
|
||||
./sim-ha-config.py --role primary > r1.conf
|
||||
./sim-ha-config.py --role secondary > r2.conf
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
MIG = os.path.join(HERE, "..", "migration")
|
||||
|
||||
# Reuse the DHCP/DNS generator rather than hand-writing subnets: the whole
|
||||
# point is that what is proven here and what production gets share a code path.
|
||||
_spec = importlib.util.spec_from_file_location(
|
||||
"unifi_to_vyos", os.path.join(MIG, "unifi-to-vyos.py"))
|
||||
unifi_to_vyos = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(unifi_to_vyos)
|
||||
|
||||
# vlan -> (prefix, cidr). LoT is a /23 in the sim, matching production.
|
||||
VLANS = {
|
||||
1: ("172.31.1", 24),
|
||||
2: ("172.31.2", 24),
|
||||
3: ("172.31.3", 24),
|
||||
9: ("172.31.9", 24),
|
||||
10: ("172.31.10", 23),
|
||||
200: ("172.31.200", 24),
|
||||
}
|
||||
DHCP_HA_NAME = "labsim-dhcp-pair" # must not equal either host-name
|
||||
|
||||
|
||||
def group(vlan: int) -> str:
|
||||
return "native" if vlan == 1 else f"vlan{vlan}"
|
||||
|
||||
|
||||
def build(role: str) -> list[str]:
|
||||
primary = role == "primary"
|
||||
self_o, peer_o = (252, 253) if primary else (253, 252)
|
||||
prio = 200 if primary else 100
|
||||
out = [f"# labsim VyOS HA -- {role}", ""]
|
||||
|
||||
for vlan, (pfx, cidr) in VLANS.items():
|
||||
g = group(vlan)
|
||||
iface = "bond0" if vlan == 1 else f"bond0 vif {vlan}"
|
||||
out += [
|
||||
f"# VLAN {vlan}",
|
||||
# The node's own address replaces the .1 it used to hold directly;
|
||||
# .1 becomes the floating VIP, exactly as production will be.
|
||||
f"delete interfaces bonding {iface} address",
|
||||
f"set interfaces bonding {iface} address '{pfx}.{self_o}/{cidr}'",
|
||||
f"set high-availability vrrp group {g} interface bond0{'' if vlan == 1 else f'.{vlan}'}",
|
||||
f"set high-availability vrrp group {g} vrid {vlan}",
|
||||
f"set high-availability vrrp group {g} address {pfx}.1/{cidr}",
|
||||
f"set high-availability vrrp group {g} priority {prio}",
|
||||
f"set high-availability vrrp group {g} hello-source-address {pfx}.{self_o}",
|
||||
f"set high-availability vrrp group {g} peer-address {pfx}.{peer_o}",
|
||||
f"set high-availability vrrp group {g} no-preempt",
|
||||
f"set high-availability vrrp sync-group MAIN member {g}",
|
||||
"",
|
||||
]
|
||||
|
||||
out += [
|
||||
"# --- DHCP high-availability ---",
|
||||
"# The thing under test: active-passive should mean exactly one OFFER.",
|
||||
"set service dhcp-server high-availability mode active-passive",
|
||||
f"set service dhcp-server high-availability status {role}",
|
||||
f"set service dhcp-server high-availability name {DHCP_HA_NAME}",
|
||||
f"set service dhcp-server high-availability source-address 172.31.10.{self_o}",
|
||||
f"set service dhcp-server high-availability remote 172.31.10.{peer_o}",
|
||||
"",
|
||||
]
|
||||
|
||||
inv = json.load(open(os.path.join(MIG, "export", "inventory.json")))
|
||||
dhcp, stats = unifi_to_vyos.build(inv, "sim")
|
||||
out += [l for l in dhcp if l.strip() and not l.startswith("#")]
|
||||
print(f"{role}: {stats['subnets']} subnets, {stats['mappings']} mappings",
|
||||
file=sys.stderr)
|
||||
return out
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--role", choices=("primary", "secondary"), required=True)
|
||||
args = ap.parse_args()
|
||||
sys.stdout.write("\n".join(build(args.role)) + "\n")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -12,10 +12,32 @@
|
||||
# .10 the micro VM for this VLAN
|
||||
# .254 VRRP VIP (reserved, mirrors production)
|
||||
#
|
||||
# Format: vlan_id:name:sim_subnet_prefix:real_subnet(for reference)
|
||||
# Format: vlan_id:name:sim_subnet_prefix:real_subnet:[masklen]:[host_octet]
|
||||
#
|
||||
# masklen defaults to 24 and host_octet to 2. Both exist for VLAN 10, which is
|
||||
# the one VLAN that has to be a /23 here: every UniFi DHCP reservation lives in
|
||||
# LoT, and LoT spans 10.0.0.x AND 10.0.1.x, which a /24 cannot represent. With
|
||||
# /23 the mapping stays readable — 10.0.0.46 -> 172.31.10.46 and
|
||||
# 10.0.1.67 -> 172.31.11.67.
|
||||
#
|
||||
# LoT's host leg is .3 rather than .2 because 10.0.0.2 is a real reservation
|
||||
# (Hubitat) and would map straight onto the host's own address. .3 is free in
|
||||
# production and sits below the DHCP pool (which starts at .11), so it can
|
||||
# never be handed out.
|
||||
1:management:172.31.1:192.168.1.0/24
|
||||
2:k8s:172.31.2:192.168.8.0/23
|
||||
3:kvm:172.31.3:192.168.3.0/24
|
||||
9:private:172.31.9:10.0.9.0/23
|
||||
10:lot:172.31.10:10.0.0.0/23
|
||||
10:lot:172.31.10:10.0.0.0/23:23:3
|
||||
200:roomates:172.31.200:192.168.2.0/24
|
||||
|
||||
# WAN transport VLANs, mirroring production. These exist so the sim can run a
|
||||
# fake ISP on each and the switch script's WAN health checks actually execute
|
||||
# instead of printing "this delta configures no WAN -- skipping". A cutover
|
||||
# attempt failed on the WAN with nothing having tested it, because the sim
|
||||
# modelled every LAN VLAN faithfully and omitted the WAN entirely.
|
||||
#
|
||||
# No host leg is wanted here (host_octet 0 means "skip"): the ISP VMs own these
|
||||
# segments, and a host address on a WAN transport VLAN would be a lie.
|
||||
51:wan1:172.31.51:vodafone-pppoe(VLAN 51):24:0
|
||||
53:wan3:172.31.53:10gig-dhcp(VLAN 53):24:0
|
||||
|
||||
4
migration/.gitignore
vendored
Normal file
4
migration/.gitignore
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
# Raw UniFi export: contains WiFi passphrases (wlanconf) and controller auth
|
||||
# material (setting). The inventory is regenerable — never commit it.
|
||||
export/
|
||||
__pycache__/
|
||||
209
migration/CUTOVER.md
Normal file
209
migration/CUTOVER.md
Normal file
@@ -0,0 +1,209 @@
|
||||
# Cutover runbook — USG to VyOS
|
||||
|
||||
**Print this.** During the cutover there is no internet, so there is no
|
||||
assistant and no web search. Everything you need is on this page and on the
|
||||
boxes themselves.
|
||||
|
||||
## Use these addresses. Not the other ones.
|
||||
|
||||
| | use this | do NOT use |
|
||||
|---|---|---|
|
||||
| vyos001 (MASTER) | **`10.0.1.252`** | ~~192.168.8.143~~ |
|
||||
| vyos002 (BACKUP) | **`10.0.1.253`** | ~~192.168.8.144~~ |
|
||||
|
||||
`ssh vyos@10.0.1.252` — by IP, not by name.
|
||||
|
||||
**The `192.168.8.x` addresses stop working the instant the USG is unplugged.**
|
||||
That is not a maybe. Your workstation is on LoT (`10.0.0.210/23`) and reaching
|
||||
`192.168.8.x` requires routing *through the USG*:
|
||||
|
||||
```
|
||||
ip route get 192.168.8.143 -> via 10.0.0.1 <- the USG. Gone.
|
||||
ip route get 10.0.1.252 -> dev lanbr0 <- same L2. Survives.
|
||||
```
|
||||
|
||||
`10.0.1.252` and `.253` are on the LoT VLAN, the same broadcast domain as your
|
||||
workstation, so they need no gateway at all. They are the only remote path that
|
||||
survives the cutover.
|
||||
|
||||
**Between unplugging the USG and finishing the switch there is no inter-VLAN
|
||||
routing.** In that window:
|
||||
|
||||
- the **JetKVMs are unreachable** from your workstation (they are on Management
|
||||
and kvm) — they are *not* a fallback during the gap
|
||||
- **Tailscale is down** with the internet
|
||||
- your workstation keeps `10.0.0.210` (86400s lease) and can still resolve via
|
||||
`10.0.0.194`, which is also link-scope
|
||||
|
||||
If LoT SSH fails, the next step is physical console, not the network.
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| JetKVMs (after routing is restored) | `192.168.1.28`, `192.168.1.29`, `192.168.3.6` |
|
||||
| Switch script | `/config/vyos-unifi-switch` on each box |
|
||||
| Peer link | `eth3` ↔ `eth3` direct cable, 2.5 GbE, `10.255.255.0/30` — conntrack state sync |
|
||||
| Login | user `vyos` |
|
||||
|
||||
---
|
||||
|
||||
## If something is wrong, do this
|
||||
|
||||
```
|
||||
sudo /config/vyos-unifi-switch unifi
|
||||
```
|
||||
|
||||
Then reconnect the USG. That command runs no health checks, asks nothing and
|
||||
cannot refuse. It restores a byte-exact copy of the configuration the box had
|
||||
before the cutover — verified by diff, not by assumption.
|
||||
|
||||
**You do not have to be quick.** If you do nothing at all after
|
||||
`vyos-unifi-switch vyos`, the box reverts by itself within 10 minutes. Verified:
|
||||
config returns to the previous state and the box does **not** reboot
|
||||
(`uptime` and boot-id unchanged across an auto-revert).
|
||||
|
||||
---
|
||||
|
||||
## What has actually been tested
|
||||
|
||||
Proven on the labsim router (same VyOS version, isolated OVS bridge with no
|
||||
physical NIC), by loading **vyos001's real running config** and applying the
|
||||
**real production delta**:
|
||||
|
||||
- All 317 commands accepted, and the whole delta **commits** (`COMMIT OK`).
|
||||
- `unifi` mode restores the previous config **byte-exact** (diff clean).
|
||||
- Auto-revert fires when the commit is not confirmed: config returns to the
|
||||
saved state and the box does **not** reboot — `uptime` and boot-id unchanged
|
||||
across the revert.
|
||||
- Failed health checks trigger an immediate revert rather than waiting out the
|
||||
timer.
|
||||
|
||||
Two bugs were found this way and would each have failed the entire switch,
|
||||
since the delta commits as one unit: `bond0.51` did not exist for PPPoE to
|
||||
reference, and `translation port` rejects a port list.
|
||||
|
||||
**Not tested, and untestable in advance:**
|
||||
|
||||
- **PPPoE.** The line permits one session and the USG holds it. The first real
|
||||
attempt is during the cutover.
|
||||
- **The commit on the real boxes.** The rehearsal ran with vyos001's `eth2` and
|
||||
`eth3` stanzas stripped, because the sim VM has only two NICs. Those are
|
||||
plain interface configs that already work on the real hardware, but they were
|
||||
not part of what committed.
|
||||
|
||||
## Before you unplug anything
|
||||
|
||||
1. Tether your workstation to your phone if you want the assistant available.
|
||||
Cutting the USG cuts your internet, not your LAN.
|
||||
2. On **both** boxes, confirm the machinery is present:
|
||||
```
|
||||
sudo /config/vyos-unifi-switch status
|
||||
ls -la /config/modes/ # unifi.boot + to-vyos.commands
|
||||
ls -la /config/wan-secrets # must be 0600
|
||||
```
|
||||
`status` must report `mode: unifi`. If `unifi.boot` is missing, **stop** —
|
||||
there is no way back without it.
|
||||
3. Confirm the revert action is `reload`, not `reboot`:
|
||||
```
|
||||
show configuration commands | match commit-confirm
|
||||
```
|
||||
Must show `action 'reload'`. Without it a failed switch **reboots** the
|
||||
firewall instead of reverting it. The switch script refuses to run if this
|
||||
is missing, but check anyway.
|
||||
|
||||
## The cutover
|
||||
|
||||
0. **Open both SSH sessions BEFORE you unplug anything**, and leave them open:
|
||||
```
|
||||
ssh vyos@10.0.1.253 # vyos002, BACKUP
|
||||
ssh vyos@10.0.1.252 # vyos001, MASTER
|
||||
```
|
||||
If either will not connect, stop. Do not unplug the USG.
|
||||
|
||||
1. **Physically disconnect the USG.** Not just powered off — disconnected. The
|
||||
switch script refuses to run while anything still answers on a gateway
|
||||
address, because two devices on `.1` is the worst available outcome.
|
||||
You cannot switch first and unplug after, for exactly that reason.
|
||||
|
||||
2. In the **vyos002 (BACKUP)** session, first:
|
||||
```
|
||||
sudo /config/vyos-unifi-switch vyos
|
||||
```
|
||||
3. Watch the health checks. They cover PPPoE, the default route, kea, the DNS
|
||||
forwarder and reachability. On failure the script reverts immediately and
|
||||
tells you so.
|
||||
4. If vyos002 came up clean, repeat on **vyos001 (MASTER)**.
|
||||
5. Check a real client: does it get an address, and is it the *same* address as
|
||||
before? Every active client has a reservation, so it should be.
|
||||
|
||||
## What will probably go wrong first
|
||||
|
||||
**The WAN.** There are two, and they behave differently:
|
||||
|
||||
| | line | VLAN | transport | notes |
|
||||
|---|---|---|---|---|
|
||||
| WAN2 | 10 gig ISP | **53** | DHCP, public `87.192.101.48/21` | primary, distance 1 |
|
||||
| WAN1 | Vodafone | **51** | PPPoE, ~900/700 Mbit | failover, distance 10 |
|
||||
|
||||
VyOS clones the USG's WAN2 MAC (`f0:9f:c2:12:9b:4f`) on `bond0.53`, which is how
|
||||
it keeps the existing public lease rather than asking for a new one.
|
||||
|
||||
**Both boxes carry the identical WAN and NAT config.** vyos002's WAN interfaces
|
||||
are simply held administratively down, so the cloned MAC is never live on two
|
||||
boxes at once. To move the internet path to vyos002:
|
||||
|
||||
```
|
||||
configure
|
||||
delete interfaces bonding bond0 vif 53 disable
|
||||
delete interfaces pppoe pppoe0 disable
|
||||
commit; save
|
||||
```
|
||||
|
||||
Two lines. Do it only when vyos001 is genuinely down or disconnected — two boxes
|
||||
holding that MAC at once is exactly what the disable prevents.
|
||||
|
||||
PPPoE is no longer an unknown: it was proven on the USG before cutover
|
||||
(`pppoe0` came up with `90.241.226.213`, MTU 1492). What remains untested is
|
||||
VyOS dialling it, and whether the ISP hands the same lease to the cloned MAC.
|
||||
|
||||
If the WAN check fails:
|
||||
|
||||
```
|
||||
show interfaces pppoe pppoe0
|
||||
sudo journalctl -u ppp@pppoe0 -n 50 --no-pager
|
||||
```
|
||||
|
||||
Check the credential in `/config/wan-secrets` and that VLAN 51 actually reaches
|
||||
the box. If it will not come up, run `vyos-unifi-switch unifi`, reconnect the
|
||||
USG, and debug with the internet back on.
|
||||
|
||||
## Things that are true and easy to forget
|
||||
|
||||
- **WiFi keeps working, but through VyOS.** The SSIDs stay in UniFi and the APs
|
||||
are untouched, but 37 of 83 active clients are wireless and every one is on
|
||||
LoT — they get their addresses from VyOS now.
|
||||
- **DHCP leases last 24h (86400s).** A device that does not renew promptly keeps
|
||||
its old address for a while. That is fine, not a symptom.
|
||||
- **The firewalls resolve via `8.8.8.8` / `8.8.4.4`** — matching the DNS the USG
|
||||
used on its WAN. This means their own name resolution now depends on the
|
||||
*internet* being up, so between unplugging the USG and PPPoE establishing,
|
||||
the boxes have no DNS at all. That is expected and harmless: they only need
|
||||
DNS for NTP hostnames, and the switch's own health checks use it precisely to
|
||||
prove the WAN came up. Nothing in the switch itself resolves a name.
|
||||
- Internal `ad.itaz.eu` names still resolve through Google, because that zone is
|
||||
published publicly with private addresses in it (`nas001` → `10.0.0.194`,
|
||||
`kvm-macstudio1` → `192.168.3.8`). Convenient here; worth knowing it is public.
|
||||
- **The USG was a DNS resolver** for every VLAN except LoT. VyOS now runs
|
||||
`dns forwarding` in its place. If names stop resolving but IPs still work,
|
||||
that is where to look.
|
||||
- **`eth2` and `bond0.2` are both in `192.168.8.0/23`.** It works, but if you
|
||||
see odd source-address behaviour on the management NIC, that is why.
|
||||
|
||||
## Afterwards
|
||||
|
||||
Once it has been stable for a day:
|
||||
|
||||
- Re-run `migration/unifi-export.py` — the UniFi controller is no longer the
|
||||
source of truth for DHCP, and the export will drift.
|
||||
- The VPN rules (ESP, UDP 500/4500) are carried over but the VPN itself still
|
||||
terminated on the USG. Decide whether it moves.
|
||||
- `labsim` still holds a deliberate `kvm→k8s` drop rule from earlier testing.
|
||||
84
migration/_unifi.py
Executable file
84
migration/_unifi.py
Executable file
@@ -0,0 +1,84 @@
|
||||
"""Shared UniFi API client for the migration tooling.
|
||||
|
||||
The controller is a CLASSIC self-hosted UniFi Network app (server_version
|
||||
10.4.x), not UniFi OS: login is /api/login and data lives under
|
||||
/api/s/<site>/... . UniFi OS would use /api/auth/login + /proxy/network/api.
|
||||
Credentials come from the mcpctl server definition so they are not duplicated
|
||||
here.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json, re, ssl, subprocess, urllib.request, http.cookiejar
|
||||
|
||||
|
||||
def client():
|
||||
raw = subprocess.run(["mcpctl", "describe", "server", "unifi-network"],
|
||||
capture_output=True, text=True).stdout
|
||||
m = re.search(r"UNIFI_TARGETS\s+(\[.*)", raw)
|
||||
if not m:
|
||||
raise SystemExit("could not read UNIFI_TARGETS from mcpctl")
|
||||
blob = m.group(1).strip()
|
||||
try:
|
||||
targets = json.loads(blob)
|
||||
except json.JSONDecodeError:
|
||||
targets = json.loads(blob + "}" * (blob.count("{") - blob.count("}")))
|
||||
t = targets[0]
|
||||
base = t["base_url"].rstrip("/")
|
||||
auth = t.get("auth", {})
|
||||
site = t.get("default_site", "default")
|
||||
|
||||
ctx = ssl.create_default_context()
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
opener = urllib.request.build_opener(
|
||||
urllib.request.HTTPCookieProcessor(http.cookiejar.CookieJar()),
|
||||
urllib.request.HTTPSHandler(context=ctx))
|
||||
req = urllib.request.Request(
|
||||
f"{base}/api/login",
|
||||
data=json.dumps({"username": auth.get("username"),
|
||||
"password": auth.get("password")}).encode(),
|
||||
headers={"Content-Type": "application/json"})
|
||||
opener.open(req, timeout=20).read()
|
||||
return opener, base, site
|
||||
|
||||
|
||||
def get(opener, base, site, path):
|
||||
"""GET /api/s/<site>/<path>, returning the `data` list (never raising)."""
|
||||
try:
|
||||
body = opener.open(f"{base}/api/s/{site}/{path}", timeout=30).read()
|
||||
return json.loads(body).get("data", [])
|
||||
except Exception as exc:
|
||||
return {"__error__": f"{type(exc).__name__}: {exc}"}
|
||||
|
||||
|
||||
def post(opener, base, site, path, payload):
|
||||
"""POST to /api/s/<site>/<path> -- used for device commands (cmd/devmgr)."""
|
||||
req = urllib.request.Request(
|
||||
f"{base}/api/s/{site}/{path}",
|
||||
data=json.dumps(payload).encode(),
|
||||
headers={"Content-Type": "application/json"}, method="POST")
|
||||
try:
|
||||
return json.loads(opener.open(req, timeout=30).read()).get("data", [])
|
||||
except urllib.error.HTTPError as exc:
|
||||
return {"__error__": f"HTTP {exc.code}: {exc.read()[:300].decode(errors='replace')}"}
|
||||
except Exception as exc:
|
||||
return {"__error__": f"{type(exc).__name__}: {exc}"}
|
||||
|
||||
|
||||
def put(opener, base, site, path, payload):
|
||||
"""PUT to /api/s/<site>/<path>. Returns the `data` list or an __error__ dict.
|
||||
|
||||
Classic controllers accept the session cookie alone -- no CSRF token, which
|
||||
UniFi OS would require. Errors are returned rather than raised so a caller
|
||||
changing production config can report and stop rather than traceback.
|
||||
"""
|
||||
req = urllib.request.Request(
|
||||
f"{base}/api/s/{site}/{path}",
|
||||
data=json.dumps(payload).encode(),
|
||||
headers={"Content-Type": "application/json"}, method="PUT")
|
||||
try:
|
||||
return json.loads(opener.open(req, timeout=30).read()).get("data", [])
|
||||
except urllib.error.HTTPError as exc:
|
||||
return {"__error__": f"HTTP {exc.code}: {exc.read()[:300].decode(errors='replace')}"}
|
||||
except Exception as exc:
|
||||
return {"__error__": f"{type(exc).__name__}: {exc}"}
|
||||
335
migration/unifi-export.py
Executable file
335
migration/unifi-export.py
Executable file
@@ -0,0 +1,335 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Export everything from the UniFi controller that the VyOS cutover must preserve.
|
||||
|
||||
The point is that nothing quietly stops working after the switch. That means
|
||||
capturing not just the networks but every DHCP reservation, every port forward
|
||||
and every firewall rule — the things nobody remembers configuring until they
|
||||
break.
|
||||
|
||||
Writes one JSON file per endpoint into ./export/ (raw, unmodified — the source
|
||||
of truth) plus inventory.json, a normalised view used by the VyOS generator.
|
||||
|
||||
./unifi-export.py # export to ./export/
|
||||
./unifi-export.py --out /tmp/x # elsewhere
|
||||
./unifi-export.py --summary # print a human summary of what was found
|
||||
|
||||
WARNING: the raw export contains secrets (wlanconf holds WiFi passphrases,
|
||||
setting holds RADIUS/auth material). ./export/ is gitignored — keep it that way.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
import _unifi
|
||||
|
||||
# endpoint -> why it matters for the cutover
|
||||
ENDPOINTS = {
|
||||
"rest/networkconf": "networks: VLANs, subnets, DHCP ranges, DNS, lease time",
|
||||
"rest/user": "known clients — this is where fixed DHCP reservations live",
|
||||
"stat/sta": "currently active clients and their live IPs",
|
||||
"rest/firewallrule": "firewall rules",
|
||||
"rest/firewallgroup": "address/port groups referenced by rules",
|
||||
"rest/portforward": "port forwards (inbound NAT)",
|
||||
"rest/routing": "static routes",
|
||||
"rest/dhcpoption": "custom DHCP options",
|
||||
"rest/wlanconf": "wireless networks (VLAN bindings)",
|
||||
"stat/device": "switches/APs incl. per-port VLAN config",
|
||||
"rest/setting": "controller settings (incl. USG/gateway config)",
|
||||
"rest/usergroup": "bandwidth groups referenced by clients",
|
||||
"rest/dynamicdns": "dynamic DNS",
|
||||
}
|
||||
|
||||
|
||||
def build_inventory(raw: dict) -> dict:
|
||||
"""Normalise the parts a migration actually has to reproduce."""
|
||||
nets_by_id = {n["_id"]: n for n in raw.get("rest/networkconf", []) if isinstance(n, dict)}
|
||||
|
||||
networks = []
|
||||
for n in raw.get("rest/networkconf", []):
|
||||
if not isinstance(n, dict):
|
||||
continue
|
||||
networks.append({
|
||||
"id": n.get("_id"),
|
||||
"name": n.get("name"),
|
||||
"purpose": n.get("purpose"),
|
||||
"vlan": n.get("vlan"),
|
||||
"vlan_enabled": n.get("vlan_enabled"),
|
||||
"subnet": n.get("ip_subnet"),
|
||||
"domain_name": n.get("domain_name"),
|
||||
"dhcp_enabled": n.get("dhcpd_enabled"),
|
||||
"dhcp_start": n.get("dhcpd_start"),
|
||||
"dhcp_stop": n.get("dhcpd_stop"),
|
||||
"dhcp_lease": n.get("dhcpd_leasetime"),
|
||||
"dhcp_dns": [n.get(f"dhcpd_dns_{i}") for i in (1, 2, 3, 4) if n.get(f"dhcpd_dns_{i}")],
|
||||
"dhcp_gateway": n.get("dhcpd_gateway") or n.get("dhcpd_gateway_enabled"),
|
||||
"dhcp_ntp": [n.get(f"dhcpd_ntp_{i}") for i in (1, 2) if n.get(f"dhcpd_ntp_{i}")],
|
||||
"igmp_snooping": n.get("igmp_snooping"),
|
||||
"enabled": n.get("enabled", True),
|
||||
})
|
||||
|
||||
# ip_subnet is the GATEWAY address with a prefix ("10.0.0.1/23"), not the
|
||||
# network address — so derive the real subnet before matching against it.
|
||||
subnets = []
|
||||
for n in networks:
|
||||
if not n["subnet"]:
|
||||
continue
|
||||
try:
|
||||
iface = ipaddress.ip_interface(n["subnet"])
|
||||
except ValueError:
|
||||
continue
|
||||
subnets.append((iface.network, n))
|
||||
|
||||
def resolve_net(ip: str | None, network_id: str | None) -> dict:
|
||||
"""Which network does this reservation belong to?
|
||||
|
||||
Most reservations here (23 of 31 as of the first export) carry no
|
||||
network_id at all — UniFi simply does not bind them. VyOS needs the
|
||||
subnet to place a static-mapping, so fall back to containment.
|
||||
"""
|
||||
if network_id and network_id in nets_by_id:
|
||||
nid = nets_by_id[network_id]
|
||||
return {"name": nid.get("name"), "vlan": nid.get("vlan"), "by": "network_id"}
|
||||
if ip:
|
||||
try:
|
||||
addr = ipaddress.ip_address(ip)
|
||||
except ValueError:
|
||||
return {"name": None, "vlan": None, "by": "unresolved"}
|
||||
for net, meta in subnets:
|
||||
if addr in net:
|
||||
return {"name": meta["name"], "vlan": meta["vlan"], "by": "subnet"}
|
||||
return {"name": None, "vlan": None, "by": "unresolved"}
|
||||
|
||||
# Fixed reservations: the single most important thing to carry over, and
|
||||
# the easiest to lose — nobody has these written down anywhere else.
|
||||
reservations = []
|
||||
for u in raw.get("rest/user", []):
|
||||
if not isinstance(u, dict) or not u.get("use_fixedip"):
|
||||
continue
|
||||
net = resolve_net(u.get("fixed_ip"), u.get("network_id"))
|
||||
reservations.append({
|
||||
"mac": (u.get("mac") or "").lower(),
|
||||
"ip": u.get("fixed_ip"),
|
||||
"name": u.get("name") or u.get("hostname") or "",
|
||||
"hostname": u.get("hostname") or "",
|
||||
"network_id": u.get("network_id"),
|
||||
"network_name": net["name"],
|
||||
"network_vlan": net["vlan"],
|
||||
"resolved_by": net["by"],
|
||||
"note": (u.get("note") or "").strip(),
|
||||
})
|
||||
reservations.sort(key=lambda r: tuple(int(p) for p in r["ip"].split(".")) if r["ip"] else (0,))
|
||||
|
||||
# Active leases without a reservation: these devices work today by luck of
|
||||
# the lease database. After a DHCP server swap they get a NEW address.
|
||||
reserved_macs = {r["mac"] for r in reservations}
|
||||
dynamic = []
|
||||
for c in raw.get("stat/sta", []):
|
||||
if not isinstance(c, dict):
|
||||
continue
|
||||
mac = (c.get("mac") or "").lower()
|
||||
if mac in reserved_macs or not c.get("ip"):
|
||||
continue
|
||||
dynamic.append({
|
||||
"mac": mac,
|
||||
"ip": c.get("ip"),
|
||||
"name": c.get("name") or c.get("hostname") or "",
|
||||
"network": c.get("network"),
|
||||
})
|
||||
dynamic.sort(key=lambda r: tuple(int(p) for p in r["ip"].split(".")) if r["ip"] else (0,))
|
||||
|
||||
port_forwards = [{
|
||||
"name": p.get("name"), "enabled": p.get("enabled"),
|
||||
"proto": p.get("proto"), "src": p.get("src"),
|
||||
"dst_port": p.get("dst_port"), "fwd": p.get("fwd"),
|
||||
"fwd_port": p.get("fwd_port"), "log": p.get("log"),
|
||||
} for p in raw.get("rest/portforward", []) if isinstance(p, dict)]
|
||||
|
||||
firewall_rules = [{
|
||||
"name": r.get("name"), "enabled": r.get("enabled"), "action": r.get("action"),
|
||||
"ruleset": r.get("ruleset"), "rule_index": r.get("rule_index"),
|
||||
"protocol": r.get("protocol"),
|
||||
"src_address": r.get("src_address"), "dst_address": r.get("dst_address"),
|
||||
"src_firewallgroup_ids": r.get("src_firewallgroup_ids"),
|
||||
"dst_firewallgroup_ids": r.get("dst_firewallgroup_ids"),
|
||||
"src_networkconf_id": r.get("src_networkconf_id"),
|
||||
"dst_networkconf_id": r.get("dst_networkconf_id"),
|
||||
} for r in raw.get("rest/firewallrule", []) if isinstance(r, dict)]
|
||||
|
||||
firewall_groups = [{
|
||||
"id": g.get("_id"), "name": g.get("name"),
|
||||
"type": g.get("group_type"), "members": g.get("group_members"),
|
||||
} for g in raw.get("rest/firewallgroup", []) if isinstance(g, dict)]
|
||||
|
||||
static_routes = [{
|
||||
"name": r.get("name"), "enabled": r.get("enabled"),
|
||||
"network": r.get("static-route_network"),
|
||||
"nexthop": r.get("static-route_nexthop"),
|
||||
"distance": r.get("static-route_distance"),
|
||||
"type": r.get("static-route_type"),
|
||||
} for r in raw.get("rest/routing", []) if isinstance(r, dict)]
|
||||
|
||||
return {
|
||||
"networks": networks,
|
||||
"reservations": reservations,
|
||||
"dynamic_clients": dynamic,
|
||||
"port_forwards": port_forwards,
|
||||
"firewall_rules": firewall_rules,
|
||||
"firewall_groups": firewall_groups,
|
||||
"static_routes": static_routes,
|
||||
"warnings": find_warnings(networks, reservations),
|
||||
}
|
||||
|
||||
|
||||
def find_warnings(networks: list, reservations: list) -> list:
|
||||
"""Things that are fine under UniFi but bite when rebuilt on VyOS."""
|
||||
warns = []
|
||||
|
||||
for n in networks:
|
||||
if not n["subnet"]:
|
||||
continue
|
||||
try:
|
||||
iface = ipaddress.ip_interface(n["subnet"])
|
||||
except ValueError:
|
||||
warns.append({"kind": "bad_subnet", "network": n["name"], "detail": n["subnet"]})
|
||||
continue
|
||||
# UniFi stores the gateway in ip_subnet. A gateway equal to the network
|
||||
# address is legal in a /23 but plenty of tooling rejects it, so it must
|
||||
# not be discovered during the cutover window.
|
||||
if iface.ip == iface.network.network_address:
|
||||
warns.append({
|
||||
"kind": "gateway_is_network_address", "network": n["name"],
|
||||
"detail": f"gateway {iface.ip} is the network address of {iface.network}",
|
||||
})
|
||||
|
||||
# Reservations that sit inside the dynamic pool. UniFi's dhcpd tolerates
|
||||
# this; whether VyOS does depends on its DHCP backend, so every one of these
|
||||
# is a config that must be proven on the sim before cutover.
|
||||
ranges = []
|
||||
for n in networks:
|
||||
if n["dhcp_enabled"] and n["dhcp_start"] and n["dhcp_stop"]:
|
||||
try:
|
||||
ranges.append((n["name"], ipaddress.ip_address(n["dhcp_start"]),
|
||||
ipaddress.ip_address(n["dhcp_stop"])))
|
||||
except ValueError:
|
||||
pass
|
||||
inside = []
|
||||
for r in reservations:
|
||||
if not r["ip"]:
|
||||
continue
|
||||
try:
|
||||
addr = ipaddress.ip_address(r["ip"])
|
||||
except ValueError:
|
||||
continue
|
||||
for name, lo, hi in ranges:
|
||||
if lo <= addr <= hi:
|
||||
inside.append(f"{r['ip']} ({r['name'] or r['mac']}) in {name} pool")
|
||||
break
|
||||
if inside:
|
||||
warns.append({"kind": "reservation_inside_dhcp_pool",
|
||||
"count": len(inside), "detail": inside})
|
||||
|
||||
unresolved = [f"{r['ip']} {r['mac']} {r['name']}"
|
||||
for r in reservations if r["resolved_by"] == "unresolved"]
|
||||
if unresolved:
|
||||
warns.append({"kind": "reservation_matches_no_subnet",
|
||||
"count": len(unresolved), "detail": unresolved})
|
||||
return warns
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--out", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), "export"))
|
||||
ap.add_argument("--summary", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
os.makedirs(args.out, exist_ok=True)
|
||||
opener, base, site = _unifi.client()
|
||||
print(f"controller {base} site {site}")
|
||||
|
||||
raw: dict = {}
|
||||
errors = []
|
||||
for path, why in ENDPOINTS.items():
|
||||
data = _unifi.get(opener, base, site, path)
|
||||
if isinstance(data, dict) and "__error__" in data:
|
||||
errors.append((path, data["__error__"]))
|
||||
print(f" {path:22} FAILED {data['__error__'][:50]}")
|
||||
continue
|
||||
raw[path] = data
|
||||
fname = path.replace("/", "_") + ".json"
|
||||
with open(os.path.join(args.out, fname), "w") as fh:
|
||||
json.dump(data, fh, indent=2, sort_keys=True)
|
||||
print(f" {path:22} {len(data):4d} -> {fname}")
|
||||
|
||||
inventory = build_inventory(raw)
|
||||
with open(os.path.join(args.out, "inventory.json"), "w") as fh:
|
||||
json.dump(inventory, fh, indent=2, sort_keys=True)
|
||||
|
||||
print(f"\nwrote {args.out}/inventory.json")
|
||||
print(f" networks {len(inventory['networks'])}")
|
||||
print(f" DHCP reservations {len(inventory['reservations'])}")
|
||||
print(f" dynamic clients {len(inventory['dynamic_clients'])} (no reservation — see summary)")
|
||||
print(f" port forwards {len(inventory['port_forwards'])}")
|
||||
print(f" firewall rules {len(inventory['firewall_rules'])}")
|
||||
print(f" static routes {len(inventory['static_routes'])}")
|
||||
if errors:
|
||||
print(f" endpoints failed {len(errors)}: {[e[0] for e in errors]}")
|
||||
|
||||
if args.summary:
|
||||
print_summary(inventory)
|
||||
return 0
|
||||
|
||||
|
||||
def print_summary(inv: dict) -> None:
|
||||
print("\n=== networks ===")
|
||||
print(f"{'name':22} {'vlan':>5} {'subnet':20} {'dhcp range':32} lease")
|
||||
for n in sorted(inv["networks"], key=lambda x: (x["vlan"] or 0)):
|
||||
rng = f"{n['dhcp_start']} - {n['dhcp_stop']}" if n["dhcp_enabled"] else "(dhcp off)"
|
||||
print(f"{(n['name'] or '')[:22]:22} {str(n['vlan'] or '-'):>5} "
|
||||
f"{(n['subnet'] or '-'):20} {rng:32} {n['dhcp_lease'] or '-'}")
|
||||
|
||||
print(f"\n=== DHCP reservations ({len(inv['reservations'])}) ===")
|
||||
for r in inv["reservations"]:
|
||||
print(f" {r['ip']:16} {r['mac']:18} {(r['network_name'] or '?')[:14]:14} {r['name'][:30]}")
|
||||
|
||||
if inv["port_forwards"]:
|
||||
print(f"\n=== port forwards ({len(inv['port_forwards'])}) ===")
|
||||
for p in inv["port_forwards"]:
|
||||
state = "" if p["enabled"] else " [DISABLED]"
|
||||
print(f" {p['proto']:6} {str(p['src']):16}:{str(p['dst_port']):11} -> "
|
||||
f"{p['fwd']}:{p['fwd_port']} {p['name']}{state}")
|
||||
|
||||
if inv["firewall_rules"]:
|
||||
print(f"\n=== firewall rules ({len(inv['firewall_rules'])}) ===")
|
||||
for r in inv["firewall_rules"]:
|
||||
state = "" if r["enabled"] else " [DISABLED]"
|
||||
print(f" {str(r['ruleset']):22} {str(r['action']):8} {r['name']}{state}")
|
||||
|
||||
if inv["warnings"]:
|
||||
print(f"\n=== {len(inv['warnings'])} things to settle before cutover ===")
|
||||
for w in inv["warnings"]:
|
||||
n = w.get("count")
|
||||
print(f" [{w['kind']}]" + (f" x{n}" if n else ""))
|
||||
det = w["detail"]
|
||||
for line in (det if isinstance(det, list) else [det])[:6]:
|
||||
print(f" {line}")
|
||||
if isinstance(det, list) and len(det) > 6:
|
||||
print(f" ... and {len(det) - 6} more (see inventory.json)")
|
||||
|
||||
n_dyn = len(inv["dynamic_clients"])
|
||||
if n_dyn:
|
||||
print(f"\n=== {n_dyn} active clients WITHOUT a reservation ===")
|
||||
print(" These hold their address only via the current lease database. A DHCP")
|
||||
print(" server swap hands them a different one — fine for phones, not fine for")
|
||||
print(" anything another host reaches by IP. Review before cutover:")
|
||||
for c in inv["dynamic_clients"][:40]:
|
||||
print(f" {c['ip']:16} {c['mac']:18} {(c['network'] or '')[:14]:14} {c['name'][:30]}")
|
||||
if n_dyn > 40:
|
||||
print(f" ... and {n_dyn - 40} more (see inventory.json)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
236
migration/unifi-reserve-all.py
Executable file
236
migration/unifi-reserve-all.py
Executable file
@@ -0,0 +1,236 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Reserve every active client at the address it already has.
|
||||
|
||||
Why this exists: kea does not inherit UniFi's lease database. At cutover it
|
||||
starts with an empty view of who holds what, so it can hand an address that is
|
||||
currently in use to a different device. Reservations are what carry "this
|
||||
device has this address" across the switch, because they live in config rather
|
||||
than in lease state.
|
||||
|
||||
Dry run by default -- this writes to the live controller, and 40-odd writes is
|
||||
not something to trigger by accident.
|
||||
|
||||
./unifi-reserve-all.py # show the plan, change nothing
|
||||
./unifi-reserve-all.py --apply # write them
|
||||
./unifi-reserve-all.py --skip-random # omit randomised/private MACs
|
||||
|
||||
Only clients on networks that actually run DHCP are considered, which
|
||||
automatically excludes WAN transit VLANs where a reservation is meaningless.
|
||||
Anything already reserved is left alone, and an address already reserved to a
|
||||
different MAC is reported and skipped rather than stolen.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ipaddress
|
||||
import sys
|
||||
|
||||
import _unifi
|
||||
|
||||
# The VRRP virtual addresses, read from `show configuration commands` on
|
||||
# vyos001. UniFi sees these as ordinary client addresses because the firewalls'
|
||||
# bond MACs answer for them, and their reported IP flips between the real
|
||||
# interface address and the VIP. Reserving one would put a DHCP reservation on
|
||||
# the gateway address itself.
|
||||
VIPS = {
|
||||
"192.168.1.254", "192.168.9.254", "192.168.3.254",
|
||||
"10.0.9.254", "10.0.1.254", "192.168.2.254",
|
||||
}
|
||||
|
||||
# Every MAC the two firewalls own (bond0/eth0/eth1 share one, eth2 and eth3
|
||||
# have their own). These interfaces are statically configured routers, not DHCP
|
||||
# clients -- except eth2, which is deliberately reserved and already handled.
|
||||
ROUTER_MACS = {
|
||||
"64:62:66:25:96:45", "64:62:66:25:96:46", "64:62:66:25:96:48", # vyos001
|
||||
"64:62:66:25:96:51", "64:62:66:25:96:52", "64:62:66:25:96:54", # vyos002
|
||||
}
|
||||
|
||||
|
||||
def is_random_mac(mac: str) -> bool:
|
||||
"""Locally-administered bit set => a privacy/randomised MAC.
|
||||
|
||||
Worth calling out: such a device re-randomises periodically, so the
|
||||
reservation stops matching it and becomes dead config. Harmless, but it
|
||||
will never do what it looks like it does.
|
||||
"""
|
||||
try:
|
||||
return bool(int(mac.split(":")[0], 16) & 0x02)
|
||||
except (ValueError, IndexError):
|
||||
return False
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--apply", action="store_true", help="actually write (default: dry run)")
|
||||
ap.add_argument("--skip-random", action="store_true", help="omit randomised MACs")
|
||||
ap.add_argument("--plan", default="reservation-plan.tsv",
|
||||
help="dry run WRITES this file; --apply READS it and applies "
|
||||
"exactly what it contains")
|
||||
args = ap.parse_args()
|
||||
|
||||
opener, base, site = _unifi.client()
|
||||
users = _unifi.get(opener, base, site, "rest/user")
|
||||
nets = _unifi.get(opener, base, site, "rest/networkconf")
|
||||
sta = _unifi.get(opener, base, site, "stat/sta")
|
||||
for blob in (users, nets, sta):
|
||||
if isinstance(blob, dict):
|
||||
print(f"error reading controller: {blob['__error__']}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# Only networks that serve DHCP: a reservation on a WAN transit VLAN means
|
||||
# nothing, and those are exactly the ones without dhcpd_enabled.
|
||||
serving = []
|
||||
for n in nets:
|
||||
if not (n.get("dhcpd_enabled") and n.get("ip_subnet")):
|
||||
continue
|
||||
try:
|
||||
serving.append((ipaddress.ip_interface(n["ip_subnet"]).network, n))
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
by_mac = {(u.get("mac") or "").lower(): u for u in users}
|
||||
taken = {u.get("fixed_ip"): (u.get("mac") or "").lower()
|
||||
for u in users if u.get("use_fixedip")}
|
||||
|
||||
plan, skipped = [], []
|
||||
for c in sta:
|
||||
mac, ip = (c.get("mac") or "").lower(), c.get("ip")
|
||||
if not mac or not ip:
|
||||
continue
|
||||
label = c.get("name") or c.get("hostname") or "?"
|
||||
user = by_mac.get(mac)
|
||||
|
||||
if ip in VIPS:
|
||||
skipped.append((ip, mac, label, "VRRP virtual address - not a client"))
|
||||
continue
|
||||
if mac in ROUTER_MACS:
|
||||
skipped.append((ip, mac, label, "firewall's own interface - statically configured"))
|
||||
continue
|
||||
|
||||
net = next((n for netw, n in serving if ipaddress.ip_address(ip) in netw), None)
|
||||
if net is None:
|
||||
skipped.append((ip, mac, label, "not on a DHCP-serving network"))
|
||||
continue
|
||||
# The gateway is the router, not a lease.
|
||||
if ip == str(ipaddress.ip_interface(net["ip_subnet"]).ip):
|
||||
skipped.append((ip, mac, label, "network gateway address"))
|
||||
continue
|
||||
if user is None:
|
||||
skipped.append((ip, mac, label, "not a known client on the controller"))
|
||||
continue
|
||||
if user.get("use_fixedip"):
|
||||
if user.get("fixed_ip") != ip:
|
||||
skipped.append((ip, mac, label,
|
||||
f"already reserved at {user.get('fixed_ip')} - left alone"))
|
||||
continue
|
||||
if ip in taken and taken[ip] != mac:
|
||||
skipped.append((ip, mac, label,
|
||||
f"address already reserved to {taken[ip]}"))
|
||||
continue
|
||||
if args.skip_random and is_random_mac(mac):
|
||||
skipped.append((ip, mac, label, "randomised MAC (--skip-random)"))
|
||||
continue
|
||||
plan.append((ip, mac, label, user, net))
|
||||
|
||||
# Generic safety net: if two MACs report the same current address, at most
|
||||
# one of them can legitimately keep it and we cannot tell which. Drop both
|
||||
# and say so -- this is exactly how the VRRP VIPs first showed up.
|
||||
counts: dict[str, int] = {}
|
||||
for ip, *_ in plan:
|
||||
counts[ip] = counts.get(ip, 0) + 1
|
||||
contested = {ip for ip, n in counts.items() if n > 1}
|
||||
if contested:
|
||||
for ip, mac, label, _u, _n in [p for p in plan if p[0] in contested]:
|
||||
skipped.append((ip, mac, label, "address claimed by more than one MAC"))
|
||||
plan = [p for p in plan if p[0] not in contested]
|
||||
|
||||
plan.sort(key=lambda r: ipaddress.ip_address(r[0]))
|
||||
|
||||
print(f"=== plan: {len(plan)} new reservation(s) ===")
|
||||
for ip, mac, label, _u, net in plan:
|
||||
flag = " [randomised MAC]" if is_random_mac(mac) else ""
|
||||
print(f" {ip:16} {mac:18} {label[:28]:28} {net.get('name')}{flag}")
|
||||
if skipped:
|
||||
print(f"\n=== skipped ({len(skipped)}) ===")
|
||||
for ip, mac, label, why in sorted(skipped):
|
||||
print(f" {ip:16} {mac:18} {label[:24]:24} {why}")
|
||||
|
||||
n_rand = sum(1 for p in plan if is_random_mac(p[1]))
|
||||
if n_rand:
|
||||
print(f"\nnote: {n_rand} of these use randomised MACs. The reservation "
|
||||
f"stops matching once the device re-randomises.")
|
||||
|
||||
if not args.apply:
|
||||
with open(args.plan, "w") as fh:
|
||||
for ip, mac, label, _u, _n in plan:
|
||||
fh.write(f"{mac}\t{ip}\t{label}\n")
|
||||
print(f"\ndry run -- nothing written to the controller.")
|
||||
print(f"plan saved to {args.plan}; re-run with --apply to apply exactly that.")
|
||||
return 0
|
||||
|
||||
# Apply the plan that was REVIEWED, not one recomputed now.
|
||||
#
|
||||
# This cost a k8s node an outage. The apply used to re-read stat/sta, and a
|
||||
# client that renewed between the dry run and the apply got pinned to
|
||||
# whatever transient address it happened to hold at that instant -- worker1
|
||||
# was reviewed at .13 and written as .242. A plan you looked at and a plan
|
||||
# that gets applied must be the same object.
|
||||
try:
|
||||
with open(args.plan) as fh:
|
||||
reviewed = {}
|
||||
for line in fh:
|
||||
parts = line.rstrip("\n").split("\t")
|
||||
if len(parts) >= 2:
|
||||
reviewed[parts[0].lower()] = parts[1]
|
||||
except OSError:
|
||||
print(f"no plan at {args.plan}. Run without --apply first and review it.",
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
|
||||
drifted = [(ip, mac) for ip, mac, _l, _u, _n in plan
|
||||
if mac in reviewed and reviewed[mac] != ip]
|
||||
for ip, mac in drifted:
|
||||
print(f" note: {mac} now reports {ip}, plan says {reviewed[mac]} -- "
|
||||
f"applying the plan", file=sys.stderr)
|
||||
|
||||
plan = [(reviewed[mac], mac, label, user, net)
|
||||
for ip, mac, label, user, net in plan if mac in reviewed]
|
||||
print(f"applying {len(plan)} reservation(s) from {args.plan}")
|
||||
|
||||
print()
|
||||
ok = fail = 0
|
||||
for ip, mac, label, user, net in plan:
|
||||
res = _unifi.put(opener, base, site, f"rest/user/{user['_id']}",
|
||||
{"use_fixedip": True, "fixed_ip": ip, "network_id": net["_id"]})
|
||||
if isinstance(res, dict):
|
||||
print(f" FAILED {ip:16} {mac} {res['__error__'][:70]}")
|
||||
fail += 1
|
||||
else:
|
||||
ok += 1
|
||||
|
||||
# Read back rather than trusting the write responses.
|
||||
after = _unifi.get(opener, base, site, "rest/user")
|
||||
live = {(u.get("mac") or "").lower() for u in after if u.get("use_fixedip")}
|
||||
verified = sum(1 for _ip, mac, _l, _u, _n in plan if mac in live)
|
||||
print(f"\nwrote {ok}, failed {fail}, verified live {verified}/{len(plan)}")
|
||||
print(f"total reservations on the controller now: "
|
||||
f"{sum(1 for u in after if u.get('use_fixedip'))}")
|
||||
|
||||
# Writing the controller is only half the job. The gateway applies config
|
||||
# on its own schedule, and a device running config from before these
|
||||
# changes will hand out addresses that disagree with what the controller
|
||||
# shows -- which is how a k8s node ended up unable to get any lease at all
|
||||
# while the controller looked perfectly correct.
|
||||
print("\nThe controller now disagrees with what the gateway is running.")
|
||||
print("Push it to the device and wait for state to return to 'connected':")
|
||||
print(" python3 -c \"import _unifi; o,b,s=_unifi.client(); "
|
||||
"print(_unifi.post(o,b,s,'cmd/devmgr',"
|
||||
"{'cmd':'force-provision','mac':'<gateway-mac>'}))\"")
|
||||
print("Then verify a real DISCOVER is answered before trusting it:")
|
||||
print(" ssh vyos@<fw> 'sudo nmap --script broadcast-dhcp-discover -e eth2 "
|
||||
"--script-args broadcast-dhcp-discover.mac=<client-mac>'")
|
||||
return 0 if fail == 0 and verified == len(plan) else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
106
migration/unifi-reserve.py
Executable file
106
migration/unifi-reserve.py
Executable file
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Set a fixed-IP reservation in UniFi, so it cannot lease that address away.
|
||||
|
||||
Written for the firewalls' management NICs: their addresses are pinned static
|
||||
on the VyOS side, but UniFi still owns the pool they sit in and would happily
|
||||
hand the same address to something else. A reservation closes that gap while
|
||||
the USG is still the DHCP server, and it keeps the management address identical
|
||||
in both cutover modes.
|
||||
|
||||
./unifi-reserve.py 64:62:66:25:96:47 192.168.8.143
|
||||
./unifi-reserve.py --dry-run <mac> <ip>
|
||||
|
||||
Idempotent: an existing, matching reservation is reported and left alone. This
|
||||
writes to the live controller, so it verifies by reading the record back rather
|
||||
than trusting the response.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ipaddress
|
||||
import sys
|
||||
|
||||
import _unifi
|
||||
|
||||
|
||||
def find_network(nets: list, ip: str) -> dict | None:
|
||||
"""Which configured network contains this address?
|
||||
|
||||
ip_subnet holds the gateway address with a prefix ("192.168.8.1/23"), so
|
||||
the network has to be derived from it rather than compared directly.
|
||||
"""
|
||||
addr = ipaddress.ip_address(ip)
|
||||
for n in nets:
|
||||
raw = n.get("ip_subnet")
|
||||
if not raw:
|
||||
continue
|
||||
try:
|
||||
if addr in ipaddress.ip_interface(raw).network:
|
||||
return n
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("mac")
|
||||
ap.add_argument("ip")
|
||||
ap.add_argument("--dry-run", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
mac = args.mac.lower().replace("-", ":")
|
||||
ipaddress.ip_address(args.ip) # fail early on a typo
|
||||
|
||||
opener, base, site = _unifi.client()
|
||||
users = _unifi.get(opener, base, site, "rest/user")
|
||||
nets = _unifi.get(opener, base, site, "rest/networkconf")
|
||||
for blob in (users, nets):
|
||||
if isinstance(blob, dict):
|
||||
print(f"error reading controller: {blob['__error__']}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
user = next((u for u in users if (u.get("mac") or "").lower() == mac), None)
|
||||
if user is None:
|
||||
print(f"{mac} is not a known client -- connect it once, or create it "
|
||||
f"in the UI first", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
net = find_network(nets, args.ip)
|
||||
if net is None:
|
||||
print(f"no configured network contains {args.ip}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
label = user.get("name") or user.get("hostname") or mac
|
||||
if user.get("use_fixedip") and user.get("fixed_ip") == args.ip:
|
||||
print(f"{label} ({mac}) already reserved at {args.ip} -- nothing to do")
|
||||
return 0
|
||||
if user.get("use_fixedip"):
|
||||
print(f"WARNING: {label} currently reserved at {user.get('fixed_ip')}, "
|
||||
f"changing to {args.ip}", file=sys.stderr)
|
||||
|
||||
print(f"{label} ({mac}) -> {args.ip} on '{net.get('name')}' (VLAN {net.get('vlan') or 'native'})")
|
||||
if args.dry_run:
|
||||
print(" --dry-run: not writing")
|
||||
return 0
|
||||
|
||||
payload = {"use_fixedip": True, "fixed_ip": args.ip, "network_id": net["_id"]}
|
||||
res = _unifi.put(opener, base, site, f"rest/user/{user['_id']}", payload)
|
||||
if isinstance(res, dict):
|
||||
print(f" write failed: {res['__error__']}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# Read it back: the controller accepting a PUT is not proof it stored what
|
||||
# we asked for.
|
||||
after = _unifi.get(opener, base, site, "rest/user")
|
||||
check = next((u for u in after if (u.get("mac") or "").lower() == mac), {})
|
||||
if check.get("use_fixedip") and check.get("fixed_ip") == args.ip:
|
||||
print(f" verified: reservation is live")
|
||||
return 0
|
||||
print(f" VERIFY FAILED: controller reports use_fixedip="
|
||||
f"{check.get('use_fixedip')} fixed_ip={check.get('fixed_ip')}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
251
migration/unifi-to-vyos.py
Executable file
251
migration/unifi-to-vyos.py
Executable file
@@ -0,0 +1,251 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Turn the UniFi export into the VyOS config that replaces it.
|
||||
|
||||
Scope is deliberately narrow: DHCP and DNS. Those are the services the USG owns
|
||||
that VyOS must reproduce byte-for-byte in behaviour, because getting them wrong
|
||||
means clients lose their addresses or their name resolution. Everything else
|
||||
either stays on UniFi (wireless), has no VyOS equivalent (user groups), or is
|
||||
hand-written because it is not in the export (WAN, NAT, VRRP).
|
||||
|
||||
This is not a general UniFi-to-VyOS converter and should not grow into one.
|
||||
|
||||
./unifi-to-vyos.py --mode prod # the cutover artifact
|
||||
./unifi-to-vyos.py --mode sim # same MACs, labsim addresses
|
||||
./unifi-to-vyos.py --mode prod --check # counts only, no output
|
||||
|
||||
Both modes come from one code path on purpose: the config proven in labsim and
|
||||
the config applied to the firewalls must not be able to drift apart.
|
||||
|
||||
DNS note: UniFi hands out the gateway's own IP as resolver whenever a network
|
||||
has no explicit dhcpd_dns -- true for 5 of the 6 VLANs, verified by labmaster
|
||||
resolving against 192.168.8.1. So VyOS must run `service dns forwarding` or
|
||||
those VLANs lose DNS entirely at cutover. LoT's explicit 10.0.0.194 is preserved
|
||||
as-is.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
# Upstream resolvers for VyOS's own forwarder -- the same pair the USG used on
|
||||
# its WAN (wan_dns1/wan_dns2). The NAS at 10.0.0.194 is deliberately NOT here:
|
||||
# it is legacy for ad.itaz.eu, and those records now live in Cloudflare, so the
|
||||
# zone resolves publicly (verified: nas001.ad.itaz.eu and kvm-macstudio1 both
|
||||
# answer from 8.8.8.8). That means no conditional forward is needed and the NAS
|
||||
# is out of the DNS path entirely.
|
||||
UPSTREAM_DNS = ["8.8.8.8", "8.8.4.4"]
|
||||
|
||||
# labsim equivalents, keyed by VLAN id. Only VLAN 10 needs a /23: every one of
|
||||
# the 31 reservations is in LoT, which spans 10.0.0.x and 10.0.1.x, and a /24
|
||||
# cannot represent that. k8s and Private are also /23 in production but hold no
|
||||
# reservations, so they keep their existing /24 and their DHCP range is clamped
|
||||
# (reported at generation time -- never silently).
|
||||
SIM_SUBNETS = {
|
||||
1: "172.31.1.0/24",
|
||||
2: "172.31.2.0/24",
|
||||
3: "172.31.3.0/24",
|
||||
9: "172.31.9.0/24",
|
||||
10: "172.31.10.0/23",
|
||||
200: "172.31.200.0/24",
|
||||
}
|
||||
|
||||
|
||||
def vlan_of(net: dict) -> int:
|
||||
"""VLAN id, treating the untagged Management network as 1.
|
||||
|
||||
UniFi stores vlan=None for the native network; the VyOS side already uses
|
||||
vrid 1 for it (high-availability group 'native'), so 1 is the consistent id.
|
||||
"""
|
||||
return int(net["vlan"]) if net.get("vlan") else 1
|
||||
|
||||
|
||||
def sanitize(name: str, fallback: str) -> str:
|
||||
"""Reduce a UniFi client name to something VyOS will accept as a node name.
|
||||
|
||||
VyOS validates static-mapping names as *hostnames*, so underscores are
|
||||
rejected outright -- verified: `Dongle-M_C0D4` fails with "Invalid static
|
||||
mapping hostname". Letters, digits and hyphens only, no leading digit or
|
||||
hyphen, no trailing hyphen.
|
||||
"""
|
||||
cleaned = re.sub(r"[^A-Za-z0-9-]", "-", (name or "").strip())
|
||||
cleaned = re.sub(r"-{2,}", "-", cleaned).strip("-")
|
||||
if cleaned and cleaned[0].isdigit():
|
||||
cleaned = "h" + cleaned
|
||||
return cleaned or fallback
|
||||
|
||||
|
||||
class Mapper:
|
||||
"""Translates production addresses into the target mode's address space.
|
||||
|
||||
In prod mode this is the identity. In sim mode an address is mapped by its
|
||||
offset from the network address, so the host part is preserved: 10.0.0.46
|
||||
-> 172.31.10.46 and 10.0.1.67 -> 172.31.11.67. That is what makes the sim
|
||||
test meaningful -- the MAC is identical and the host octet is recognisable.
|
||||
"""
|
||||
|
||||
def __init__(self, mode: str, networks: list) -> None:
|
||||
self.mode = mode
|
||||
self.clamped: list[str] = []
|
||||
self.map: dict[int, tuple] = {}
|
||||
for n in networks:
|
||||
prod = ipaddress.ip_network(
|
||||
ipaddress.ip_interface(n["subnet"]).network)
|
||||
if mode == "sim":
|
||||
sim = ipaddress.ip_network(SIM_SUBNETS[vlan_of(n)])
|
||||
else:
|
||||
sim = prod
|
||||
self.map[vlan_of(n)] = (prod, sim)
|
||||
|
||||
def net(self, vlan: int) -> ipaddress.IPv4Network:
|
||||
return self.map[vlan][1]
|
||||
|
||||
def addr(self, vlan: int, ip: str, what: str) -> str | None:
|
||||
"""Map one address, or None if it does not fit the target subnet."""
|
||||
prod, sim = self.map[vlan]
|
||||
offset = int(ipaddress.ip_address(ip)) - int(prod.network_address)
|
||||
if offset < 0 or offset >= sim.num_addresses:
|
||||
self.clamped.append(f"{what}: {ip} does not fit {sim}")
|
||||
return None
|
||||
return str(ipaddress.ip_address(int(sim.network_address) + offset))
|
||||
|
||||
def gateway(self, vlan: int, net: dict) -> str:
|
||||
"""The address clients are told to use as their default route.
|
||||
|
||||
Production: the USG's current address (VIPs move .254 -> .1 at cutover),
|
||||
so no client has to change anything. Sim: the sim router at .1.
|
||||
"""
|
||||
if self.mode == "sim":
|
||||
return str(self.net(vlan).network_address + 1)
|
||||
return str(ipaddress.ip_interface(net["subnet"]).ip)
|
||||
|
||||
|
||||
def build(inv: dict, mode: str) -> tuple[list[str], dict]:
|
||||
nets = [n for n in inv["networks"] if n["dhcp_enabled"] and n["subnet"]]
|
||||
nets.sort(key=vlan_of)
|
||||
m = Mapper(mode, nets)
|
||||
|
||||
out: list[str] = []
|
||||
used_tags: set[str] = set()
|
||||
stats = {"subnets": 0, "mappings": 0, "dropped": []}
|
||||
by_vlan: dict[int, list] = {}
|
||||
for r in inv["reservations"]:
|
||||
if r["network_vlan"] is None and r["network_name"] != "Management":
|
||||
# resolved_by == "unresolved"; cannot place it without a subnet
|
||||
stats["dropped"].append(f"{r['ip']} {r['mac']} (no network)")
|
||||
continue
|
||||
by_vlan.setdefault(r["network_vlan"] or 1, []).append(r)
|
||||
|
||||
out.append("# --- DHCP ---------------------------------------------------")
|
||||
for n in nets:
|
||||
vlan = vlan_of(n)
|
||||
sub = m.net(vlan)
|
||||
base = f"set service dhcp-server shared-network-name {sanitize(n['name'], f'vlan{vlan}')} subnet {sub}"
|
||||
gw = m.gateway(vlan, n)
|
||||
|
||||
out.append("")
|
||||
out.append(f"# {n['name']} (VLAN {vlan}) <- {n['subnet']}")
|
||||
# subnet-id is required by kea and must be stable across regenerations;
|
||||
# the VLAN id is already the unique per-network number in this lab.
|
||||
out.append(f"{base} subnet-id {vlan}")
|
||||
out.append(f"{base} option default-router {gw}")
|
||||
|
||||
# Every VLAN is handed the gateway as its resolver, so all lookups go
|
||||
# through VyOS and out to the upstreams above. UniFi set an explicit
|
||||
# resolver on LoT only (the NAS); that is deliberately not carried over
|
||||
# -- the NAS is legacy and pointing clients at it would keep it in the
|
||||
# path for one VLAN and not the others.
|
||||
out.append(f"{base} option name-server {gw}")
|
||||
|
||||
if n["domain_name"]:
|
||||
out.append(f"{base} option domain-name '{n['domain_name']}'")
|
||||
if n["dhcp_lease"]:
|
||||
out.append(f"{base} lease {n['dhcp_lease']}")
|
||||
|
||||
start = m.addr(vlan, n["dhcp_start"], f"{n['name']} range start")
|
||||
stop = m.addr(vlan, n["dhcp_stop"], f"{n['name']} range stop")
|
||||
if start is None:
|
||||
start = str(sub.network_address + 11)
|
||||
if stop is None:
|
||||
# Clamp to the last usable address rather than dropping the pool.
|
||||
stop = str(sub.broadcast_address - 1)
|
||||
out.append(f"{base} range LAN start {start}")
|
||||
out.append(f"{base} range LAN stop {stop}")
|
||||
stats["subnets"] += 1
|
||||
|
||||
for r in sorted(by_vlan.get(vlan, []), key=lambda x: ipaddress.ip_address(x["ip"])):
|
||||
ip = m.addr(vlan, r["ip"], f"reservation {r['name']}")
|
||||
if ip is None:
|
||||
stats["dropped"].append(f"{r['ip']} {r['mac']} ({r['name']})")
|
||||
continue
|
||||
tag = sanitize(r["name"] or r["hostname"], "host-" + r["mac"].replace(":", ""))
|
||||
# Distinct clients can sanitize to the same name; a collision would
|
||||
# silently overwrite one reservation with another's address.
|
||||
if tag in used_tags:
|
||||
tag = f"{tag}-{r['mac'].replace(':', '')[-4:]}"
|
||||
used_tags.add(tag)
|
||||
out.append(f"{base} static-mapping {tag} mac {r['mac']}")
|
||||
out.append(f"{base} static-mapping {tag} ip-address {ip}")
|
||||
stats["mappings"] += 1
|
||||
|
||||
out.append("")
|
||||
out.append("# --- DNS ----------------------------------------------------")
|
||||
out.append("# The USG resolves for 5 of 6 VLANs today (it hands out its own")
|
||||
out.append("# address when dhcpd_dns is empty). Without this, they lose DNS.")
|
||||
for n in nets:
|
||||
vlan = vlan_of(n)
|
||||
out.append(f"set service dns forwarding listen-address {m.gateway(vlan, n)}")
|
||||
out.append(f"set service dns forwarding allow-from {m.net(vlan)}")
|
||||
for ns in UPSTREAM_DNS:
|
||||
out.append(f"set service dns forwarding name-server {ns}")
|
||||
out.append("set service dns forwarding cache-size 10000")
|
||||
|
||||
stats["clamped"] = m.clamped
|
||||
return out, stats
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
here = os.path.dirname(os.path.abspath(__file__))
|
||||
ap.add_argument("--mode", choices=("prod", "sim"), required=True)
|
||||
ap.add_argument("--inventory", default=os.path.join(here, "export", "inventory.json"))
|
||||
ap.add_argument("-o", "--out")
|
||||
ap.add_argument("--check", action="store_true", help="counts only, no config")
|
||||
args = ap.parse_args()
|
||||
|
||||
with open(args.inventory) as fh:
|
||||
inv = json.load(fh)
|
||||
|
||||
lines, stats = build(inv, args.mode)
|
||||
|
||||
expected = len(inv["reservations"])
|
||||
print(f"mode={args.mode} subnets={stats['subnets']} "
|
||||
f"static-mappings={stats['mappings']}/{expected}", file=sys.stderr)
|
||||
for c in stats["clamped"]:
|
||||
print(f" clamped: {c}", file=sys.stderr)
|
||||
for d in stats["dropped"]:
|
||||
print(f" DROPPED: {d}", file=sys.stderr)
|
||||
|
||||
if stats["mappings"] != expected and args.mode == "prod":
|
||||
print(f"ERROR: {expected - stats['mappings']} reservation(s) missing from "
|
||||
f"prod output -- every one must survive the cutover", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if args.check:
|
||||
return 0
|
||||
|
||||
text = "\n".join(lines) + "\n"
|
||||
if args.out:
|
||||
with open(args.out, "w") as fh:
|
||||
fh.write(text)
|
||||
print(f"wrote {args.out}", file=sys.stderr)
|
||||
else:
|
||||
sys.stdout.write(text)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
491
migration/vyos-mode-delta.py
Executable file
491
migration/vyos-mode-delta.py
Executable file
@@ -0,0 +1,491 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate the delta that turns a passive VyOS pair into the gateway.
|
||||
|
||||
The switch works as: load the known-good `unifi.boot` snapshot, apply this
|
||||
delta, commit-confirm. Deriving the gateway mode from base+delta every time
|
||||
means there is no inverse to maintain and no drift between two hand-kept
|
||||
configs -- the revert is just loading the snapshot again.
|
||||
|
||||
./vyos-mode-delta.py --priority 200 -o to-vyos.commands # vyos001 (master)
|
||||
./vyos-mode-delta.py --priority 100 -o to-vyos.commands # vyos002 (backup)
|
||||
./vyos-mode-delta.py --emit-secrets /path/wan-secrets # credentials, 0600
|
||||
|
||||
The PPPoE password is NOT written into the delta. The delta carries the
|
||||
placeholder @@WAN_PASSWORD@@ and the switch script substitutes it at apply time
|
||||
from /config/wan-secrets, so the generated artifact can be read, diffed and
|
||||
copied around without carrying a credential.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import importlib.util
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
# unifi-to-vyos.py has hyphens, so it cannot be imported by name. Reuse it
|
||||
# rather than duplicating the DHCP/DNS generation -- the whole point is that
|
||||
# what labsim proved and what production gets come from one code path.
|
||||
_spec = importlib.util.spec_from_file_location(
|
||||
"unifi_to_vyos", os.path.join(HERE, "unifi-to-vyos.py"))
|
||||
unifi_to_vyos = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(unifi_to_vyos)
|
||||
|
||||
# Two WANs, established by reading the live USG rather than the UniFi fields
|
||||
# (which report wan_type=dhcp for both and are simply wrong):
|
||||
#
|
||||
# WAN1 Vodafone, PPPoE on the USG's eth0, ~900/700 Mbit. Verified working:
|
||||
# pppoe0 came up with 90.241.226.213 peer 84.65.128.1, MTU 1492.
|
||||
# WAN2 10 gig ISP, plain DHCP on the USG's eth2, public 87.192.101.48/21
|
||||
# gw 87.192.96.1. This is what carries traffic today.
|
||||
#
|
||||
# Both reach the USG as untagged access ports but are carried across the switch
|
||||
# fabric as vlan-only networks 51 and 53, so VyOS picks them up as bond vifs.
|
||||
WAN_PPPOE_VIF = "bond0.51" # Vodafone
|
||||
WAN_PPPOE_IF = "pppoe0"
|
||||
WAN_DHCP_VIF = "bond0.53" # 10 gig ISP
|
||||
|
||||
# The DHCP lease is bound to the MAC, so cloning the USG's WAN2 MAC is how VyOS
|
||||
# keeps 87.192.101.48 instead of negotiating a fresh lease -- or getting none,
|
||||
# if the ISP hands out one per line. Only ONE box may carry this at a time.
|
||||
WAN_DHCP_MAC = "f0:9f:c2:12:9b:4f"
|
||||
|
||||
# Route distances: the 10 gig line wins, Vodafone is failover.
|
||||
#
|
||||
# The live 10 gig default route is owned by `protocols failover`, so that losing
|
||||
# the ISP *without* losing carrier withdraws it instead of black-holing every
|
||||
# packet -- a DHCP-installed route never withdraws on a dead upstream.
|
||||
#
|
||||
# The vif still needs default-route-distance rather than no-default-route:
|
||||
# vyos-failover resolves a dhcp-interface gateway by reading new_routers out of
|
||||
# /run/dhclient/dhclient_<if>.lease, and no-default-route leaves that field
|
||||
# EMPTY, so the daemon finds no next hop and installs nothing. Verified on
|
||||
# vyos001: with no-default-route the default route fell through to Vodafone.
|
||||
#
|
||||
# So DHCP keeps a route, deliberately demoted BELOW Vodafone. Order of
|
||||
# preference: failover's kernel route (distance 0) > pppoe (10) > DHCP (210).
|
||||
# The demoted route is never selected while pppoe is up, so it cannot re-create
|
||||
# the black-hole it exists to avoid.
|
||||
DIST_PPPOE = 10
|
||||
DIST_DHCP_FALLBACK = 210
|
||||
|
||||
# Health-checked primary. Two targets, any-available, so one resolver having a
|
||||
# bad day is not read as "the line is down". Verified on the sim: failover and
|
||||
# failback both inside 5s with the router's own interface still UP.
|
||||
FAILOVER_METRIC = 1
|
||||
FAILOVER_TARGETS = ["8.8.8.8", "1.1.1.1"]
|
||||
FAILOVER_TIMEOUT = 5
|
||||
|
||||
PLACEHOLDER = "@@WAN_PASSWORD@@"
|
||||
|
||||
# Per-VLAN interface addresses of each node, read from the live boxes. VRRP
|
||||
# unicast (hello-source-address/peer-address) needs both ends explicitly, and
|
||||
# these are NOT derivable from the subnet -- VLAN 3 is .4/.5 while everything
|
||||
# else is .252/.253.
|
||||
# vlan: (vyos001, vyos002)
|
||||
NODE_ADDRS = {
|
||||
1: ("192.168.1.252", "192.168.1.253"),
|
||||
2: ("192.168.9.252", "192.168.9.253"),
|
||||
3: ("192.168.3.4", "192.168.3.5"),
|
||||
9: ("10.8.0.252", "10.8.0.253"),
|
||||
10: ("10.0.1.252", "10.0.1.253"),
|
||||
200: ("192.168.2.252", "192.168.2.253"),
|
||||
}
|
||||
|
||||
# Dedicated point-to-point link for conntrack state sync (eth3 <-> eth3).
|
||||
CONNTRACK_ADDRS = ("10.255.255.1/30", "10.255.255.2/30")
|
||||
CONNTRACK_IF = "eth3"
|
||||
|
||||
# kea HA talks over TCP 647. The LoT addresses are used because they are stable
|
||||
# and reachable today without the conntrack cable being plugged in.
|
||||
DHCP_HA_NAME = "vyos-dhcp-pair" # must NOT equal either system host-name
|
||||
|
||||
|
||||
|
||||
def vrrp_group(vlan: int) -> str:
|
||||
"""VRRP group names as configured on the boxes: 'native' for the untagged
|
||||
VLAN, 'vlan<id>' otherwise."""
|
||||
return "native" if vlan == 1 else f"vlan{vlan}"
|
||||
|
||||
|
||||
def build_delta(inv: dict, priority: int, wan_user: str, with_wan: bool,
|
||||
conntrack_link: bool) -> list[str]:
|
||||
out: list[str] = []
|
||||
primary = priority >= 200 # vyos001 is the master/primary
|
||||
self_i, peer_i = (0, 1) if primary else (1, 0)
|
||||
nets = [n for n in inv["networks"] if n["dhcp_enabled"] and n["subnet"]]
|
||||
nets.sort(key=unifi_to_vyos.vlan_of)
|
||||
|
||||
out += [
|
||||
"# ==========================================================",
|
||||
"# Delta: passive VyOS pair -> gateway. Applied on top of a",
|
||||
"# freshly loaded unifi.boot, never on top of itself.",
|
||||
"# ==========================================================",
|
||||
"",
|
||||
"# An unconfirmed commit must reload the previous config, NOT reboot.",
|
||||
"# 'reboot' is the VyOS default and would turn a failed switch into a",
|
||||
"# real outage on the box that is meant to be carrying the network.",
|
||||
"set system config-management commit-confirm action reload",
|
||||
"",
|
||||
"# --- gateway addresses ------------------------------------",
|
||||
"# The VIP takes over the address the USG holds today, so no client",
|
||||
"# changes anything: no renewal needed, hardcoded gateways keep working.",
|
||||
]
|
||||
for n in nets:
|
||||
vlan = unifi_to_vyos.vlan_of(n)
|
||||
grp = vrrp_group(vlan)
|
||||
iface = ipaddress.ip_interface(n["subnet"])
|
||||
out.append(f"# {n['name']} (VLAN {vlan}) -> {iface.with_prefixlen}")
|
||||
# Delete the whole address node rather than a computed old value.
|
||||
# `address` is multi-value, and the current VIPs are NOT at
|
||||
# network+254 on the /23 networks -- they are 192.168.9.254,
|
||||
# 10.0.9.254 and 10.0.1.254, in the upper half. A delete naming the
|
||||
# wrong address fails quietly and leaves the group holding two VIPs.
|
||||
out.append(f"delete high-availability vrrp group {grp} address")
|
||||
out.append(f"set high-availability vrrp group {grp} address {iface.with_prefixlen}")
|
||||
out.append(f"set high-availability vrrp group {grp} priority {priority}")
|
||||
own, peer = NODE_ADDRS[vlan] if primary else NODE_ADDRS[vlan][::-1]
|
||||
# Unicast VRRP: the walkthrough sets both ends explicitly rather than
|
||||
# relying on multicast, which is more predictable across a switch fabric.
|
||||
out.append(f"set high-availability vrrp group {grp} hello-source-address {own}")
|
||||
out.append(f"set high-availability vrrp group {grp} peer-address {peer}")
|
||||
# Without no-preempt a recovered box reclaims the VIP immediately --
|
||||
# before conntrack state has synced -- and drops every established
|
||||
# connection. If preemption is ever wanted, preempt-delay must be >=
|
||||
# the conntrack-sync purge-timeout.
|
||||
out.append(f"set high-availability vrrp group {grp} no-preempt")
|
||||
|
||||
out += [
|
||||
"",
|
||||
]
|
||||
|
||||
|
||||
out += [
|
||||
"",
|
||||
"# --- stateful tracking (BOTH boxes) ------------------------",
|
||||
"# VyOS only engages conntrack when a firewall or NAT exists. The",
|
||||
"# backup has no WAN and therefore no NAT, so without this rule it",
|
||||
"# tracks nothing -- and conntrack-sync entries replicated to a box",
|
||||
"# whose conntrack is not engaged cannot be used when it takes over.",
|
||||
"# Verified in labsim: zero conntrack entries until a state-matching",
|
||||
"# rule was present, then replication began immediately.",
|
||||
"set firewall ipv4 forward filter default-action accept",
|
||||
"set firewall ipv4 forward filter rule 10 action accept",
|
||||
"set firewall ipv4 forward filter rule 10 state established",
|
||||
"set firewall ipv4 forward filter rule 10 state related",
|
||||
"set firewall ipv4 forward filter rule 10 description 'stateful tracking'",
|
||||
]
|
||||
|
||||
if True: # WAN config on BOTH boxes; see the disable block below
|
||||
out += [
|
||||
"# --- WAN -----------------------------------------------",
|
||||
"# Both vifs must be created before anything references them.",
|
||||
"# Neither firewall has vif 51 or 53 today (only 2, 3, 9, 10, 200),",
|
||||
"# and pppoe source-interface points at an interface that must",
|
||||
"# already exist -- without this the commit fails and, since the",
|
||||
"# delta commits as one unit, takes the whole switch with it.",
|
||||
f"set interfaces bonding bond0 vif {WAN_PPPOE_VIF.split('.')[1]} description 'WAN1 Vodafone (PPPoE)'",
|
||||
f"set interfaces bonding bond0 vif {WAN_DHCP_VIF.split('.')[1]} description 'WAN2 10gig ISP (DHCP)'",
|
||||
"",
|
||||
"# WAN2, the 10 gig line -- primary. The cloned MAC is what keeps",
|
||||
"# the existing public lease (87.192.101.48) instead of asking for",
|
||||
"# a new one. Only the box carrying the WAN may set this.",
|
||||
f"set interfaces bonding bond0 vif {WAN_DHCP_VIF.split('.')[1]} mac '{WAN_DHCP_MAC}'",
|
||||
f"set interfaces bonding bond0 vif {WAN_DHCP_VIF.split('.')[1]} address dhcp",
|
||||
# Demoted below Vodafone; `protocols failover` owns the live route.
|
||||
# NOT no-default-route -- that blanks new_routers in the lease and
|
||||
# leaves the failover daemon with no gateway to install.
|
||||
f"set interfaces bonding bond0 vif {WAN_DHCP_VIF.split('.')[1]} dhcp-options default-route-distance {DIST_DHCP_FALLBACK}",
|
||||
"",
|
||||
"# WAN1, Vodafone -- failover at a higher distance. Verified working",
|
||||
"# on the USG: pppoe0 came up with a public address, MTU 1492.",
|
||||
f"set interfaces pppoe {WAN_PPPOE_IF} source-interface {WAN_PPPOE_VIF}",
|
||||
f"set interfaces pppoe {WAN_PPPOE_IF} authentication username '{wan_user}'",
|
||||
f"set interfaces pppoe {WAN_PPPOE_IF} authentication password '{PLACEHOLDER}'",
|
||||
f"set interfaces pppoe {WAN_PPPOE_IF} mtu 1492",
|
||||
f"set interfaces pppoe {WAN_PPPOE_IF} default-route-distance {DIST_PPPOE}",
|
||||
# The peer's resolvers would otherwise overwrite resolv.conf.
|
||||
f"set interfaces pppoe {WAN_PPPOE_IF} no-peer-dns",
|
||||
"",
|
||||
"# The static default route exists only for unifi mode, where the",
|
||||
"# USG is the next hop. Both WANs supply one here.",
|
||||
"delete protocols static route 0.0.0.0/0",
|
||||
"",
|
||||
"# --- Health-checked primary ----------------------------",
|
||||
"# Without this, failover only fires when bond0.53 loses carrier",
|
||||
"# or its lease. An ISP that keeps the link up while dropping",
|
||||
"# traffic -- the common failure -- would black-hole everything,",
|
||||
"# because a DHCP-installed route has nothing to withdraw it.",
|
||||
"#",
|
||||
"# vyos-failover pings each target bound to the interface",
|
||||
"# (`ping -I bond0.53`), so the backup can never be validated",
|
||||
"# through the primary's path and vice versa. On withdrawal the",
|
||||
"# kernel falls through to Vodafone's distance-10 route.",
|
||||
f"set protocols failover route 0.0.0.0/0 dhcp-interface {WAN_DHCP_VIF} check type icmp",
|
||||
f"set protocols failover route 0.0.0.0/0 dhcp-interface {WAN_DHCP_VIF} check policy any-available",
|
||||
f"set protocols failover route 0.0.0.0/0 dhcp-interface {WAN_DHCP_VIF} check timeout {FAILOVER_TIMEOUT}",
|
||||
f"set protocols failover route 0.0.0.0/0 dhcp-interface {WAN_DHCP_VIF} metric {FAILOVER_METRIC}",
|
||||
*[
|
||||
f"set protocols failover route 0.0.0.0/0 dhcp-interface {WAN_DHCP_VIF} check target {t}"
|
||||
for t in FAILOVER_TARGETS
|
||||
],
|
||||
"",
|
||||
"# --- NAT -----------------------------------------------",
|
||||
f"set nat source rule 100 outbound-interface name {WAN_DHCP_VIF}",
|
||||
"set nat source rule 100 translation address masquerade",
|
||||
"set nat source rule 100 description 'LAN out via the 10gig line'",
|
||||
f"set nat source rule 110 outbound-interface name {WAN_PPPOE_IF}",
|
||||
"set nat source rule 110 translation address masquerade",
|
||||
"set nat source rule 110 description 'LAN out via Vodafone (failover)'",
|
||||
]
|
||||
|
||||
if not with_wan:
|
||||
# The backup carries the identical WAN and NAT config but with the
|
||||
# interfaces administratively DOWN. The cloned MAC is therefore never
|
||||
# live on two boxes at once, while everything needed to route and
|
||||
# masquerade is already present -- taking over is enabling two
|
||||
# interfaces, not rebuilding a config under pressure.
|
||||
#
|
||||
# NAT rules naming a down interface are harmless: VyOS warns at commit
|
||||
# ("Interface ... does not exist!") and commits anyway, verified.
|
||||
out += [
|
||||
"",
|
||||
"# --- WAN held DOWN on this box -----------------------------",
|
||||
"# Enable these two to take over the internet path:",
|
||||
f"# set interfaces bonding bond0 vif {WAN_DHCP_VIF.split('.')[1]} disable <- delete this",
|
||||
f"# set interfaces pppoe {WAN_PPPOE_IF} disable <- and this",
|
||||
f"set interfaces bonding bond0 vif {WAN_DHCP_VIF.split('.')[1]} disable",
|
||||
f"set interfaces pppoe {WAN_PPPOE_IF} disable",
|
||||
]
|
||||
|
||||
if True:
|
||||
# Port forwards and the WAN firewall go on BOTH boxes. They name
|
||||
# interfaces that are present-but-disabled on the backup, which VyOS
|
||||
# accepts (it warns and commits). Putting them here means a failover is
|
||||
# enabling an interface, not reconstructing NAT under pressure.
|
||||
# Port forwards, straight from UniFi.
|
||||
for i, p in enumerate(inv["port_forwards"]):
|
||||
if not p.get("enabled"):
|
||||
continue
|
||||
rule = 100 + i * 10
|
||||
proto = p["proto"] # tcp | udp | tcp_udp -- all valid VyOS values
|
||||
out += [
|
||||
"",
|
||||
f"set nat destination rule {rule} description '{p['name']}'",
|
||||
f"set nat destination rule {rule} inbound-interface name {WAN_DHCP_VIF}",
|
||||
f"set nat destination rule {rule} protocol {proto}",
|
||||
f"set nat destination rule {rule} destination port '{p['dst_port']}'",
|
||||
f"set nat destination rule {rule} translation address {p['fwd']}",
|
||||
]
|
||||
# `destination port` accepts a comma list but `translation port` does
|
||||
# NOT -- "16881,6881 is not a valid service name" -- because mapping a
|
||||
# list onto a list is ambiguous. Every forward here maps a port to
|
||||
# itself, and omitting translation port makes VyOS preserve the
|
||||
# original, which is exactly right. Only emit it when it genuinely
|
||||
# differs, and refuse rather than guess when a differing list appears.
|
||||
if p["fwd_port"] != p["dst_port"]:
|
||||
if "," in str(p["fwd_port"]) or "," in str(p["dst_port"]):
|
||||
raise SystemExit(
|
||||
f"port forward '{p['name']}' remaps a LIST of ports "
|
||||
f"({p['dst_port']} -> {p['fwd_port']}). VyOS cannot express "
|
||||
f"that in one rule; split it into one rule per port by hand.")
|
||||
out.append(f"set nat destination rule {rule} translation port '{p['fwd_port']}'")
|
||||
|
||||
out += [
|
||||
"",
|
||||
"# --- firewall ----------------------------------------------",
|
||||
"# VyOS defaults to accepting everything. The USG has an implicit",
|
||||
"# WAN drop, so migrating the port forwards alone would leave the",
|
||||
"# router's own services and the whole LAN reachable from the WAN.",
|
||||
"#",
|
||||
"# Scoped to the WAN interface rather than a global default-action",
|
||||
"# drop: that way a mistake here cannot lock anyone out over the LAN,",
|
||||
"# which is the only path back in during a cutover.",
|
||||
"",
|
||||
"# Traffic TO the router.",
|
||||
"set firewall ipv4 input filter default-action accept",
|
||||
"set firewall ipv4 input filter rule 100 action accept",
|
||||
"set firewall ipv4 input filter rule 100 state established",
|
||||
"set firewall ipv4 input filter rule 100 state related",
|
||||
"set firewall ipv4 input filter rule 100 description 'established/related'",
|
||||
]
|
||||
# The two WAN_LOCAL accepts carried over from UniFi.
|
||||
out += [
|
||||
"",
|
||||
"set firewall ipv4 input filter rule 110 action accept",
|
||||
"set firewall ipv4 input filter rule 110 protocol esp",
|
||||
f"set firewall ipv4 input filter rule 110 inbound-interface name {WAN_DHCP_VIF}",
|
||||
"set firewall ipv4 input filter rule 110 description 'VPN accept ESP (from UniFi WAN_LOCAL)'",
|
||||
"",
|
||||
"set firewall ipv4 input filter rule 120 action accept",
|
||||
"set firewall ipv4 input filter rule 120 protocol udp",
|
||||
"set firewall ipv4 input filter rule 120 destination port '500,4500'",
|
||||
f"set firewall ipv4 input filter rule 120 inbound-interface name {WAN_DHCP_VIF}",
|
||||
"set firewall ipv4 input filter rule 120 description 'VPN accept UDP500/4500 (from UniFi WAN_LOCAL)'",
|
||||
"",
|
||||
"set firewall ipv4 input filter rule 130 action accept",
|
||||
"set firewall ipv4 input filter rule 130 protocol icmp",
|
||||
f"set firewall ipv4 input filter rule 130 inbound-interface name {WAN_DHCP_VIF}",
|
||||
"set firewall ipv4 input filter rule 130 description 'ICMP to the router (path MTU discovery)'",
|
||||
"",
|
||||
"# Everything else arriving from the WAN is dropped. LAN is untouched.",
|
||||
"set firewall ipv4 input filter rule 900 action drop",
|
||||
f"set firewall ipv4 input filter rule 900 inbound-interface name {WAN_DHCP_VIF}",
|
||||
f"set firewall ipv4 input filter rule 910 action drop",
|
||||
f"set firewall ipv4 input filter rule 910 inbound-interface name {WAN_PPPOE_IF}",
|
||||
"set firewall ipv4 input filter rule 910 description 'drop all other WAN-to-router (Vodafone)'",
|
||||
"set firewall ipv4 input filter rule 900 description 'drop all other WAN-to-router'",
|
||||
"",
|
||||
"# Traffic THROUGH the router.",
|
||||
"set firewall ipv4 forward filter default-action accept",
|
||||
"set firewall ipv4 forward filter rule 100 action accept",
|
||||
"set firewall ipv4 forward filter rule 100 state established",
|
||||
"set firewall ipv4 forward filter rule 100 state related",
|
||||
"set firewall ipv4 forward filter rule 100 description 'established/related'",
|
||||
]
|
||||
|
||||
# Destination NAT happens before the forward filter, so these rules must
|
||||
# match the translated destination, not the WAN address.
|
||||
for i, p in enumerate(inv["port_forwards"]):
|
||||
if not p.get("enabled"):
|
||||
continue
|
||||
rule = 200 + i * 10
|
||||
out += [
|
||||
"",
|
||||
f"set firewall ipv4 forward filter rule {rule} action accept",
|
||||
f"set firewall ipv4 forward filter rule {rule} inbound-interface name {WAN_DHCP_VIF}",
|
||||
f"set firewall ipv4 forward filter rule {rule} protocol {p['proto']}",
|
||||
f"set firewall ipv4 forward filter rule {rule} destination address {p['fwd']}",
|
||||
f"set firewall ipv4 forward filter rule {rule} destination port '{p['fwd_port']}'",
|
||||
f"set firewall ipv4 forward filter rule {rule} description 'port forward: {p['name']}'",
|
||||
]
|
||||
|
||||
out += [
|
||||
"",
|
||||
"# New inbound connections from the WAN that are not a port forward.",
|
||||
"set firewall ipv4 forward filter rule 900 action drop",
|
||||
f"set firewall ipv4 forward filter rule 900 inbound-interface name {WAN_DHCP_VIF}",
|
||||
f"set firewall ipv4 forward filter rule 910 action drop",
|
||||
f"set firewall ipv4 forward filter rule 910 inbound-interface name {WAN_PPPOE_IF}",
|
||||
"set firewall ipv4 forward filter rule 910 description 'drop unsolicited WAN-to-LAN (Vodafone)'",
|
||||
"set firewall ipv4 forward filter rule 900 description 'drop unsolicited WAN-to-LAN'",
|
||||
"",
|
||||
]
|
||||
|
||||
|
||||
# --- DHCP high-availability -------------------------------------------
|
||||
# Without this BOTH boxes run kea on the same VLANs and race to answer the
|
||||
# same broadcasts, handing different pool addresses to the same client.
|
||||
# active-passive so only the primary serves, matching the VRRP shape.
|
||||
dhcp_self, dhcp_peer = NODE_ADDRS[10][self_i], NODE_ADDRS[10][peer_i]
|
||||
out += [
|
||||
"",
|
||||
"# --- DHCP high-availability --------------------------------",
|
||||
"# Peers sync leases over TCP 647. Each subnet already carries a",
|
||||
"# unique subnet-id (keyed on VLAN id), which kea HA requires.",
|
||||
"set service dhcp-server high-availability mode active-passive",
|
||||
f"set service dhcp-server high-availability status {'primary' if primary else 'secondary'}",
|
||||
# The peer name must not collide with either system host-name.
|
||||
f"set service dhcp-server high-availability name {DHCP_HA_NAME}",
|
||||
f"set service dhcp-server high-availability source-address {dhcp_self}",
|
||||
f"set service dhcp-server high-availability remote {dhcp_peer}",
|
||||
]
|
||||
|
||||
if conntrack_link:
|
||||
# Stateful failover. Without it VRRP moves the address but every
|
||||
# established connection dies, because the backup has no conntrack
|
||||
# table. Needs the eth3 <-> eth3 cable physically present.
|
||||
out += [
|
||||
"",
|
||||
"# --- conntrack-sync ----------------------------------------",
|
||||
"# Dedicated point-to-point link: sync traffic must not compete",
|
||||
"# with production, and must not die when the LAN does.",
|
||||
f"set interfaces ethernet {CONNTRACK_IF} address {CONNTRACK_ADDRS[self_i]}",
|
||||
f"set interfaces ethernet {CONNTRACK_IF} description 'conntrack-sync peer link'",
|
||||
f"set service conntrack-sync interface {CONNTRACK_IF}",
|
||||
"set service conntrack-sync failover-mechanism vrrp sync-group MAIN",
|
||||
"set service conntrack-sync accept-protocol tcp",
|
||||
"set service conntrack-sync accept-protocol udp",
|
||||
"set service conntrack-sync accept-protocol icmp",
|
||||
"set service conntrack-sync mcast-group 225.0.0.50",
|
||||
]
|
||||
|
||||
# DHCP + DNS, from the same generator labsim proved.
|
||||
dhcp_lines, stats = unifi_to_vyos.build(inv, "prod")
|
||||
expected = len(inv["reservations"])
|
||||
if stats["mappings"] != expected:
|
||||
raise SystemExit(
|
||||
f"refusing to generate: {expected - stats['mappings']} reservation(s) "
|
||||
f"missing -- every one must survive the cutover")
|
||||
out += dhcp_lines
|
||||
return out
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--priority", type=int, required=True,
|
||||
help="VRRP priority: 200 for the master, 100 for the backup")
|
||||
ap.add_argument("--inventory", default=os.path.join(HERE, "export", "inventory.json"))
|
||||
ap.add_argument("--raw-networkconf", default=os.path.join(HERE, "export", "rest_networkconf.json"))
|
||||
ap.add_argument("--with-wan", action="store_true",
|
||||
help="configure the WAN on this box. Only ONE of the pair may have\n it, because the cloned WAN MAC must be unique.")
|
||||
ap.add_argument("--conntrack-link", action="store_true",
|
||||
help="emit conntrack-sync over the eth3 peer link. Requires the\n cable to be physically present on both boxes.")
|
||||
ap.add_argument("-o", "--out")
|
||||
ap.add_argument("--emit-secrets", metavar="PATH",
|
||||
help="write the PPPoE credential to PATH with mode 0600 and exit")
|
||||
args = ap.parse_args()
|
||||
|
||||
with open(args.inventory) as fh:
|
||||
inv = json.load(fh)
|
||||
with open(args.raw_networkconf) as fh:
|
||||
raw_nets = json.load(fh)
|
||||
|
||||
wan = next((n for n in raw_nets
|
||||
if n.get("purpose") == "wan" and n.get("wan_username")), None)
|
||||
if wan is None:
|
||||
print("no WAN network with credentials found in the export", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if args.emit_secrets:
|
||||
fd = os.open(args.emit_secrets, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
with os.fdopen(fd, "w") as fh:
|
||||
fh.write(f"WAN_PASSWORD='{wan.get('x_wan_password', '')}'\n")
|
||||
# Re-assert the mode in case the file already existed with a wider one.
|
||||
os.chmod(args.emit_secrets, 0o600)
|
||||
mode = oct(os.stat(args.emit_secrets).st_mode & 0o777)
|
||||
print(f"wrote {args.emit_secrets} (mode {mode}) for user {wan['wan_username']}",
|
||||
file=sys.stderr)
|
||||
return 0
|
||||
|
||||
lines = build_delta(inv, args.priority, wan["wan_username"], args.with_wan,
|
||||
args.conntrack_link)
|
||||
text = "\n".join(lines) + "\n"
|
||||
|
||||
# Only a WAN-carrying delta has a credential to placeholder-substitute.
|
||||
if args.with_wan and PLACEHOLDER not in text:
|
||||
print("BUG: password placeholder missing from a WAN delta", file=sys.stderr)
|
||||
return 1
|
||||
if wan.get("x_wan_password") and wan["x_wan_password"] in text:
|
||||
print("BUG: the WAN password leaked into the delta", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
n_set = sum(1 for l in lines if l.startswith("set "))
|
||||
n_del = sum(1 for l in lines if l.startswith("delete "))
|
||||
print(f"delta: {n_set} set, {n_del} delete, priority {args.priority}, "
|
||||
f"{len(inv['reservations'])} reservations", file=sys.stderr)
|
||||
|
||||
if args.out:
|
||||
with open(args.out, "w") as fh:
|
||||
fh.write(text)
|
||||
print(f"wrote {args.out}", file=sys.stderr)
|
||||
else:
|
||||
sys.stdout.write(text)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
304
migration/vyos-unifi-switch
Executable file
304
migration/vyos-unifi-switch
Executable file
@@ -0,0 +1,304 @@
|
||||
#!/bin/bash
|
||||
# Switch this VyOS box between passive (USG is the gateway) and active
|
||||
# (VyOS is the gateway). Installed at /config/vyos-unifi-switch, which
|
||||
# survives image upgrades, so it can be run from a local terminal or the
|
||||
# JetKVM console with no workstation and no internet.
|
||||
#
|
||||
# vyos-unifi-switch report the current mode
|
||||
# vyos-unifi-switch vyos take over: VIPs to .1, DHCP, DNS, PPPoE, NAT
|
||||
# vyos-unifi-switch unifi revert; the USG can then be reconnected
|
||||
#
|
||||
# READ THIS BEFORE THE CUTOVER
|
||||
#
|
||||
# `unifi` is the escape hatch. It runs no health checks, asks no questions and
|
||||
# has nothing that can refuse. If anything at all looks wrong, run it, then
|
||||
# plug the USG back in.
|
||||
#
|
||||
# `vyos` commits with commit-confirm. If it is not confirmed -- because the
|
||||
# health checks failed, or because you lost access, or because you walked away
|
||||
# -- the box returns to the saved configuration on its own. That requires
|
||||
# `system config-management commit-confirm action reload`; without it VyOS
|
||||
# REBOOTS instead, which on a gateway is an outage rather than an undo. The
|
||||
# script refuses to run if that setting is missing.
|
||||
set -uo pipefail
|
||||
|
||||
MODES=/config/modes
|
||||
UNIFI_BOOT="$MODES/unifi.boot"
|
||||
DELTA="$MODES/to-vyos.commands"
|
||||
SECRETS=/config/wan-secrets
|
||||
MARKER="$MODES/current-mode"
|
||||
CONFIRM_MINUTES="${CONFIRM_MINUTES:-10}"
|
||||
OPRUN=/opt/vyatta/bin/vyatta-op-cmd-wrapper
|
||||
|
||||
say() { printf '[switch] %s\n' "$*"; }
|
||||
warn() { printf '[switch] WARNING: %s\n' "$*" >&2; }
|
||||
die() { printf '[switch] ERROR: %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
# Run configuration commands in a real config session. Everything the caller
|
||||
# feeds in runs between `configure` and `exit`.
|
||||
run_cfg() {
|
||||
local script rc
|
||||
script="$(mktemp)"
|
||||
{
|
||||
echo 'source /opt/vyatta/etc/functions/script-template'
|
||||
echo 'configure'
|
||||
cat
|
||||
echo 'exit'
|
||||
} > "$script"
|
||||
vbash "$script"; rc=$?
|
||||
rm -f "$script"
|
||||
return $rc
|
||||
}
|
||||
|
||||
# Two traps live in this one function, both of which produced wrong answers
|
||||
# rather than errors:
|
||||
# 1. `show configuration commands` quotes values ("action 'reload'"), so the
|
||||
# quotes have to go before matching or every value-bearing check fails.
|
||||
# 2. `... | grep -q` under `set -o pipefail` reports FAILURE even on a match:
|
||||
# grep exits at the first hit, the producer takes SIGPIPE, and pipefail
|
||||
# surfaces that. Whether it triggers depends on output size, so it fails
|
||||
# intermittently. Match against a captured string instead of a pipeline.
|
||||
cfg_has() {
|
||||
local out
|
||||
out="$($OPRUN show configuration commands 2>/dev/null | tr -d "'")"
|
||||
case "$out" in *"$1"*) return 0 ;; *) return 1 ;; esac
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------- status ----
|
||||
current_mode() {
|
||||
# The marker records intent; the running config is the truth. Report the
|
||||
# config, and complain if the two disagree.
|
||||
local live="unknown"
|
||||
if cfg_has "set service dhcp-server"; then live="vyos"; else live="unifi"; fi
|
||||
echo "$live"
|
||||
}
|
||||
|
||||
show_status() {
|
||||
local live marked
|
||||
live="$(current_mode)"
|
||||
marked="$(cat "$MARKER" 2>/dev/null || echo "never set")"
|
||||
echo "host: $(hostname)"
|
||||
echo "mode: $live (marker: $marked)"
|
||||
[ "$live" != "$marked" ] && [ "$marked" != "never set" ] && \
|
||||
warn "marker disagrees with the running config -- trust the config"
|
||||
echo "VRRP:"
|
||||
$OPRUN show vrrp 2>/dev/null | sed -n '3,$p' | awk '{printf " %-9s %-12s %s\n", $1, $2, $4}'
|
||||
echo "DHCP server: $(systemctl is-active isc-kea-dhcp4-server 2>/dev/null)"
|
||||
echo "DNS forwarder: $(systemctl is-active pdns-recursor 2>/dev/null)"
|
||||
echo "WAN (pppoe0): $(ip -4 -br addr show pppoe0 2>/dev/null | awk '{print $2, $3}' || echo 'not present')"
|
||||
echo "default route: $(ip -4 route show default 2>/dev/null | head -1 || echo none)"
|
||||
echo "unsaved changes: $(cfg_unsaved)"
|
||||
}
|
||||
|
||||
cfg_unsaved() {
|
||||
if /usr/bin/config-mgmt compare >/dev/null 2>&1; then echo "no"; else echo "possibly - check 'compare saved'"; fi
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------- preflight ----
|
||||
require_files() {
|
||||
[ -r "$UNIFI_BOOT" ] || die "missing $UNIFI_BOOT -- capture it while the USG is still the gateway"
|
||||
[ -s "$UNIFI_BOOT" ] || die "$UNIFI_BOOT is empty"
|
||||
}
|
||||
|
||||
require_reload_action() {
|
||||
cfg_has "set system config-management commit-confirm action reload" && return 0
|
||||
die "commit-confirm action is not 'reload'. Without it an unconfirmed switch
|
||||
REBOOTS this box instead of reverting. Fix first:
|
||||
configure
|
||||
set system config-management commit-confirm action reload
|
||||
commit; save"
|
||||
}
|
||||
|
||||
# The single worst outcome available is two devices answering on the gateway
|
||||
# address. It costs one ARP probe to make that impossible.
|
||||
# Returns 0 (success) when something IS answering on a gateway address we are
|
||||
# about to claim -- i.e. "yes, still alive, do not proceed".
|
||||
usg_still_alive() {
|
||||
local found=0 ip dev targets
|
||||
targets="$(grep -oE "vrrp group [a-z0-9]+ address [0-9.]+" "$DELTA" 2>/dev/null | awk '{print $NF}')"
|
||||
if [ -z "$targets" ]; then
|
||||
# A delta with no VIPs cannot be probed, which means the single guard
|
||||
# against two devices sharing a gateway address is inert. Refuse by
|
||||
# default: an unprobeable delta on the real boxes is a broken delta, and
|
||||
# "warn and continue" would let the one failure this script exists to
|
||||
# prevent through unnoticed. The override is for the lab only.
|
||||
if [ "${ALLOW_NO_VIP_DELTA:-0}" = "1" ]; then
|
||||
warn "delta claims no VIPs; proceeding because ALLOW_NO_VIP_DELTA=1 (lab only)"
|
||||
return 1
|
||||
fi
|
||||
die "this delta claims no VIPs, so the gateway-address guard cannot run.
|
||||
On the real boxes that means a broken delta. If this really is a lab
|
||||
run, re-invoke with ALLOW_NO_VIP_DELTA=1."
|
||||
fi
|
||||
for ip in $targets; do
|
||||
dev="$(ip -4 route get "$ip" 2>/dev/null | sed -n 's/.* dev \([^ ]*\).*/\1/p' | head -1)"
|
||||
if [ -n "$dev" ] && command -v arping >/dev/null 2>&1; then
|
||||
# ARP is the right probe: it answers even when the host filters ICMP.
|
||||
if arping -c 2 -w 3 -f -I "$dev" "$ip" >/dev/null 2>&1; then
|
||||
warn "something already answers ARP on $ip (via $dev)"; found=1
|
||||
fi
|
||||
elif ping -c 2 -W 2 "$ip" >/dev/null 2>&1; then
|
||||
warn "something already answers ICMP on $ip"; found=1
|
||||
fi
|
||||
done
|
||||
[ "$found" -eq 1 ]
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------- to unifi -----
|
||||
to_unifi() {
|
||||
require_files
|
||||
say "reverting to unifi mode (USG is the gateway)"
|
||||
run_cfg <<EOF || die "load/commit failed -- the box is unchanged, use the console"
|
||||
load $UNIFI_BOOT
|
||||
commit
|
||||
save
|
||||
EOF
|
||||
echo "unifi" > "$MARKER"
|
||||
say "done. The USG can be reconnected."
|
||||
say "If it was already connected during this, nothing was disturbed."
|
||||
}
|
||||
|
||||
# -------------------------------------------------------------- to vyos -----
|
||||
health_checks() {
|
||||
local fails=0
|
||||
_chk() { # name, command
|
||||
if eval "$2" >/dev/null 2>&1; then say " ok $1"; else say " FAIL $1"; fails=$((fails+1)); fi
|
||||
}
|
||||
_chk "kea (DHCP) is running" "systemctl is-active --quiet isc-kea-dhcp4-server"
|
||||
_chk "DNS forwarder is running" "systemctl is-active --quiet pdns-recursor"
|
||||
|
||||
# Only assert on the WAN if this delta actually brings one up. A delta with
|
||||
# no PPPoE stanza is a lab/partial delta, and failing it on a missing
|
||||
# pppoe0 would make the script untestable anywhere but the live cutover.
|
||||
# Announced loudly, because a quietly skipped check is worse than no check.
|
||||
# A box whose delta holds its WAN interfaces DOWN is the backup: it has no
|
||||
# route out by design, and demanding one reverts a perfectly correct config.
|
||||
# This tore down vyos002 on the first successful cutover -- vyos001 went live
|
||||
# and its backup was judged unhealthy for lacking the WAN it is deliberately
|
||||
# not carrying. Same mistake as requiring the failover line: checks that do
|
||||
# not apply to the box being checked.
|
||||
if grep -qE "^set interfaces (pppoe pppoe0|bonding bond0 vif 53) disable$" "$DELTA"; then
|
||||
say " note this box holds its WAN down (backup); skipping WAN checks"
|
||||
elif grep -qE "^set interfaces (pppoe|bonding bond0 vif 5)" "$DELTA"; then
|
||||
# What matters is that SOME WAN works, not that every WAN works.
|
||||
#
|
||||
# This reverted a cutover that had genuinely succeeded. The 10 gig line came
|
||||
# up on bond0.53 and the cloned MAC was handed the same public address the
|
||||
# USG had (87.192.101.48); kea was serving real LAN clients at the same
|
||||
# moment. The only failure was pppoe0 -- the Vodafone FAILOVER line -- and
|
||||
# requiring it undid a working gateway.
|
||||
#
|
||||
# Written as [ -n "$(...)" ] rather than `... | grep -q` for the pipefail
|
||||
# reason above: a pipeline ending in grep -q cannot be trusted here.
|
||||
_chk "a default route exists" '[ -n "$(ip -4 route show default)" ]'
|
||||
_chk "internet reachable" "ping -c2 -W3 8.8.8.8"
|
||||
_chk "DNS resolves through us" "getent hosts vyos.net"
|
||||
|
||||
# Informational only: report each WAN, fail on neither. A failover line
|
||||
# being down is worth seeing, not worth reverting for.
|
||||
for _w in pppoe0 bond0.53; do
|
||||
if [ -n "$(ip -4 -br addr show "$_w" 2>/dev/null | awk '{print $3}')" ]; then
|
||||
say " ok WAN $_w has an address (informational)"
|
||||
else
|
||||
say " note WAN $_w has no address (informational, not fatal)"
|
||||
fi
|
||||
done
|
||||
else
|
||||
warn "this delta configures no WAN -- skipping all WAN health checks."
|
||||
warn "That is expected in the lab and WRONG for the real cutover."
|
||||
fi
|
||||
return $fails
|
||||
}
|
||||
|
||||
to_vyos() {
|
||||
require_files
|
||||
require_reload_action
|
||||
[ -r "$DELTA" ] || die "missing $DELTA"
|
||||
|
||||
if usg_still_alive; then
|
||||
die "refusing: something is still answering on a gateway address.
|
||||
Disconnect the USG first. Two devices on the same gateway IP is the
|
||||
one failure this script exists to prevent."
|
||||
fi
|
||||
|
||||
local pw="" tmp
|
||||
if [ -r "$SECRETS" ]; then
|
||||
# shellcheck disable=SC1090
|
||||
. "$SECRETS"; pw="${WAN_PASSWORD:-}"
|
||||
fi
|
||||
[ -n "$pw" ] || warn "no WAN_PASSWORD in $SECRETS -- PPPoE will not authenticate"
|
||||
|
||||
tmp="$(mktemp)"; chmod 600 "$tmp"
|
||||
sed "s|@@WAN_PASSWORD@@|${pw}|g" "$DELTA" | grep -vE '^\s*(#|$)' > "$tmp"
|
||||
|
||||
say "switching to vyos mode (this box becomes the gateway)"
|
||||
say "commit-confirm: ${CONFIRM_MINUTES} min to confirm, else it reverts itself"
|
||||
|
||||
# commit-confirm is TWO steps, and doing only the first commits nothing:
|
||||
# `config-mgmt commit_confirm` arms the revert timer, then a normal `commit`
|
||||
# applies the candidate config. The interactive prompt lives in the first
|
||||
# step, which is why it is invoked directly with -y instead of via the
|
||||
# `commit-confirm` alias. IN_COMMIT_CONFIRM is what the real CLI sets, and
|
||||
# the commit hooks look at it.
|
||||
if ! run_cfg < <(printf 'load %s\n' "$UNIFI_BOOT"; cat "$tmp";
|
||||
printf 'sudo sg vyattacfg "/usr/bin/config-mgmt commit_confirm -y -t=%s"\n' "$CONFIRM_MINUTES";
|
||||
printf 'export IN_COMMIT_CONFIRM=t\ncommit\nunset IN_COMMIT_CONFIRM\n'); then
|
||||
rm -f "$tmp"
|
||||
die "commit-confirm failed. Nothing was applied; the box is still in its
|
||||
previous mode. Run '$0 unifi' if you are unsure."
|
||||
fi
|
||||
rm -f "$tmp"
|
||||
|
||||
# Poll, do not sample once.
|
||||
#
|
||||
# A cutover attempt failed here on a fixed 25s wait. That is far too short for
|
||||
# a WAN: PPPoE is PADI/PADO/PADR/PADS then LCP, auth and IPCP, routinely 15-30s
|
||||
# by itself, and both lines had just been released by the USG seconds earlier.
|
||||
# ISPs commonly hold the previous session and MAC binding for minutes before
|
||||
# leasing to the "same" CPE again -- which is precisely what a cloned MAC looks
|
||||
# like to them. One sample at 25s reported a healthy setup as broken and
|
||||
# reverted it.
|
||||
#
|
||||
# There is still a deadline, because commit-confirm is running: stop well
|
||||
# before it so the decision is ours rather than the timer's.
|
||||
local budget="${HEALTH_BUDGET:-180}" waited=0 step=15
|
||||
say "committed. Polling health for up to ${budget}s (commit-confirm has ${CONFIRM_MINUTES} min)..."
|
||||
while :; do
|
||||
sleep "$step"; waited=$(( waited + step ))
|
||||
if health_checks >/dev/null 2>&1; then
|
||||
say "healthy after ${waited}s"
|
||||
break
|
||||
fi
|
||||
if [ "$waited" -ge "$budget" ]; then
|
||||
say "still unhealthy after ${waited}s -- final check:"
|
||||
break
|
||||
fi
|
||||
say " not healthy yet at ${waited}s, still waiting..."
|
||||
done
|
||||
|
||||
say "health checks:"
|
||||
if health_checks; then
|
||||
say "all checks passed -- confirming"
|
||||
/usr/bin/config-mgmt confirm >/dev/null 2>&1 || die "confirm failed; it will revert on its own shortly"
|
||||
run_cfg <<'EOF' || warn "save failed -- config is live but will not survive a reboot"
|
||||
save
|
||||
EOF
|
||||
echo "vyos" > "$MARKER"
|
||||
say "vyos mode is live and saved."
|
||||
else
|
||||
warn "health checks FAILED -- reverting now rather than waiting out the timer"
|
||||
/usr/bin/config-mgmt revert_soft >/dev/null 2>&1 \
|
||||
|| warn "revert_soft failed; the commit-confirm timer will still fire within ${CONFIRM_MINUTES} min"
|
||||
echo "unifi" > "$MARKER"
|
||||
die "reverted to the previous configuration. Reconnect the USG.
|
||||
Check: ip addr show pppoe0; journalctl -u pppd; $0 status"
|
||||
fi
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------- main -----
|
||||
case "${1:-status}" in
|
||||
vyos) to_vyos ;;
|
||||
unifi) to_unifi ;;
|
||||
status) show_status ;;
|
||||
*) echo "usage: $(basename "$0") [vyos|unifi|status]" >&2; exit 2 ;;
|
||||
esac
|
||||
1
pulumi-vyos/Pulumi.labsim.yaml
Normal file
1
pulumi-vyos/Pulumi.labsim.yaml
Normal file
@@ -0,0 +1 @@
|
||||
encryptionsalt: v1:tMNG4q79HiI=:v1:K35iEOw3pgyukCr6:LmKO37/jghjyT2sLuoLmCQIDPOIsgA==
|
||||
3
pulumi-vyos/Pulumi.yaml
Normal file
3
pulumi-vyos/Pulumi.yaml
Normal file
@@ -0,0 +1,3 @@
|
||||
name: vyos-proto
|
||||
runtime: nodejs
|
||||
description: Prototype — VyOS config subtrees as Pulumi resources, with commit-confirm
|
||||
72
pulumi-vyos/README.md
Normal file
72
pulumi-vyos/README.md
Normal file
@@ -0,0 +1,72 @@
|
||||
# VyOS as Pulumi resources — prototype
|
||||
|
||||
Proves that VyOS config can be managed from the same Pulumi plan as the
|
||||
Kubernetes side, **without** giving up the safety property that matters on a
|
||||
gateway: a config push that breaks your access undoes itself.
|
||||
|
||||
## Why not the community Terraform providers
|
||||
|
||||
They are better than expected. `foltik/vyos`'s `vyos_config_block_tree` flattens
|
||||
an entire subtree into one payload and sends **one POST to `/configure`** —
|
||||
so one resource is one commit, not one commit per config line. That worry was
|
||||
unfounded.
|
||||
|
||||
What they do *not* do is send `confirm_time`. The payload is only
|
||||
`op`/`path`/`value`, so every change is an unprotected commit. On a router you
|
||||
reach *through* the router, that is the difference between a mistake and an
|
||||
outage.
|
||||
|
||||
## What the VyOS API actually supports
|
||||
|
||||
Read from `rest/models.py` and `rest/routers.py` on the box, then verified by
|
||||
hand against a live router:
|
||||
|
||||
- **Batching**: a list of operations in one request, applied as one commit.
|
||||
- **commit-confirm**: `confirm_time` on the request starts the revert timer.
|
||||
Response says `Initialized commit-confirm; N minutes to confirm before reload`.
|
||||
|
||||
Three details that cost time and are easy to get wrong:
|
||||
|
||||
1. **`confirm_time` is only read when the body parses as `ConfigureListModel`** —
|
||||
i.e. `{"commands": [...], "confirm_time": N}`. A **bare array** is accepted
|
||||
and committed happily with **no timer armed**. It looks identical to success.
|
||||
The resource therefore checks the response actually mentions commit-confirm
|
||||
and refuses to continue if it does not.
|
||||
2. **There is no `/confirm` endpoint.** Confirming is an op on `/configure`.
|
||||
3. **Confirm still requires a `path` field**, even though it ignores it — the
|
||||
Union resolves to `ConfigureModel`, which mandates `path`. Without it you get
|
||||
`missing 'path' field` and the timer keeps running.
|
||||
|
||||
## Shape
|
||||
|
||||
One resource per **subtree**, not per line:
|
||||
|
||||
```ts
|
||||
new VyosConfigTree("dns-forwarding", {
|
||||
host, apiKey,
|
||||
path: ["service", "dns", "forwarding"],
|
||||
commands: [["cache-size", "20000"], ["name-server", "8.8.8.8"]],
|
||||
confirmMinutes: 2,
|
||||
});
|
||||
```
|
||||
|
||||
Apply is `delete <path>` + all the `set`s in one request, so the result is the
|
||||
declared state rather than a merge with whatever was there — which is what makes
|
||||
`pulumi up` converge instead of accumulate.
|
||||
|
||||
## Verified on labsim
|
||||
|
||||
- `pulumi up` create and update both land in ~6s, each a single
|
||||
commit-confirmed transaction; update shows `[diff: ~commands]`.
|
||||
- Auto-revert observed: an unconfirmed commit returned the router to its saved
|
||||
config on its own.
|
||||
- `pulumi destroy` removes the subtree.
|
||||
|
||||
## Not done
|
||||
|
||||
- The API is HTTP with a self-signed certificate and `rejectUnauthorized: false`.
|
||||
Bind it to the management VLAN or the peer link and install a real
|
||||
certificate before this goes near production.
|
||||
- No `refresh`/drift detection yet: `read` is not implemented, so out-of-band
|
||||
changes are not noticed until the next `up` overwrites them.
|
||||
- The cutover itself should still use `vyos-unifi-switch`. This is for day-2.
|
||||
27
pulumi-vyos/index.ts
Normal file
27
pulumi-vyos/index.ts
Normal file
@@ -0,0 +1,27 @@
|
||||
import * as pulumi from "@pulumi/pulumi";
|
||||
import { VyosConfigTree } from "./vyosConfigTree";
|
||||
|
||||
const cfg = new pulumi.Config();
|
||||
const host = cfg.get("host") ?? "172.31.1.252";
|
||||
const apiKey = cfg.get("apiKey") ?? "labsim-proto-key";
|
||||
|
||||
// One subtree, one resource, one commit. This is the shape a BGP change would
|
||||
// take: edit the commands array, `pulumi up`, and it lands as a single
|
||||
// commit-confirmed transaction alongside whatever Kubernetes resources changed
|
||||
// in the same plan.
|
||||
const dnsForwarding = new VyosConfigTree("dns-forwarding", {
|
||||
host, apiKey,
|
||||
path: ["service", "dns", "forwarding"],
|
||||
commands: [
|
||||
["cache-size", "20000"],
|
||||
["listen-address", "172.31.10.1"],
|
||||
["allow-from", "172.31.10.0/23"],
|
||||
["name-server", "8.8.8.8"],
|
||||
["name-server", "8.8.4.4"],
|
||||
["name-server", "1.1.1.1"],
|
||||
],
|
||||
confirmMinutes: 2,
|
||||
save: true,
|
||||
});
|
||||
|
||||
export const managed = dnsForwarding.id;
|
||||
11
pulumi-vyos/package.json
Normal file
11
pulumi-vyos/package.json
Normal file
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "vyos-proto",
|
||||
"main": "index.ts",
|
||||
"devDependencies": {
|
||||
"@types/node": "^22",
|
||||
"typescript": "^5.9.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"@pulumi/pulumi": "^3.140.0"
|
||||
}
|
||||
}
|
||||
1822
pulumi-vyos/pnpm-lock.yaml
generated
Normal file
1822
pulumi-vyos/pnpm-lock.yaml
generated
Normal file
File diff suppressed because it is too large
Load Diff
2
pulumi-vyos/tsconfig.json
Normal file
2
pulumi-vyos/tsconfig.json
Normal file
@@ -0,0 +1,2 @@
|
||||
{ "compilerOptions": { "strict": true, "target": "es2020", "module": "commonjs",
|
||||
"moduleResolution": "node", "skipLibCheck": true, "esModuleInterop": true } }
|
||||
205
pulumi-vyos/vyosConfigTree.ts
Normal file
205
pulumi-vyos/vyosConfigTree.ts
Normal file
@@ -0,0 +1,205 @@
|
||||
import * as pulumi from "@pulumi/pulumi";
|
||||
import * as https from "https";
|
||||
|
||||
/**
|
||||
* A VyOS config subtree, managed as ONE Pulumi resource.
|
||||
*
|
||||
* Why a dynamic provider rather than the community Terraform providers: they
|
||||
* batch correctly (one `vyos_config_block_tree` becomes a single POST to
|
||||
* /configure, so one commit) but they never send `confirm_time`. VyOS's own API
|
||||
* supports it -- `ConfigureListModel.confirm_time`, and "A non-zero confirm_time
|
||||
* will start commit-confirm timer on commit" in rest/routers.py -- so a config
|
||||
* push that breaks your access to the router can undo itself. On a gateway that
|
||||
* is the difference between a mistake and an outage, and it is worth ~150 lines
|
||||
* to keep.
|
||||
*
|
||||
* The unit of change is a SUBTREE, not a line. `protocols bgp` is one resource;
|
||||
* so is `service dhcp-server`. That keeps one Pulumi resource == one commit,
|
||||
* rather than turning 300 config lines into 300 commits with no ordering
|
||||
* guarantee and no way to revert them as a unit.
|
||||
*/
|
||||
|
||||
export interface VyosConfigTreeArgs {
|
||||
/** Router address, e.g. "10.0.1.252". */
|
||||
host: pulumi.Input<string>;
|
||||
/** API key from `set service https api keys id <n> key <k>`. */
|
||||
apiKey: pulumi.Input<string>;
|
||||
/** Subtree root as a path array, e.g. ["protocols", "bgp"]. */
|
||||
path: pulumi.Input<string[]>;
|
||||
/**
|
||||
* Desired state of the subtree: `set` command suffixes relative to `path`,
|
||||
* each already split into path components with the value last.
|
||||
*/
|
||||
commands: pulumi.Input<string[][]>;
|
||||
/**
|
||||
* Minutes before an unconfirmed commit reverts itself. 0 disables it.
|
||||
* Non-zero is strongly preferred for anything reachable only through the
|
||||
* router being changed.
|
||||
*/
|
||||
confirmMinutes?: pulumi.Input<number>;
|
||||
/** Persist to config.boot after a successful confirm. */
|
||||
save?: pulumi.Input<boolean>;
|
||||
}
|
||||
|
||||
interface Inputs {
|
||||
host: string;
|
||||
apiKey: string;
|
||||
path: string[];
|
||||
commands: string[][];
|
||||
confirmMinutes: number;
|
||||
save: boolean;
|
||||
}
|
||||
|
||||
// Kept inside the module so the dynamic provider can serialise it.
|
||||
function apiCall(
|
||||
host: string, apiKey: string, endpoint: string,
|
||||
form: Record<string, string>,
|
||||
): Promise<any> {
|
||||
const body = Object.entries(form)
|
||||
.map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`)
|
||||
.join("&");
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = https.request({
|
||||
host, port: 443, path: `/${endpoint}`, method: "POST",
|
||||
// VyOS ships a self-signed certificate by default. Verification is
|
||||
// disabled deliberately; if this ever leaves a trusted segment,
|
||||
// install a real certificate and turn it back on.
|
||||
rejectUnauthorized: false,
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Content-Length": Buffer.byteLength(body),
|
||||
},
|
||||
timeout: 120_000,
|
||||
}, (res) => {
|
||||
let data = "";
|
||||
res.on("data", (c) => (data += c));
|
||||
res.on("end", () => {
|
||||
try {
|
||||
const parsed = JSON.parse(data);
|
||||
if (parsed.success === false) {
|
||||
reject(new Error(`VyOS API: ${parsed.error ?? data}`));
|
||||
} else {
|
||||
resolve(parsed);
|
||||
}
|
||||
} catch {
|
||||
reject(new Error(`VyOS API returned non-JSON: ${data.slice(0, 300)}`));
|
||||
}
|
||||
});
|
||||
});
|
||||
req.on("error", reject);
|
||||
req.on("timeout", () => { req.destroy(); reject(new Error("VyOS API timed out")); });
|
||||
req.write(body);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace a subtree: delete it, then set the desired state, in ONE request so
|
||||
* it is a single commit. Deleting first makes the result the declared state
|
||||
* rather than a merge with whatever was there -- which is what makes `pulumi
|
||||
* up` converge instead of accumulating.
|
||||
*/
|
||||
async function applyTree(i: Inputs, desired: string[][]): Promise<void> {
|
||||
const commands: any[] = [{ op: "delete", path: i.path }];
|
||||
for (const c of desired) {
|
||||
commands.push({ op: "set", path: [...i.path, ...c] });
|
||||
}
|
||||
|
||||
// The payload MUST be {commands: [...], confirm_time: N}, not a bare array.
|
||||
// VyOS only reads confirm_time when the body parses as ConfigureListModel
|
||||
// (`if isinstance(data, (ConfigureModel, ConfigureListModel, ...))` in
|
||||
// rest/routers.py). A bare array is accepted and committed happily -- with
|
||||
// NO timer armed, silently discarding the safety net. Verified both ways:
|
||||
// bare array gives no "Initialized commit-confirm" in the response, the
|
||||
// object form returns "Initialized commit-confirm; N minutes to confirm".
|
||||
const payload: any = { commands };
|
||||
if (i.confirmMinutes > 0) payload.confirm_time = i.confirmMinutes;
|
||||
|
||||
const res = await apiCall(i.host, i.apiKey, "configure",
|
||||
{ key: i.apiKey, data: JSON.stringify(payload) });
|
||||
|
||||
if (i.confirmMinutes > 0) {
|
||||
// Refuse to continue if the timer did not actually arm -- otherwise a
|
||||
// silent fallback to an unprotected commit is exactly the failure this
|
||||
// resource exists to prevent.
|
||||
const said = String((res && res.data) || "");
|
||||
if (!said.includes("commit-confirm")) {
|
||||
throw new Error(
|
||||
"commit-confirm was requested but VyOS did not arm a timer " +
|
||||
`(response: ${said.trim() || "<empty>"}). Refusing to proceed.`);
|
||||
}
|
||||
}
|
||||
|
||||
if (i.confirmMinutes > 0) {
|
||||
// The commit landed but is on a timer. Reaching the API again proves
|
||||
// the box is still answering *after* the change -- the only evidence
|
||||
// worth confirming on. If this throws we deliberately do NOT confirm,
|
||||
// and the router reverts itself.
|
||||
await apiCall(i.host, i.apiKey, "retrieve", {
|
||||
key: i.apiKey,
|
||||
data: JSON.stringify({ op: "showConfig", path: [] }),
|
||||
});
|
||||
// There is no /confirm endpoint -- confirm is an op on /configure, and
|
||||
// it requires a `path` field even though it ignores it (the Union
|
||||
// resolves to ConfigureModel, which mandates path). Without it the API
|
||||
// answers "missing 'path' field" and the timer keeps running.
|
||||
await apiCall(i.host, i.apiKey, "configure", {
|
||||
key: i.apiKey,
|
||||
data: JSON.stringify({ op: "confirm", path: [] }),
|
||||
});
|
||||
}
|
||||
|
||||
if (i.save) {
|
||||
await apiCall(i.host, i.apiKey, "config-file", {
|
||||
key: i.apiKey, data: JSON.stringify({ op: "save" }),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const provider: pulumi.dynamic.ResourceProvider = {
|
||||
async create(inputs: Inputs) {
|
||||
await applyTree(inputs, inputs.commands);
|
||||
return { id: `${inputs.host}:${inputs.path.join("/")}`, outs: inputs };
|
||||
},
|
||||
|
||||
async update(_id, _old: Inputs, news: Inputs) {
|
||||
await applyTree(news, news.commands);
|
||||
return { outs: news };
|
||||
},
|
||||
|
||||
async delete(_id, props: Inputs) {
|
||||
const form: Record<string, string> = {
|
||||
key: props.apiKey,
|
||||
data: JSON.stringify([{ op: "delete", path: props.path }]),
|
||||
};
|
||||
if (props.confirmMinutes > 0) form.confirm_time = String(props.confirmMinutes);
|
||||
await apiCall(props.host, props.apiKey, "configure", form);
|
||||
if (props.confirmMinutes > 0) {
|
||||
await apiCall(props.host, props.apiKey, "configure",
|
||||
{ key: props.apiKey, data: JSON.stringify({ op: "confirm", path: [] }) });
|
||||
}
|
||||
},
|
||||
|
||||
async diff(_id, olds: Inputs, news: Inputs) {
|
||||
const changed =
|
||||
JSON.stringify(olds.commands) !== JSON.stringify(news.commands) ||
|
||||
JSON.stringify(olds.path) !== JSON.stringify(news.path) ||
|
||||
olds.host !== news.host;
|
||||
return {
|
||||
changes: changed,
|
||||
// Changing which subtree or which router is a different resource.
|
||||
replaces: olds.host !== news.host ||
|
||||
JSON.stringify(olds.path) !== JSON.stringify(news.path) ? ["path"] : [],
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
export class VyosConfigTree extends pulumi.dynamic.Resource {
|
||||
constructor(name: string, args: VyosConfigTreeArgs, opts?: pulumi.CustomResourceOptions) {
|
||||
super(provider, name, {
|
||||
confirmMinutes: 5,
|
||||
save: true,
|
||||
...args,
|
||||
}, opts);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user