internaluploader: server + docs

One-way upload server: authentik SSO pairing (QR + app-initiated deep
link), hashed device tokens, streamed sha256-verified uploads into the
metube directory as <sha256>.<ext>.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

server/src/config.ts

@@ -0,0 +1,50 @@
+/** All configuration is environment variables; the deployment cites each name. */
+export interface Config {
+  port: number;
+  /** e.g. https://puploader.ad.itaz.eu — used for the OIDC redirect and QR payload. */
+  publicOrigin: string;
+  oidc: {
+    issuer: string;
+    clientId: string;
+    clientSecret: string;
+    requiredGroup: string;
+  };
+  cookieSecret: string;
+  /** Where finished files land — the metube download directory bind mount. */
+  downloadsDir: string;
+  /** Persistent state (device token store). A docker named volume. */
+  stateDir: string;
+  maxUploadBytes: number;
+  /** Deep-link scheme the Android app registers, for /pair/app. */
+  appScheme: string;
+  pairingCodeTtlSeconds: number;
+}
+
+function required(name: string): string {
+  const value = process.env[name];
+  if (!value) {
+    throw new Error(`Missing required environment variable ${name}`);
+  }
+  return value;
+}
+
+export function loadConfig(): Config {
+  return {
+    port: Number(process.env.PU_PORT ?? 8080),
+    publicOrigin: required("PU_PUBLIC_ORIGIN").replace(/\/$/, ""),
+    oidc: {
+      issuer: required("PU_OIDC_ISSUER"),
+      clientId: required("PU_OIDC_CLIENT_ID"),
+      clientSecret: required("PU_OIDC_CLIENT_SECRET"),
+      requiredGroup: required("PU_REQUIRED_GROUP"),
+    },
+    cookieSecret: required("PU_COOKIE_SECRET"),
+    downloadsDir: process.env.PU_DOWNLOADS_DIR ?? "/downloads",
+    stateDir: process.env.PU_STATE_DIR ?? "/state",
+    // 32 GiB default: large enough for any single video, small enough that a
+    // runaway client cannot fill the volume in one request unnoticed.
+    maxUploadBytes: Number(process.env.PU_MAX_UPLOAD_BYTES ?? 32 * 1024 ** 3),
+    appScheme: process.env.PU_APP_SCHEME ?? "internaluploader",
+    pairingCodeTtlSeconds: Number(process.env.PU_PAIRING_CODE_TTL_SECONDS ?? 300),
+  };
+}

server/src/index.ts

@@ -0,0 +1,178 @@
+import cookieSession from "cookie-session";
+import express, { type NextFunction, type Request, type Response } from "express";
+
+import { loadConfig } from "./config.ts";
+import { beginAuth, completeAuth, isAuthorised } from "./oidc.ts";
+import { deniedPage, pairingPage, qrPayload } from "./pages.ts";
+import { TokenStore } from "./tokens.ts";
+import { UploadDir, isValidObjectName } from "./upload.ts";
+
+const config = loadConfig();
+const tokens = new TokenStore(config.stateDir, config.pairingCodeTtlSeconds * 1000);
+const uploads = new UploadDir(config.downloadsDir, config.maxUploadBytes);
+
+const app = express();
+app.disable("x-powered-by");
+// TLS terminates at the Cilium ingress; without this `secure` cookies die.
+app.set("trust proxy", 1);
+
+app.use(
+  cookieSession({
+    name: "puploader",
+    secret: config.cookieSecret,
+    httpOnly: true,
+    sameSite: "lax",
+    secure: true,
+    maxAge: 8 * 60 * 60 * 1000,
+  }),
+);
+
+app.get("/healthz", (_req, res) => {
+  res.type("text/plain").send("ok");
+});
+
+/** Browser SSO gate: redirects to authentik when there is no session. */
+function requireBrowserSso(req: Request, res: Response, next: NextFunction): void {
+  const session = req.session as CookieSessionInterfaces.CookieSessionObject;
+  if (typeof session.username === "string") {
+    next();
+    return;
+  }
+  beginAuth(config)
+    .then((auth) => {
+      session.oidcState = auth.state;
+      session.oidcVerifier = auth.codeVerifier;
+      session.returnTo = req.path;
+      res.redirect(auth.url);
+    })
+    .catch(next);
+}
+
+app.get("/oidc/callback", (req, res, next) => {
+  const session = req.session as CookieSessionInterfaces.CookieSessionObject;
+  const state = session.oidcState;
+  const codeVerifier = session.oidcVerifier;
+  if (typeof state !== "string" || typeof codeVerifier !== "string") {
+    res.status(400).type("text/plain").send("no login in progress");
+    return;
+  }
+  completeAuth(config, req, { state, codeVerifier })
+    .then((identity) => {
+      delete session.oidcState;
+      delete session.oidcVerifier;
+      if (!isAuthorised(config, identity)) {
+        res.status(403).type("html").send(deniedPage(identity.username, config.oidc.requiredGroup));
+        return;
+      }
+      session.username = identity.username;
+      const returnTo = typeof session.returnTo === "string" ? session.returnTo : "/";
+      delete session.returnTo;
+      res.redirect(returnTo);
+    })
+    .catch(next);
+});
+
+app.get("/", requireBrowserSso, (req, res, next) => {
+  const session = req.session as CookieSessionInterfaces.CookieSessionObject;
+  const username = session.username as string;
+  const pairing = tokens.issuePairingCode(username);
+  pairingPage(username, qrPayload(config.publicOrigin, pairing.code), config.pairingCodeTtlSeconds)
+    .then((html) => res.type("html").send(html))
+    .catch(next);
+});
+
+/**
+ * App-initiated SSO: the app opens this in a Custom Tab; after the same
+ * browser login the pairing code goes back through the deep link. The app
+ * never sees OIDC — the phone's browser does, exactly like the QR path.
+ */
+app.get("/pair/app", requireBrowserSso, (req, res) => {
+  const session = req.session as CookieSessionInterfaces.CookieSessionObject;
+  const pairing = tokens.issuePairingCode(session.username as string);
+  res.redirect(`${config.appScheme}://pair?c=${pairing.code}`);
+});
+
+app.post("/api/pair", express.json({ limit: "4kb" }), (req, res) => {
+  const code = (req.body as { code?: unknown })?.code;
+  if (typeof code !== "string" || !/^[0-9a-f]{64}$/.test(code)) {
+    res.status(400).json({ error: "malformed pairing code" });
+    return;
+  }
+  const redeemed = tokens.redeemPairingCode(code);
+  if (!redeemed) {
+    res.status(410).json({ error: "pairing code expired or already used" });
+    return;
+  }
+  console.log(`paired device #${tokens.deviceCount} for ${redeemed.username}`);
+  res.json({ token: redeemed.token, name: redeemed.username });
+});
+
+/** Bearer gate for the upload API. */
+function requireDeviceToken(req: Request, res: Response, next: NextFunction): void {
+  const header = req.headers.authorization ?? "";
+  const token = header.startsWith("Bearer ") ? header.slice(7) : undefined;
+  const username = token ? tokens.authenticate(token) : undefined;
+  if (!username) {
+    res.status(401).json({ error: "invalid device token" });
+    return;
+  }
+  res.locals.username = username;
+  next();
+}
+
+app.head("/api/object/:name", requireDeviceToken, (req, res) => {
+  const name = req.params.name;
+  if (typeof name !== "string" || !isValidObjectName(name)) {
+    res.status(400).end();
+    return;
+  }
+  uploads
+    .exists(name)
+    .then((exists) => res.status(exists ? 204 : 404).end())
+    .catch(() => res.status(500).end());
+});
+
+app.put("/api/upload", requireDeviceToken, (req, res, next) => {
+  const name = req.query.name;
+  if (typeof name !== "string" || !isValidObjectName(name)) {
+    res.status(400).json({ error: "name must be <sha256 hex>.<extension>" });
+    return;
+  }
+  uploads
+    .receive(name, req)
+    .then((outcome) => {
+      switch (outcome.status) {
+        case "created":
+          console.log(`stored ${name} for ${res.locals.username}`);
+          res.status(201).json({ stored: name });
+          return;
+        case "exists":
+          res.status(200).json({ stored: name, existed: true });
+          return;
+        case "hash-mismatch":
+          res.status(409).json({ error: "content hash mismatch", actual: outcome.actual });
+          return;
+        case "too-large":
+          res.status(413).json({ error: "upload exceeds the configured maximum" });
+          return;
+      }
+    })
+    .catch(next);
+});
+
+// Uniform error surface; the details go to the log, not the client.
+app.use((error: unknown, _req: Request, res: Response, _next: NextFunction) => {
+  console.error(error);
+  if (!res.headersSent) {
+    res.status(500).json({ error: "internal error" });
+  }
+});
+
+async function main(): Promise<void> {
+  await uploads.init();
+  app.listen(config.port, () => {
+    console.log(`internaluploader-server listening on :${config.port}`);
+  });
+}
+
+void main();

server/src/oidc.ts

@@ -0,0 +1,106 @@
+import type { IncomingMessage } from "node:http";
+
+import { Issuer, generators, type Client } from "openid-client";
+
+import type { Config } from "./config.ts";
+
+let issuerPromise: Promise<Issuer<Client>> | undefined;
+
+/** Discovery, memoised — but NOT on failure, or one early miss poisons the pod. */
+export async function getIssuer(config: Config): Promise<Issuer<Client>> {
+  if (!issuerPromise) {
+    issuerPromise = Issuer.discover(config.oidc.issuer).catch((error: unknown) => {
+      issuerPromise = undefined;
+      throw error;
+    });
+  }
+  return issuerPromise;
+}
+
+export function redirectUri(config: Config): string {
+  return `${config.publicOrigin}/oidc/callback`;
+}
+
+async function clientFor(config: Config): Promise<Client> {
+  const issuer = await getIssuer(config);
+  return new issuer.Client({
+    client_id: config.oidc.clientId,
+    client_secret: config.oidc.clientSecret,
+    redirect_uris: [redirectUri(config)],
+    response_types: ["code"],
+  });
+}
+
+export interface AuthRequest {
+  url: string;
+  state: string;
+  codeVerifier: string;
+}
+
+export async function beginAuth(config: Config): Promise<AuthRequest> {
+  const client = await clientFor(config);
+  const state = generators.state();
+  const codeVerifier = generators.codeVerifier();
+
+  return {
+    // No `prompt: login` here, deliberately — unlike qr-login, an existing
+    // authentik session is exactly what we want to reuse: pairing is an
+    // authorization act, not a step-up authentication.
+    url: client.authorizationUrl({
+      scope: "openid email profile",
+      state,
+      code_challenge: generators.codeChallenge(codeVerifier),
+      code_challenge_method: "S256",
+    }),
+    state,
+    codeVerifier,
+  };
+}
+
+export interface Identity {
+  username: string;
+  groups: string[];
+}
+
+export async function completeAuth(
+  config: Config,
+  request: IncomingMessage,
+  expected: { state: string; codeVerifier: string },
+): Promise<Identity> {
+  const client = await clientFor(config);
+  const uri = redirectUri(config);
+  const tokenSet = await client.callback(uri, client.callbackParams(request), {
+    state: expected.state,
+    code_verifier: expected.codeVerifier,
+  });
+
+  const claims = tokenSet.claims();
+  const groups = Array.isArray(claims.groups)
+    ? claims.groups.filter((g): g is string => typeof g === "string")
+    : undefined;
+
+  // Fail closed, and say which failure it is. An absent claim means the
+  // provider is missing its scope mapping; treating that as "no groups" would
+  // lock everyone out silently, and as "allow" would let everyone in silently.
+  if (groups === undefined) {
+    throw new Error(
+      "ID token carries no `groups` claim — the authentik provider is missing " +
+        "the profile scope mapping, so membership cannot be checked",
+    );
+  }
+
+  const username =
+    typeof claims.preferred_username === "string" ? claims.preferred_username : undefined;
+  if (!username) {
+    throw new Error(
+      "ID token carries no `preferred_username` claim — uploads could not be " +
+        "attributed to a user",
+    );
+  }
+
+  return { username, groups };
+}
+
+export function isAuthorised(config: Config, identity: Identity): boolean {
+  return identity.groups.includes(config.oidc.requiredGroup);
+}

server/src/pages.ts

@@ -0,0 +1,73 @@
+import { toDataURL } from "qrcode";
+
+export interface QrPayload {
+  v: 1;
+  u: string;
+  c: string;
+}
+
+export function qrPayload(publicOrigin: string, code: string): string {
+  const payload: QrPayload = { v: 1, u: publicOrigin, c: code };
+  return JSON.stringify(payload);
+}
+
+/**
+ * The whole UI: one page with a QR code. Deliberately says nothing about
+ * where files go or how to reach the server — the QR carries everything the
+ * app needs, and this page is already behind SSO.
+ */
+export async function pairingPage(
+  username: string,
+  payload: string,
+  ttlSeconds: number,
+): Promise<string> {
+  const qrDataUri = await toDataURL(payload, { margin: 1, width: 320 });
+  return `<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<meta name="robots" content="noindex">
+<title>InternalUploader — pair a device</title>
+<style>
+  :root { color-scheme: light dark; }
+  body {
+    font-family: system-ui, sans-serif;
+    display: flex; flex-direction: column; align-items: center;
+    justify-content: center; min-height: 100vh; margin: 0; gap: 1rem;
+    background: Canvas; color: CanvasText; text-align: center; padding: 0 1rem;
+  }
+  img { border-radius: 8px; background: white; padding: 8px; }
+  small { opacity: .6 }
+  a.button {
+    display: inline-block; padding: .5rem 1rem; border-radius: 6px;
+    border: 1px solid currentColor; text-decoration: none; color: inherit;
+  }
+</style>
+</head>
+<body>
+  <h1>Pair a device</h1>
+  <p>Signed in as <strong>${escapeHtml(username)}</strong>. Scan with the InternalUploader app.</p>
+  <img src="${qrDataUri}" alt="pairing QR code" width="320" height="320">
+  <small>Code is single-use and expires in ${Math.round(ttlSeconds / 60)} minutes.</small>
+  <a class="button" href="/">New code</a>
+</body>
+</html>`;
+}
+
+export function deniedPage(username: string, group: string): string {
+  return `<!doctype html>
+<html lang="en"><head><meta charset="utf-8"><title>Not authorised</title></head>
+<body style="font-family: system-ui, sans-serif; text-align: center; margin-top: 4rem">
+<h1>Not authorised</h1>
+<p><strong>${escapeHtml(username)}</strong> is not in the <strong>${escapeHtml(group)}</strong> group.</p>
+</body></html>`;
+}
+
+function escapeHtml(value: string): string {
+  return value
+    .replaceAll("&", "&amp;")
+    .replaceAll("<", "&lt;")
+    .replaceAll(">", "&gt;")
+    .replaceAll('"', "&quot;");
+}

server/src/tokens.ts

@@ -0,0 +1,126 @@
+import { createHash, randomBytes, timingSafeEqual } from "node:crypto";
+import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+/**
+ * Pairing codes and device tokens.
+ *
+ * Pairing codes are IN MEMORY on purpose: they live five minutes and are spent
+ * once, so surviving a restart buys nothing — and a restart invalidating any
+ * outstanding QR is the safe direction.
+ *
+ * Device tokens persist, HASHED. The stored file never contains anything a
+ * reader could replay; the plaintext token exists only in the pairing response
+ * and on the phone.
+ */
+
+export interface PairingCode {
+  code: string;
+  username: string;
+  expiresAt: number;
+}
+
+export interface DeviceRecord {
+  /** sha256 hex of the bearer token. */
+  tokenHash: string;
+  username: string;
+  createdAt: string;
+  /** Coarse (per-day) so routine uploads do not rewrite the file constantly. */
+  lastUsedDay: string;
+}
+
+function sha256Hex(value: string): string {
+  return createHash("sha256").update(value).digest("hex");
+}
+
+function today(): string {
+  return new Date().toISOString().slice(0, 10);
+}
+
+export class TokenStore {
+  private readonly file: string;
+  private readonly tmp: string;
+  private readonly pairingTtlMs: number;
+  private devices: DeviceRecord[] = [];
+  private pending = new Map<string, PairingCode>();
+
+  // No parameter properties: `node --test` runs this file under strip-only
+  // type stripping, which rejects them.
+  constructor(stateDir: string, pairingTtlMs: number) {
+    this.pairingTtlMs = pairingTtlMs;
+    mkdirSync(stateDir, { recursive: true });
+    this.file = join(stateDir, "devices.json");
+    this.tmp = join(stateDir, "devices.json.tmp");
+    try {
+      this.devices = JSON.parse(readFileSync(this.file, "utf8")) as DeviceRecord[];
+    } catch {
+      this.devices = [];
+    }
+  }
+
+  private persist(): void {
+    // Write-then-rename so a crash mid-write cannot truncate the live store.
+    writeFileSync(this.tmp, JSON.stringify(this.devices, null, 2));
+    renameSync(this.tmp, this.file);
+  }
+
+  /** Mint a single-use pairing code for a browser-authenticated user. */
+  issuePairingCode(username: string): PairingCode {
+    this.sweep();
+    const entry: PairingCode = {
+      code: randomBytes(32).toString("hex"),
+      username,
+      expiresAt: Date.now() + this.pairingTtlMs,
+    };
+    this.pending.set(entry.code, entry);
+    return entry;
+  }
+
+  /** Exchange a pairing code for a device token. Single use, TTL enforced. */
+  redeemPairingCode(code: string): { token: string; username: string } | undefined {
+    this.sweep();
+    const entry = this.pending.get(code);
+    if (!entry) {
+      return undefined;
+    }
+    this.pending.delete(entry.code);
+    const token = randomBytes(32).toString("base64url");
+    this.devices.push({
+      tokenHash: sha256Hex(token),
+      username: entry.username,
+      createdAt: new Date().toISOString(),
+      lastUsedDay: today(),
+    });
+    this.persist();
+    return { token, username: entry.username };
+  }
+
+  /** Bearer-token check. Returns the owning username, or undefined. */
+  authenticate(token: string): string | undefined {
+    const hash = Buffer.from(sha256Hex(token), "hex");
+    for (const device of this.devices) {
+      const stored = Buffer.from(device.tokenHash, "hex");
+      if (stored.length === hash.length && timingSafeEqual(stored, hash)) {
+        if (device.lastUsedDay !== today()) {
+          device.lastUsedDay = today();
+          this.persist();
+        }
+        return device.username;
+      }
+    }
+    return undefined;
+  }
+
+  get deviceCount(): number {
+    return this.devices.length;
+  }
+
+  private sweep(): void {
+    const now = Date.now();
+    for (const [code, entry] of this.pending) {
+      if (entry.expiresAt <= now) {
+        this.pending.delete(code);
+      }
+    }
+  }
+}

server/src/upload.ts

@@ -0,0 +1,118 @@
+import { createHash, randomUUID } from "node:crypto";
+import { createWriteStream } from "node:fs";
+import { mkdir, rename, stat, unlink } from "node:fs/promises";
+import { join } from "node:path";
+import type { Readable } from "node:stream";
+
+/**
+ * `<64 hex sha256>` plus an optional short, lowercase, alphanumeric extension.
+ * Everything else is rejected before any filesystem call — the name is the only
+ * client-controlled thing that ever reaches a path.
+ */
+const NAME_PATTERN = /^[0-9a-f]{64}(\.[a-z0-9]{1,16})?$/;
+
+export function isValidObjectName(name: string): boolean {
+  return NAME_PATTERN.test(name);
+}
+
+export function hashOf(name: string): string {
+  return name.slice(0, 64);
+}
+
+export type UploadOutcome =
+  | { status: "created" }
+  | { status: "exists" }
+  | { status: "hash-mismatch"; actual: string }
+  | { status: "too-large" };
+
+export class UploadDir {
+  private readonly downloadsDir: string;
+  private readonly maxBytes: number;
+  private readonly tmpDir: string;
+
+  // No parameter properties: `node --test` runs this file under strip-only
+  // type stripping, which rejects them.
+  constructor(downloadsDir: string, maxBytes: number) {
+    this.downloadsDir = downloadsDir;
+    this.maxBytes = maxBytes;
+    // Same filesystem as the destination, or the final rename stops being
+    // atomic and a half-written file could be seen with its final name.
+    this.tmpDir = join(downloadsDir, ".internaluploader-tmp");
+  }
+
+  async init(): Promise<void> {
+    await mkdir(this.tmpDir, { recursive: true });
+  }
+
+  async exists(name: string): Promise<boolean> {
+    try {
+      await stat(join(this.downloadsDir, name));
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Stream a request body to disk, verify its sha256 matches the declared
+   * name, and only then rename it into the downloads directory. Idempotent:
+   * a name that already exists is reported without touching anything.
+   */
+  async receive(name: string, body: Readable): Promise<UploadOutcome> {
+    if (await this.exists(name)) {
+      return { status: "exists" };
+    }
+
+    const tmpPath = join(this.tmpDir, randomUUID());
+    const hash = createHash("sha256");
+    let received = 0;
+
+    try {
+      await new Promise<void>((resolve, reject) => {
+        const out = createWriteStream(tmpPath, { flags: "wx", mode: 0o644 });
+        body.on("data", (chunk: Buffer) => {
+          received += chunk.length;
+          if (received > this.maxBytes) {
+            body.destroy();
+            out.destroy();
+            reject(new TooLargeError());
+            return;
+          }
+          hash.update(chunk);
+        });
+        body.on("error", reject);
+        out.on("error", reject);
+        out.on("finish", resolve);
+        body.pipe(out);
+      });
+
+      const actual = hash.digest("hex");
+      if (actual !== hashOf(name)) {
+        await unlink(tmpPath);
+        return { status: "hash-mismatch", actual };
+      }
+
+      // Lost race with a concurrent identical upload: same content, so the
+      // file already on disk is exactly what this request carried.
+      if (await this.exists(name)) {
+        await unlink(tmpPath);
+        return { status: "exists" };
+      }
+
+      await rename(tmpPath, join(this.downloadsDir, name));
+      return { status: "created" };
+    } catch (error) {
+      await unlink(tmpPath).catch(() => {});
+      if (error instanceof TooLargeError) {
+        return { status: "too-large" };
+      }
+      throw error;
+    }
+  }
+}
+
+class TooLargeError extends Error {
+  constructor() {
+    super("upload exceeds the configured maximum");
+  }
+}