From b0233918ffa6bf3913f71cdccea1302a55a7b763 Mon Sep 17 00:00:00 2001 From: Michal Date: Sun, 9 Aug 2026 18:17:32 +0100 Subject: [PATCH] feat(servers): persistent volumes + self-hosted web search and docs templates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instances are immutable and get recreated on any server edit, so anything an MCP server wrote to its container filesystem was lost at exactly that point. That ruled out every stateful MCP server, docs-mcp among them: its index is a SQLite file (better-sqlite3 + sqlite-vec) and it has no external-database mode, so no amount of Postgres helps. A server or template can now declare volumes. The backing store is keyed on the server, not the instance — `mcpctl--` — which is the whole point: an instance-scoped claim would be destroyed precisely when the data needs to survive. On Kubernetes that is a PVC ensured in the servers namespace before the pod is created and never deleted with it; on Docker, a named volume (named, not anonymous, so `removeContainer`'s `v: true` leaves it alone). Claims are ReadWriteOnce, so volumes and replicas > 1 are mutually exclusive; validation rejects that pair instead of leaving the extra replicas unschedulable. storageClassName is omitted rather than sent empty when no class is configured — to Kubernetes those mean different things. Also fixes a pre-existing bug in the same path: seedTemplates dropped `runtime`, so every PyPI-backed template seeded from YAML silently defaulted to node and would run `npx` against a package that only exists on PyPI. `unifi-network` declares `runtime: python` and had been seeding with runtime unset. Templates added, all self-hosted and none needing an API key: - duckduckgo — no backing service at all - searxng — needs a SearXNG engine (compose profile in stack/) - docs-mcp — open-source Context7/Ref alternative, uses the new volume Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017BMXdb2qZbPSh8Q7XpTyjB --- README.md | 56 +++++++ completions/mcpctl.bash | 2 +- completions/mcpctl.fish | 1 + docs/web-search.md | 146 ++++++++++++++++++ src/cli/src/commands/apply.ts | 9 ++ src/cli/src/commands/create.ts | 61 ++++++++ src/cli/src/commands/describe.ts | 16 ++ src/cli/tests/commands/create.test.ts | 30 +++- .../migration.sql | 14 ++ src/db/prisma/schema.prisma | 2 + src/db/src/seed/index.ts | 18 +++ .../src/repositories/mcp-server.repository.ts | 2 + .../src/repositories/template.repository.ts | 2 + .../src/services/backup/backup-service.ts | 2 + .../src/services/backup/restore-service.ts | 5 + .../src/services/docker/container-manager.ts | 7 + src/mcpd/src/services/instance.service.ts | 20 +++ .../services/k8s/kubernetes-orchestrator.ts | 48 +++++- .../src/services/k8s/manifest-generator.ts | 77 ++++++++- src/mcpd/src/services/orchestrator.ts | 26 ++++ src/mcpd/src/validation/mcp-server.schema.ts | 14 +- src/mcpd/src/validation/template.schema.ts | 16 ++ src/mcpd/tests/instance-service.test.ts | 50 +++++- src/mcpd/tests/k8s-manifest.test.ts | 65 ++++++++ stack/.env.example | 5 + stack/docker-compose.yml | 34 ++++ stack/searxng/settings.yml | 14 ++ templates/docs-mcp.yaml | 39 +++++ templates/duckduckgo.yaml | 24 +++ templates/searxng.yaml | 25 +++ 30 files changed, 822 insertions(+), 8 deletions(-) create mode 100644 docs/web-search.md create mode 100644 src/db/prisma/migrations/20260809120000_add_server_volumes/migration.sql create mode 100644 stack/searxng/settings.yml create mode 100644 templates/docs-mcp.yaml create mode 100644 templates/duckduckgo.yaml create mode 100644 templates/searxng.yaml diff --git a/README.md b/README.md index 3f7f839..90ccf47 100644 --- a/README.md +++ b/README.md @@ -743,6 +743,62 @@ mcpctl create server my-ha \ --env-from-secret ha-secrets ``` +### Persistent volumes + +Instances are immutable — any server edit recreates them — so by default +anything an MCP server writes to its container filesystem is lost at that point. +A server (or template) can declare volumes that outlive its instances: + +```bash +mcpctl create server docs \ + --from-template docs-mcp \ + --volume data:/data:20:longhorn # NAME:/mount/path[:SIZE_GB[:STORAGE_CLASS]] +``` + +```yaml +# ...or declaratively +volumes: + - name: data + mountPath: /data + sizeGb: 20 + storageClass: longhorn +``` + +The backing store is named after the **server**, not the instance — +`mcpctl--` — which is what lets the data survive. On Kubernetes +that is a PVC created in the servers namespace on first start and left in place +afterwards; on Docker it is a named volume. Neither is deleted when an instance +or the server goes away, so reclaiming the space is a deliberate +`kubectl delete pvc` / `docker volume rm`. + +Notes: + +- Claims are `ReadWriteOnce`, so a server with volumes is limited to one + replica. Asking for more is rejected at validation rather than leaving the + extra replicas unschedulable. +- `storageClass` defaults to `MCPD_VOLUME_STORAGE_CLASS`, and is omitted + entirely when neither is set. **Set it explicitly on any cluster with more + than one default StorageClass**, where an omitted class binds + nondeterministically. +- Growing a volume is an explicit `kubectl edit pvc` (the class must allow + expansion). mcpctl never resizes an existing claim, because most PVC fields + are immutable after binding. +- Backups capture the volume *declaration*, not its contents. + +### Web search and docs lookup + +The `duckduckgo`, `searxng` and `docs-mcp` templates give an agent web search and +version-pinned library documentation. All are self-hosted and none needs an API key. +`duckduckgo` needs no backing service at all: + +```bash +mcpctl create server websearch --from-template duckduckgo +``` + +See [docs/web-search.md](docs/web-search.md) for the SearXNG engine setup (including +the `json` format setting, without which every search silently returns nothing) and +the `docs-mcp` index-persistence caveat. + ## Gated Sessions Projects using the `default` or `gate` plugin are **gated**. When Claude connects to a gated project: diff --git a/completions/mcpctl.bash b/completions/mcpctl.bash index 8dbfce5..814d165 100644 --- a/completions/mcpctl.bash +++ b/completions/mcpctl.bash @@ -188,7 +188,7 @@ _mcpctl() { else case "$create_sub" in server) - COMPREPLY=($(compgen -W "-d --description --package-name --runtime --docker-image --transport --repository-url --external-url --command --container-port --replicas --env --from-template --env-from-secret --force -h --help" -- "$cur")) + COMPREPLY=($(compgen -W "-d --description --package-name --runtime --docker-image --transport --repository-url --external-url --command --container-port --replicas --env --volume --from-template --env-from-secret --force -h --help" -- "$cur")) ;; secret) COMPREPLY=($(compgen -W "--data --force -h --help" -- "$cur")) diff --git a/completions/mcpctl.fish b/completions/mcpctl.fish index b4b8da1..cceb9ea 100644 --- a/completions/mcpctl.fish +++ b/completions/mcpctl.fish @@ -357,6 +357,7 @@ complete -c mcpctl -n "__mcpctl_subcmd_active create server" -l command -d 'Comm complete -c mcpctl -n "__mcpctl_subcmd_active create server" -l container-port -d 'Container port number' -x complete -c mcpctl -n "__mcpctl_subcmd_active create server" -l replicas -d 'Number of replicas' -x complete -c mcpctl -n "__mcpctl_subcmd_active create server" -l env -d 'Env var: KEY=value (inline) or KEY=secretRef:SECRET:KEY (secret ref, repeat for multiple)' -x +complete -c mcpctl -n "__mcpctl_subcmd_active create server" -l volume -d 'Persistent volume: NAME:/mount/path[:SIZE_GB[:STORAGE_CLASS]] (repeat for multiple)' -x complete -c mcpctl -n "__mcpctl_subcmd_active create server" -l from-template -d 'Create from template (name or name:version)' -x complete -c mcpctl -n "__mcpctl_subcmd_active create server" -l env-from-secret -d 'Map template env vars from a secret' -x complete -c mcpctl -n "__mcpctl_subcmd_active create server" -l force -d 'Update if already exists' diff --git a/docs/web-search.md b/docs/web-search.md new file mode 100644 index 0000000..af8c389 --- /dev/null +++ b/docs/web-search.md @@ -0,0 +1,146 @@ +# Web search and documentation lookup + +Three templates. All self-hosted, none needs an API key or a vendor account, and +all three are deployed by mcpctl like any other server. + +| Template | Package / image | Needs | +|---|---|---| +| `duckduckgo` | `duckduckgo-mcp-server` (python) | nothing | +| `searxng` | `mcp-searxng` (node) | a SearXNG instance | +| `docs-mcp` | `ghcr.io/arabold/docs-mcp-server` | nothing (see the persistence caveat) | + +Search and docs are different jobs, not competing options. A search engine will +hand you a 2023 blog post with a stale method signature; a docs index cannot +tell you why a daemon is crash-looping. Attach both to a project that does real +engineering work. + +`docs-mcp` is the open-source replacement for Context7 / Ref.tools — same job, +but the index lives on your infrastructure and can include private repos. + +## Start here: `duckduckgo` + +The only one with no infrastructure behind it. It scrapes DuckDuckGo's HTML +endpoint directly, so there is no engine to run and no key to hold. + +```bash +mcpctl create server websearch --from-template duckduckgo --env DDG_SAFE_SEARCH=OFF +mcpctl get instances | grep websearch # RUNNING / healthy within ~a minute +``` + +Tools: `search` (`query`, `max_results`, `region`) and `fetch_content` (`url`, +`start_index`, `max_length`) for pulling a result as markdown. + +The tradeoff is honest: scraping has no SLA. DuckDuckGo can change its markup or +rate-limit you, and the server caps itself at 30 searches/min. Set +`DDG_SEARCH_BACKEND=curl` if bot checks start biting. When it becomes a problem, +move to `searxng`. + +## Better results: `searxng` + +SearXNG aggregates ~25 engines and gives you real filtering, which is what keeps +search results from flooding a context window: + +- `min_score` (0.0–1.0) — relevance floor. The most useful knob; start at `0.3`. +- `time_range` — `day` / `week` / `month` / `year`. Essential for "did this break + in the last release" questions. +- `language`, `safesearch`, `pageno`. + +`web_url_read` then takes `section`, `paragraphRange`, `startChar`/`maxLength` +and `readHeadings`, so you pull one section of a long page rather than all of it. +Bounded PDF extraction is included, which covers most vendor documentation. + +The cost is that SearXNG is a service you have to run. It is plain +infrastructure, not an MCP server, so mcpctl has nothing to manage it with — it +belongs in your cluster or in the compose stack: + +```bash +cd stack +cp .env.example .env # set SEARXNG_SECRET +docker compose --profile websearch up -d +``` + +Then point the MCP server at it: + +```bash +mcpctl create secret searxng-conf --data SEARXNG_URL=http://mcpctl-searxng:8080 +mcpctl create server searxng --from-template searxng --env-from-secret searxng-conf +``` + +### The one SearXNG gotcha + +SearXNG ships with `search.formats: [html]`. Every `format=json` request against +a stock instance returns **403**, and `mcp-searxng` comes back empty with no +useful error. `stack/searxng/settings.yml` exists only to add `json` to that +list — it is the one setting with no environment-variable override. + +This is also why pointing the template at a *public* SearXNG instance usually +fails: nearly all of them leave the JSON API off. + +The compose healthcheck probes `format=json` specifically, so a misconfigured +instance shows up as unhealthy rather than as silently empty search results. + +## Documentation: `docs-mcp` + +```bash +mcpctl create server docs --from-template docs-mcp +``` + +Tools: `scrape_docs` (index a site, GitHub repo, npm or PyPI package, or local +files), `search_docs` (query, optionally pinned to a version), and `fetch_url`. + +Index the things you actually run — Pulumi, the Kubernetes API, Grafana, the +Terraform provider docs — rather than everything. + +### Persistence + +The scraped index is a SQLite file (`better-sqlite3` + `sqlite-vec`) under +`/data`. There is **no external-database mode** — no `DATABASE_URL`, no +pgvector — so a Postgres cluster cannot help here. The template declares a +volume instead: + +```yaml +volumes: + - name: data + mountPath: /data + sizeGb: 20 +``` + +The backing PVC is named after the *server* (`mcpctl-docs-data`), not the +instance, so editing the server or restarting the pod re-attaches to the same +index rather than starting empty. See "Persistent volumes" in the README. + +The template carries no `healthCheck`, because `search_docs` needs a library +argument that only exists after a scrape — a synthetic probe would report +unhealthy on a fresh instance. + +### Embeddings + +Keyword search works out of the box. Semantic search is noticeably better and +needs an embedding model. To keep it local, point the server at Ollama: + +```bash +ollama pull nomic-embed-text +mcpctl create server docs --from-template docs-mcp \ + --env DOCS_MCP_EMBEDDING_MODEL=openai:nomic-embed-text \ + --env OPENAI_API_BASE=http://ollama:11434/v1 \ + --env OPENAI_API_KEY=ollama +``` + +Ollama's OpenAI-compatible endpoint is why the provider prefix is `openai:` — +the key is a placeholder and never leaves the network. + +Changing the embedding model invalidates the index: embeddings from different +models are not comparable, so everything has to be re-scraped. + +## Alternatives considered + +- **Tavily / Brave / Exa / Perplexity** — better formatted results, all require + an API key and send every query to a vendor. +- **Context7** — the popular docs MCP, but a cloud index of public-docs snippets + only. `docs-mcp` covers the same ground locally and takes private sources. +- **agent-search** — bundles SearXNG and a 10-strategy extraction cascade in one + deploy. Attractive, but at ~67 stars it is too young to build the stack on. + Worth revisiting. +- **Writing our own** — the hard parts are extraction heuristics and the + embedding pipeline, and all three projects above already solved them under + MIT. diff --git a/src/cli/src/commands/apply.ts b/src/cli/src/commands/apply.ts index 5360a1a..c4a32a6 100644 --- a/src/cli/src/commands/apply.ts +++ b/src/cli/src/commands/apply.ts @@ -20,6 +20,13 @@ const ServerEnvEntrySchema = z.object({ }).optional(), }); +const VolumeSpecSchema = z.object({ + name: z.string().min(1).max(50).regex(/^[a-z0-9-]+$/), + mountPath: z.string().min(1).regex(/^\//, 'mountPath must be absolute'), + sizeGb: z.number().int().min(1).max(1024).default(10), + storageClass: z.string().optional(), +}); + const ServerSpecSchema = z.object({ name: z.string().min(1), description: z.string().default(''), @@ -34,6 +41,7 @@ const ServerSpecSchema = z.object({ replicas: z.number().int().min(0).max(10).default(1), env: z.array(ServerEnvEntrySchema).default([]), healthCheck: HealthCheckSchema.optional(), + volumes: z.array(VolumeSpecSchema).default([]), }); const SecretSpecSchema = z.object({ @@ -125,6 +133,7 @@ const TemplateSpecSchema = z.object({ replicas: z.number().int().min(0).max(10).default(1), env: z.array(TemplateEnvEntrySchema).default([]), healthCheck: HealthCheckSchema.optional(), + volumes: z.array(VolumeSpecSchema).default([]), }); const UserSpecSchema = z.object({ diff --git a/src/cli/src/commands/create.ts b/src/cli/src/commands/create.ts index 6f64801..1b252fe 100644 --- a/src/cli/src/commands/create.ts +++ b/src/cli/src/commands/create.ts @@ -103,6 +103,53 @@ function parseServerEnv(entries: string[]): ServerEnvEntry[] { }); } +export interface ServerVolume { + name: string; + mountPath: string; + sizeGb: number; + storageClass?: string; +} + +/** + * Parse `NAME:/mount/path[:SIZE_GB[:STORAGE_CLASS]]`. + * + * Split on ':' rather than a regex so a Windows-style or otherwise odd mount + * path fails loudly on the leading-slash check instead of being silently + * mis-parsed into the wrong field. + */ +export function parseServerVolumes(entries: string[]): ServerVolume[] { + return entries.map((entry) => { + const parts = entry.split(':'); + if (parts.length < 2 || parts.length > 4) { + throw new Error( + `Invalid volume format '${entry}'. Expected NAME:/mount/path[:SIZE_GB[:STORAGE_CLASS]]`, + ); + } + const [name, mountPath, sizeRaw, storageClass] = parts; + if (name === undefined || name === '') { + throw new Error(`Invalid volume '${entry}': name is empty`); + } + if (mountPath === undefined || !mountPath.startsWith('/')) { + throw new Error(`Invalid volume '${entry}': mount path must be absolute (start with '/')`); + } + + let sizeGb = 10; + if (sizeRaw !== undefined && sizeRaw !== '') { + sizeGb = Number(sizeRaw); + if (!Number.isInteger(sizeGb) || sizeGb < 1) { + throw new Error(`Invalid volume '${entry}': size must be a positive whole number of GiB`); + } + } + + return { + name, + mountPath, + sizeGb, + ...(storageClass !== undefined && storageClass !== '' ? { storageClass } : {}), + }; + }); +} + function parseEnvEntries(entries: string[]): Record { const result: Record = {}; for (const entry of entries) { @@ -136,6 +183,7 @@ export function createCreateCommand(deps: CreateCommandDeps): Command { .option('--container-port ', 'Container port number') .option('--replicas ', 'Number of replicas') .option('--env ', 'Env var: KEY=value (inline) or KEY=secretRef:SECRET:KEY (secret ref, repeat for multiple)', collect, []) + .option('--volume ', 'Persistent volume: NAME:/mount/path[:SIZE_GB[:STORAGE_CLASS]] (repeat for multiple)', collect, []) .option('--from-template ', 'Create from template (name or name:version)') .option('--env-from-secret ', 'Map template env vars from a secret') .option('--force', 'Update if already exists') @@ -233,6 +281,19 @@ export function createCreateCommand(deps: CreateCommandDeps): Command { } body.env = merged; } + if (opts.volume.length > 0) { + // Merge by volume name, same as --env, so a CLI volume can retarget one + // the template declared without duplicating the mount. + const cliVolumes = parseServerVolumes(opts.volume as string[]); + const existing = (body.volumes as ServerVolume[] | undefined) ?? []; + const merged = [...existing]; + for (const vol of cliVolumes) { + const idx = merged.findIndex((v) => v.name === vol.name); + if (idx >= 0) merged[idx] = vol; + else merged.push(vol); + } + body.volumes = merged; + } // Defaults when no template if (!opts.fromTemplate) { diff --git a/src/cli/src/commands/describe.ts b/src/cli/src/commands/describe.ts index bda4ffb..7bd6a62 100644 --- a/src/cli/src/commands/describe.ts +++ b/src/cli/src/commands/describe.ts @@ -51,6 +51,22 @@ function formatServerDetail(server: Record): string { } } + const volumes = server.volumes as Array<{ name: string; mountPath: string; sizeGb?: number; storageClass?: string }> | undefined; + if (volumes && volumes.length > 0) { + lines.push(''); + lines.push('Volumes:'); + const volW = Math.max(6, ...volumes.map((v) => v.name.length)) + 2; + lines.push(` ${'NAME'.padEnd(volW)}MOUNT PATH${' '.repeat(12)}SIZE CLAIM`); + for (const v of volumes) { + const size = `${v.sizeGb ?? 10}Gi`; + const cls = v.storageClass !== undefined && v.storageClass !== '' ? ` (${v.storageClass})` : ''; + lines.push( + ` ${v.name.padEnd(volW)}${pad(v.mountPath, 22)}${pad(size, 7)}` + + `mcpctl-${server.name as string}-${v.name}${cls}`, + ); + } + } + const hc = server.healthCheck as { tool: string; arguments?: Record; intervalSeconds?: number; timeoutSeconds?: number; failureThreshold?: number } | null; if (hc) { lines.push(''); diff --git a/src/cli/tests/commands/create.test.ts b/src/cli/tests/commands/create.test.ts index d1d2f32..2840276 100644 --- a/src/cli/tests/commands/create.test.ts +++ b/src/cli/tests/commands/create.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; -import { createCreateCommand } from '../../src/commands/create.js'; +import { createCreateCommand, parseServerVolumes } from '../../src/commands/create.js'; import { type ApiClient, ApiError } from '../../src/api-client.js'; function mockClient(): ApiClient { @@ -556,3 +556,31 @@ describe('create command', () => { }); }); }); + +describe('parseServerVolumes', () => { + it('parses name, path, size and storage class', () => { + expect(parseServerVolumes(['data:/data:20:longhorn'])).toEqual([ + { name: 'data', mountPath: '/data', sizeGb: 20, storageClass: 'longhorn' }, + ]); + }); + + it('defaults size to 10Gi and omits storageClass', () => { + expect(parseServerVolumes(['data:/data'])).toEqual([ + { name: 'data', mountPath: '/data', sizeGb: 10 }, + ]); + }); + + it('rejects a relative mount path', () => { + expect(() => parseServerVolumes(['data:data'])).toThrow(/must be absolute/); + }); + + it('rejects a non-integer or zero size', () => { + expect(() => parseServerVolumes(['data:/data:abc'])).toThrow(/positive whole number/); + expect(() => parseServerVolumes(['data:/data:0'])).toThrow(/positive whole number/); + }); + + it('rejects a malformed spec', () => { + expect(() => parseServerVolumes(['data'])).toThrow(/Invalid volume format/); + expect(() => parseServerVolumes(['a:/b:1:c:d'])).toThrow(/Invalid volume format/); + }); +}); diff --git a/src/db/prisma/migrations/20260809120000_add_server_volumes/migration.sql b/src/db/prisma/migrations/20260809120000_add_server_volumes/migration.sql new file mode 100644 index 0000000..c87371a --- /dev/null +++ b/src/db/prisma/migrations/20260809120000_add_server_volumes/migration.sql @@ -0,0 +1,14 @@ +-- Persistent volumes for MCP servers. +-- +-- Instances are immutable and get recreated on any server edit, so anything an +-- MCP server writes to its container filesystem was previously lost at that +-- point. A volume is declared on the *server* and backed by a Docker named +-- volume or a Kubernetes PVC whose identity is derived from the server name, +-- so it outlives the instances that mount it. +-- +-- Shape: [{ "name": "data", "mountPath": "/data", "sizeGb": 10, +-- "storageClass": "longhorn" }] +-- +-- Defaults to an empty array, so existing rows keep today's behaviour. +ALTER TABLE "McpServer" ADD COLUMN "volumes" JSONB NOT NULL DEFAULT '[]'; +ALTER TABLE "McpTemplate" ADD COLUMN "volumes" JSONB NOT NULL DEFAULT '[]'; diff --git a/src/db/prisma/schema.prisma b/src/db/prisma/schema.prisma index b8de194..7b71bc8 100644 --- a/src/db/prisma/schema.prisma +++ b/src/db/prisma/schema.prisma @@ -70,6 +70,7 @@ model McpServer { replicas Int @default(1) env Json @default("[]") healthCheck Json? + volumes Json @default("[]") version Int @default(1) createdAt DateTime @default(now()) updatedAt DateTime @updatedAt @@ -107,6 +108,7 @@ model McpTemplate { replicas Int @default(1) env Json @default("[]") healthCheck Json? + volumes Json @default("[]") createdAt DateTime @default(now()) updatedAt DateTime @updatedAt diff --git a/src/db/src/seed/index.ts b/src/db/src/seed/index.ts index e8872e3..50e8ea7 100644 --- a/src/db/src/seed/index.ts +++ b/src/db/src/seed/index.ts @@ -16,11 +16,24 @@ export interface HealthCheckSpec { failureThreshold?: number; } +export interface SeedVolumeSpec { + name: string; + mountPath: string; + sizeGb?: number; + storageClass?: string; +} + export interface SeedTemplate { name: string; version: string; description: string; packageName?: string; + /** + * Package runtime (node, python, ...). Selects the runner image and the + * spawn command — `uvx` vs `npx`. Dropping it silently defaults the template + * to node, which makes every PyPI-backed template fail at first start. + */ + runtime?: string; dockerImage?: string; transport: 'STDIO' | 'SSE' | 'STREAMABLE_HTTP'; repositoryUrl?: string; @@ -30,6 +43,7 @@ export interface SeedTemplate { replicas?: number; env?: TemplateEnvEntry[]; healthCheck?: HealthCheckSpec; + volumes?: SeedVolumeSpec[]; } export async function seedTemplates( @@ -45,6 +59,7 @@ export async function seedTemplates( version: tpl.version, description: tpl.description, packageName: tpl.packageName ?? null, + runtime: tpl.runtime ?? null, dockerImage: tpl.dockerImage ?? null, transport: tpl.transport, repositoryUrl: tpl.repositoryUrl ?? null, @@ -54,12 +69,14 @@ export async function seedTemplates( replicas: tpl.replicas ?? 1, env: (tpl.env ?? []) as unknown as Prisma.InputJsonValue, healthCheck: (tpl.healthCheck ?? Prisma.JsonNull) as unknown as Prisma.InputJsonValue, + volumes: (tpl.volumes ?? []) as unknown as Prisma.InputJsonValue, }, create: { name: tpl.name, version: tpl.version, description: tpl.description, packageName: tpl.packageName ?? null, + runtime: tpl.runtime ?? null, dockerImage: tpl.dockerImage ?? null, transport: tpl.transport, repositoryUrl: tpl.repositoryUrl ?? null, @@ -69,6 +86,7 @@ export async function seedTemplates( replicas: tpl.replicas ?? 1, env: (tpl.env ?? []) as unknown as Prisma.InputJsonValue, healthCheck: (tpl.healthCheck ?? Prisma.JsonNull) as unknown as Prisma.InputJsonValue, + volumes: (tpl.volumes ?? []) as unknown as Prisma.InputJsonValue, }, }); upserted++; diff --git a/src/mcpd/src/repositories/mcp-server.repository.ts b/src/mcpd/src/repositories/mcp-server.repository.ts index 0f668c7..2e26621 100644 --- a/src/mcpd/src/repositories/mcp-server.repository.ts +++ b/src/mcpd/src/repositories/mcp-server.repository.ts @@ -33,6 +33,7 @@ export class McpServerRepository implements IMcpServerRepository { replicas: data.replicas, env: data.env, healthCheck: (data.healthCheck ?? Prisma.JsonNull) as Prisma.InputJsonValue, + volumes: data.volumes, }, }); } @@ -51,6 +52,7 @@ export class McpServerRepository implements IMcpServerRepository { if (data.replicas !== undefined) updateData['replicas'] = data.replicas; if (data.env !== undefined) updateData['env'] = data.env; if (data.healthCheck !== undefined) updateData['healthCheck'] = (data.healthCheck ?? Prisma.JsonNull) as Prisma.InputJsonValue; + if (data.volumes !== undefined) updateData['volumes'] = data.volumes; return this.prisma.mcpServer.update({ where: { id }, data: updateData }); } diff --git a/src/mcpd/src/repositories/template.repository.ts b/src/mcpd/src/repositories/template.repository.ts index f5bdd3a..7be1b9d 100644 --- a/src/mcpd/src/repositories/template.repository.ts +++ b/src/mcpd/src/repositories/template.repository.ts @@ -52,6 +52,7 @@ export class TemplateRepository implements ITemplateRepository { replicas: data.replicas, env: (data.env ?? []) as unknown as Prisma.InputJsonValue, healthCheck: (data.healthCheck ?? Prisma.JsonNull) as Prisma.InputJsonValue, + volumes: (data.volumes ?? []) as unknown as Prisma.InputJsonValue, }, }); } @@ -71,6 +72,7 @@ export class TemplateRepository implements ITemplateRepository { if (data.replicas !== undefined) updateData.replicas = data.replicas; if (data.env !== undefined) updateData.env = (data.env ?? []) as Prisma.InputJsonValue; if (data.healthCheck !== undefined) updateData.healthCheck = (data.healthCheck ?? Prisma.JsonNull) as Prisma.InputJsonValue; + if (data.volumes !== undefined) updateData.volumes = (data.volumes ?? []) as Prisma.InputJsonValue; return this.prisma.mcpTemplate.update({ where: { id }, diff --git a/src/mcpd/src/services/backup/backup-service.ts b/src/mcpd/src/services/backup/backup-service.ts index d3b1f89..c6410b3 100644 --- a/src/mcpd/src/services/backup/backup-service.ts +++ b/src/mcpd/src/services/backup/backup-service.ts @@ -39,6 +39,7 @@ export interface BackupServer { replicas: number; env: unknown; healthCheck: unknown; + volumes: unknown; } export interface BackupSecret { @@ -140,6 +141,7 @@ export class BackupService { replicas: s.replicas, env: s.env, healthCheck: s.healthCheck, + volumes: s.volumes, })); } diff --git a/src/mcpd/src/services/backup/restore-service.ts b/src/mcpd/src/services/backup/restore-service.ts index feed096..2aa9deb 100644 --- a/src/mcpd/src/services/backup/restore-service.ts +++ b/src/mcpd/src/services/backup/restore-service.ts @@ -172,6 +172,11 @@ export class RestoreService { transport: server.transport as 'STDIO' | 'SSE' | 'STREAMABLE_HTTP', replicas: server.replicas ?? 1, env: (server.env ?? []) as Array<{ name: string; value?: string; valueFrom?: { secretRef: { name: string; key: string } } }>, + // Restores the volume *declaration* only. The backing PVC / named + // volume is not part of the backup — a restored server re-attaches to + // a claim of the same name if one survived, and otherwise starts on a + // fresh empty volume. + volumes: (server.volumes ?? []) as Parameters[0]['volumes'], }; if (server.packageName) createData.packageName = server.packageName; if (server.runtime) createData.runtime = server.runtime; diff --git a/src/mcpd/src/services/docker/container-manager.ts b/src/mcpd/src/services/docker/container-manager.ts index 236cc71..b591b28 100644 --- a/src/mcpd/src/services/docker/container-manager.ts +++ b/src/mcpd/src/services/docker/container-manager.ts @@ -91,6 +91,13 @@ export class DockerContainerManager implements McpOrchestrator { Memory: memoryLimit, ...(nanoCpus ? { NanoCpus: nanoCpus } : {}), NetworkMode: spec.network ?? 'bridge', + // Named volumes, created on demand by the engine. Named (not + // anonymous) matters: `removeContainer` passes `v: true`, which reaps + // anonymous volumes but leaves named ones — which is what lets the + // data outlive the instance. + ...(spec.volumes && spec.volumes.length > 0 + ? { Binds: spec.volumes.map((v) => `${v.claimName}:${v.mountPath}`) } + : {}), }, }; if (spec.command) { diff --git a/src/mcpd/src/services/instance.service.ts b/src/mcpd/src/services/instance.service.ts index 39175ff..a722317 100644 --- a/src/mcpd/src/services/instance.service.ts +++ b/src/mcpd/src/services/instance.service.ts @@ -372,6 +372,26 @@ export class InstanceService { if (server.transport === 'SSE' || server.transport === 'STREAMABLE_HTTP') { spec.containerPort = server.containerPort ?? 3000; } + + // Volumes are keyed on the server, not this instance: `spec.name` carries + // the instance id and a claim named after it would be torn down on the + // next server edit — exactly when the data has to survive. + const volumes = (server.volumes ?? []) as Array<{ + name: string; + mountPath: string; + sizeGb?: number; + storageClass?: string; + }>; + if (volumes.length > 0) { + spec.volumes = volumes.map((v) => ({ + claimName: `mcpctl-${server.name}-${v.name}`, + mountPath: v.mountPath, + sizeGb: v.sizeGb ?? 10, + ...(v.storageClass !== undefined && v.storageClass !== '' + ? { storageClass: v.storageClass } + : {}), + })); + } // Package-based servers: command = [packageName, ...args] (entrypoint handles execution) // Docker-image servers: use explicit command if provided if (pkgCommand) { diff --git a/src/mcpd/src/services/k8s/kubernetes-orchestrator.ts b/src/mcpd/src/services/k8s/kubernetes-orchestrator.ts index 515aa46..0d6bd58 100644 --- a/src/mcpd/src/services/k8s/kubernetes-orchestrator.ts +++ b/src/mcpd/src/services/k8s/kubernetes-orchestrator.ts @@ -4,13 +4,14 @@ import type { ContainerSpec, ContainerInfo, ContainerLogs, + ContainerVolume, ExecResult, InteractiveExec, } from '../orchestrator.js'; import { K8sOfficialClient } from './k8s-client-official.js'; import type { K8sOfficialClientConfig } from './k8s-client-official.js'; -import { generatePodSpec } from './manifest-generator.js'; -import type { V1Pod } from '@kubernetes/client-node'; +import { generatePodSpec, generatePvcSpec, sanitizeName } from './manifest-generator.js'; +import type { V1Pod, V1PersistentVolumeClaim } from '@kubernetes/client-node'; function mapPodState(pod: V1Pod): ContainerInfo['state'] { const cs = pod.status?.containerStatuses?.[0]; @@ -82,6 +83,12 @@ export class KubernetesOrchestrator implements McpOrchestrator { async createContainer(spec: ContainerSpec): Promise { await this.ensureNamespace(this.namespace); + // PVCs must exist before the pod references them, or the pod stays Pending + // on an unbound claim. + for (const volume of spec.volumes ?? []) { + await this.ensurePvc(volume, spec.labels); + } + const manifest = generatePodSpec(spec, this.namespace); const pod = await this.client.core.createNamespacedPod({ namespace: this.namespace, @@ -323,6 +330,43 @@ export class KubernetesOrchestrator implements McpOrchestrator { } } + /** + * Create the PVC backing a server volume if it is not already there. + * + * Deliberately never updates an existing claim: most PVC spec fields are + * immutable after binding, and silently resizing someone's storage is not a + * side effect a container start should have. Growing a volume is an explicit + * `kubectl edit pvc` (the class must allow expansion). + */ + private async ensurePvc( + volume: ContainerVolume, + labels?: Record, + ): Promise { + const name = sanitizeName(volume.claimName); + try { + await this.client.core.readNamespacedPersistentVolumeClaim({ + name, + namespace: this.namespace, + }); + return; // Already there — reuse it, data and all. + } catch (err: unknown) { + const status = (err as { statusCode?: number }).statusCode + ?? (err as { response?: { statusCode?: number } }).response?.statusCode; + if (status !== 404) throw err; + } + + try { + await this.client.core.createNamespacedPersistentVolumeClaim({ + namespace: this.namespace, + body: generatePvcSpec(volume, this.namespace, labels) as V1PersistentVolumeClaim, + }); + } catch (createErr: unknown) { + const status = (createErr as { statusCode?: number }).statusCode + ?? (createErr as { response?: { statusCode?: number } }).response?.statusCode; + if (status !== 409) throw createErr; // Lost a create race — fine. + } + } + getNamespace(): string { return this.namespace; } diff --git a/src/mcpd/src/services/k8s/manifest-generator.ts b/src/mcpd/src/services/k8s/manifest-generator.ts index 670679d..90de376 100644 --- a/src/mcpd/src/services/k8s/manifest-generator.ts +++ b/src/mcpd/src/services/k8s/manifest-generator.ts @@ -1,4 +1,4 @@ -import type { ContainerSpec } from '../orchestrator.js'; +import type { ContainerSpec, ContainerVolume } from '../orchestrator.js'; import { DEFAULT_MEMORY_LIMIT, DEFAULT_NANO_CPUS } from '../orchestrator.js'; const MCPCTL_LABEL = 'mcpctl.managed'; @@ -20,6 +20,7 @@ export interface K8sPodManifest { env?: Array<{ name: string; value: string }>; ports?: Array<{ containerPort: number }>; stdin?: boolean; + volumeMounts?: Array<{ name: string; mountPath: string }>; resources: { limits: { memory: string; cpu: string }; requests: { memory: string; cpu: string }; @@ -35,6 +36,22 @@ export interface K8sPodManifest { restartPolicy: 'Always' | 'Never' | 'OnFailure'; automountServiceAccountToken: boolean; nodeSelector?: Record; + volumes?: Array<{ name: string; persistentVolumeClaim: { claimName: string } }>; + }; +} + +export interface K8sPvcManifest { + apiVersion: 'v1'; + kind: 'PersistentVolumeClaim'; + metadata: { + name: string; + namespace: string; + labels: Record; + }; + spec: { + accessModes: string[]; + resources: { requests: { storage: string } }; + storageClassName?: string; }; } @@ -78,6 +95,16 @@ function sanitizeName(name: string): string { return name.toLowerCase().replace(/[^a-z0-9-]/g, '-').replace(/^-+|-+$/g, '').slice(0, 63); } +function buildPodVolumes(spec: ContainerSpec): Pick { + if (!spec.volumes || spec.volumes.length === 0) return {}; + return { + volumes: spec.volumes.map((v) => ({ + name: sanitizeName(v.claimName), + persistentVolumeClaim: { claimName: sanitizeName(v.claimName) }, + })), + }; +} + function buildLabels(spec: ContainerSpec): Record { return { [MCPCTL_LABEL]: 'true', @@ -128,9 +155,55 @@ function buildContainerSpec(spec: ContainerSpec) { container.ports = [{ containerPort: spec.containerPort }]; } + if (spec.volumes && spec.volumes.length > 0) { + container.volumeMounts = spec.volumes.map((v) => ({ + name: sanitizeName(v.claimName), + mountPath: v.mountPath, + })); + } + return container; } +/** + * PVC for a server volume. Created once per server and left in place when + * instances come and go — that outliving is the entire point of the resource. + * + * ReadWriteOnce because the backing class is typically a block volume + * (Longhorn, EBS). A server asking for >1 replica plus a volume cannot have + * both; `instance.service` rejects that combination rather than silently + * leaving replicas unschedulable. + */ +export function generatePvcSpec( + volume: ContainerVolume, + namespace: string, + labels: Record = {}, +): K8sPvcManifest { + const storageClass = volume.storageClass ?? process.env['MCPD_VOLUME_STORAGE_CLASS']; + return { + apiVersion: 'v1', + kind: 'PersistentVolumeClaim', + metadata: { + name: sanitizeName(volume.claimName), + namespace, + labels: { + [MCPCTL_LABEL]: 'true', + 'app.kubernetes.io/managed-by': 'mcpctl', + ...labels, + }, + }, + spec: { + accessModes: ['ReadWriteOnce'], + resources: { requests: { storage: `${volume.sizeGb}Gi` } }, + // Only set when known: an empty string means "no class" to Kubernetes, + // which is not the same as omitting the field (use the default class). + ...(storageClass !== undefined && storageClass !== '' + ? { storageClassName: storageClass } + : {}), + }, + }; +} + export function generatePodSpec(spec: ContainerSpec, namespace: string): K8sPodManifest { const labels = buildLabels(spec); return { @@ -146,6 +219,7 @@ export function generatePodSpec(spec: ContainerSpec, namespace: string): K8sPodM restartPolicy: 'Always', // MCP server pods don't need k8s API access automountServiceAccountToken: false, + ...buildPodVolumes(spec), // On mixed-arch clusters, constrain to the same arch as mcpd // (runner images are typically single-arch) ...(process.env['MCPD_NODE_SELECTOR'] @@ -179,6 +253,7 @@ export function generateDeploymentSpec(spec: ContainerSpec, namespace: string, r containers: [buildContainerSpec(spec)], restartPolicy: 'Always', automountServiceAccountToken: false, + ...buildPodVolumes(spec), }, }, }, diff --git a/src/mcpd/src/services/orchestrator.ts b/src/mcpd/src/services/orchestrator.ts index e6767c1..15716ad 100644 --- a/src/mcpd/src/services/orchestrator.ts +++ b/src/mcpd/src/services/orchestrator.ts @@ -2,11 +2,37 @@ * Container orchestrator abstraction. Implementations can back onto Docker, Podman, or Kubernetes. */ +/** + * A persistent volume mounted into an MCP server container. + * + * `claimName` is deliberately caller-supplied and derived from the *server*, + * not the instance: instances are immutable and get recreated on any server + * edit, so an instance-scoped volume would be destroyed exactly when the data + * needs to survive. Backed by a Docker named volume or a Kubernetes PVC, and + * never removed when an instance goes away. + */ +export interface ContainerVolume { + /** Stable backing-volume name, shared by every instance of a server. */ + claimName: string; + /** Absolute path to mount at inside the container. */ + mountPath: string; + /** Requested size in GiB. Kubernetes only — Docker named volumes are unbounded. */ + sizeGb: number; + /** + * StorageClass for the PVC, defaulting to MCPD_VOLUME_STORAGE_CLASS. + * Worth setting explicitly on clusters with more than one default class, + * where an omitted class binds nondeterministically. + */ + storageClass?: string; +} + export interface ContainerSpec { /** Docker/OCI image reference */ image: string; /** Human-readable name (used as container name prefix) */ name: string; + /** Persistent volumes to mount (survive instance recreation) */ + volumes?: ContainerVolume[]; /** Custom command to run (overrides image CMD) */ command?: string[]; /** Environment variables */ diff --git a/src/mcpd/src/validation/mcp-server.schema.ts b/src/mcpd/src/validation/mcp-server.schema.ts index 20b58dc..77b803f 100644 --- a/src/mcpd/src/validation/mcp-server.schema.ts +++ b/src/mcpd/src/validation/mcp-server.schema.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import { HealthCheckSchema } from './template.schema.js'; +import { HealthCheckSchema, VolumeSpecSchema } from './template.schema.js'; const SecretRefSchema = z.object({ name: z.string().min(1), @@ -33,7 +33,16 @@ export const CreateMcpServerSchema = z.object({ replicas: z.number().int().min(0).max(10).default(1), env: z.array(ServerEnvEntrySchema).default([]), healthCheck: HealthCheckSchema.optional(), -}); + volumes: z.array(VolumeSpecSchema).default([]), +}).refine( + (s) => s.volumes.length === 0 || s.replicas <= 1, + { + message: + 'A server with volumes cannot have replicas > 1: the backing claim is ReadWriteOnce, ' + + 'so the extra replicas would sit unschedulable on a volume they cannot mount.', + path: ['replicas'], + }, +); export const UpdateMcpServerSchema = z.object({ description: z.string().max(1000).optional(), @@ -48,6 +57,7 @@ export const UpdateMcpServerSchema = z.object({ replicas: z.number().int().min(0).max(10).optional(), env: z.array(ServerEnvEntrySchema).optional(), healthCheck: HealthCheckSchema.nullable().optional(), + volumes: z.array(VolumeSpecSchema).optional(), }); export type CreateMcpServerInput = z.infer; diff --git a/src/mcpd/src/validation/template.schema.ts b/src/mcpd/src/validation/template.schema.ts index 8e4d34c..95cc78b 100644 --- a/src/mcpd/src/validation/template.schema.ts +++ b/src/mcpd/src/validation/template.schema.ts @@ -7,6 +7,21 @@ const TemplateEnvEntrySchema = z.object({ defaultValue: z.string().optional(), }); +/** + * A persistent volume declared on a server (or template). + * + * `name` is scoped to the server; the backing Docker volume / Kubernetes PVC is + * named `mcpctl--` so it is stable across instance recreation. + */ +export const VolumeSpecSchema = z.object({ + name: z.string().min(1).max(50).regex(/^[a-z0-9-]+$/, 'Volume name must be lowercase alphanumeric with hyphens'), + mountPath: z.string().min(1).max(200).regex(/^\//, 'mountPath must be absolute'), + sizeGb: z.number().int().min(1).max(1024).default(10), + storageClass: z.string().max(100).optional(), +}); + +export type VolumeSpecInput = z.infer; + export const HealthCheckSchema = z.object({ tool: z.string().min(1), arguments: z.record(z.unknown()).default({}), @@ -32,6 +47,7 @@ export const CreateTemplateSchema = z.object({ replicas: z.number().int().min(0).max(10).default(1), env: z.array(TemplateEnvEntrySchema).default([]), healthCheck: HealthCheckSchema.optional(), + volumes: z.array(VolumeSpecSchema).default([]), }); export const UpdateTemplateSchema = CreateTemplateSchema.partial().omit({ name: true }); diff --git a/src/mcpd/tests/instance-service.test.ts b/src/mcpd/tests/instance-service.test.ts index c19b4bd..6cda9bb 100644 --- a/src/mcpd/tests/instance-service.test.ts +++ b/src/mcpd/tests/instance-service.test.ts @@ -70,10 +70,11 @@ function mockOrchestrator(): McpOrchestrator { }; } -function makeServer(overrides: Partial<{ id: string; name: string; replicas: number; dockerImage: string | null; externalUrl: string | null; transport: string; command: unknown; containerPort: number | null }> = {}) { +function makeServer(overrides: Partial<{ id: string; name: string; replicas: number; dockerImage: string | null; externalUrl: string | null; transport: string; command: unknown; containerPort: number | null; volumes: unknown }> = {}) { return { id: overrides.id ?? 'srv-1', name: overrides.name ?? 'slack', + volumes: overrides.volumes ?? [], dockerImage: overrides.dockerImage ?? 'ghcr.io/slack-mcp:latest', packageName: null, transport: overrides.transport ?? 'STDIO', @@ -143,6 +144,53 @@ describe('InstanceService', () => { }); }); + describe('volumes', () => { + it('names the claim after the server, not the instance', async () => { + // The whole point of the feature: instances are recreated on every server + // edit, so an instance-scoped claim would be destroyed exactly when the + // data needs to survive. + vi.mocked(serverRepo.findById).mockResolvedValue(makeServer({ + name: 'docs', + volumes: [{ name: 'data', mountPath: '/data', sizeGb: 20, storageClass: 'longhorn' }], + })); + vi.mocked(instanceRepo.findAll).mockResolvedValue([]); + + await service.reconcile('srv-1'); + + const spec = vi.mocked(orchestrator.createContainer).mock.calls[0]![0]; + expect(spec.volumes).toEqual([ + { claimName: 'mcpctl-docs-data', mountPath: '/data', sizeGb: 20, storageClass: 'longhorn' }, + ]); + // Container name carries the instance id; the claim must not. + expect(spec.name).toContain('inst-1'); + expect(spec.volumes![0]!.claimName).not.toContain('inst-1'); + }); + + it('defaults size and leaves storageClass unset when unspecified', async () => { + vi.mocked(serverRepo.findById).mockResolvedValue(makeServer({ + name: 'docs', + volumes: [{ name: 'data', mountPath: '/data' }], + })); + vi.mocked(instanceRepo.findAll).mockResolvedValue([]); + + await service.reconcile('srv-1'); + + const spec = vi.mocked(orchestrator.createContainer).mock.calls[0]![0]; + expect(spec.volumes![0]!.sizeGb).toBe(10); + expect(spec.volumes![0]).not.toHaveProperty('storageClass'); + }); + + it('leaves the spec free of volumes when the server declares none', async () => { + vi.mocked(serverRepo.findById).mockResolvedValue(makeServer({})); + vi.mocked(instanceRepo.findAll).mockResolvedValue([]); + + await service.reconcile('srv-1'); + + const spec = vi.mocked(orchestrator.createContainer).mock.calls[0]![0]; + expect(spec.volumes).toBeUndefined(); + }); + }); + describe('reconcile', () => { it('starts instances when below desired replicas', async () => { vi.mocked(serverRepo.findById).mockResolvedValue(makeServer({ replicas: 2 })); diff --git a/src/mcpd/tests/k8s-manifest.test.ts b/src/mcpd/tests/k8s-manifest.test.ts index 22d5429..843e80d 100644 --- a/src/mcpd/tests/k8s-manifest.test.ts +++ b/src/mcpd/tests/k8s-manifest.test.ts @@ -3,6 +3,7 @@ import { generatePodSpec, generateDeploymentSpec, generateNamespaceSpec, + generatePvcSpec, formatMemory, formatCpu, sanitizeName, @@ -156,6 +157,70 @@ describe('generateDeploymentSpec', () => { }); }); +describe('volumes', () => { + const volumeSpec: ContainerSpec = { + ...baseSpec, + volumes: [{ claimName: 'mcpctl-docs-data', mountPath: '/data', sizeGb: 20, storageClass: 'longhorn' }], + }; + + it('mounts the claim in the pod and declares the volume', () => { + const pod = generatePodSpec(volumeSpec, 'mcpctl-servers'); + expect(pod.spec.volumes).toEqual([ + { name: 'mcpctl-docs-data', persistentVolumeClaim: { claimName: 'mcpctl-docs-data' } }, + ]); + expect(pod.spec.containers[0]!.volumeMounts).toEqual([ + { name: 'mcpctl-docs-data', mountPath: '/data' }, + ]); + }); + + it('omits volume fields entirely when none are declared', () => { + const pod = generatePodSpec(baseSpec, 'mcpctl-servers'); + expect(pod.spec.volumes).toBeUndefined(); + expect(pod.spec.containers[0]!.volumeMounts).toBeUndefined(); + }); + + it('carries volumes into a deployment pod template', () => { + const dep = generateDeploymentSpec(volumeSpec, 'mcpctl-servers', 1); + expect(dep.spec.template.spec.volumes).toHaveLength(1); + expect(dep.spec.template.spec.containers[0]!.volumeMounts).toHaveLength(1); + }); + + it('builds a PVC with the requested size and class', () => { + const pvc = generatePvcSpec(volumeSpec.volumes![0]!, 'mcpctl-servers'); + expect(pvc.kind).toBe('PersistentVolumeClaim'); + expect(pvc.metadata.name).toBe('mcpctl-docs-data'); + expect(pvc.metadata.namespace).toBe('mcpctl-servers'); + expect(pvc.spec.resources.requests.storage).toBe('20Gi'); + expect(pvc.spec.accessModes).toEqual(['ReadWriteOnce']); + expect(pvc.spec.storageClassName).toBe('longhorn'); + }); + + it('omits storageClassName rather than sending an empty string', () => { + // An empty string means "no storage class" to Kubernetes, which is NOT the + // same as omitting the field (use the cluster default). + const prev = process.env['MCPD_VOLUME_STORAGE_CLASS']; + delete process.env['MCPD_VOLUME_STORAGE_CLASS']; + try { + const pvc = generatePvcSpec({ claimName: 'c', mountPath: '/d', sizeGb: 1 }, 'ns'); + expect('storageClassName' in pvc.spec).toBe(false); + } finally { + if (prev !== undefined) process.env['MCPD_VOLUME_STORAGE_CLASS'] = prev; + } + }); + + it('falls back to MCPD_VOLUME_STORAGE_CLASS when the volume omits a class', () => { + const prev = process.env['MCPD_VOLUME_STORAGE_CLASS']; + process.env['MCPD_VOLUME_STORAGE_CLASS'] = 'longhorn'; + try { + const pvc = generatePvcSpec({ claimName: 'c', mountPath: '/d', sizeGb: 5 }, 'ns'); + expect(pvc.spec.storageClassName).toBe('longhorn'); + } finally { + if (prev === undefined) delete process.env['MCPD_VOLUME_STORAGE_CLASS']; + else process.env['MCPD_VOLUME_STORAGE_CLASS'] = prev; + } + }); +}); + describe('generateNamespaceSpec', () => { it('generates namespace manifest', () => { const ns = generateNamespaceSpec('mcpctl-prod'); diff --git a/stack/.env.example b/stack/.env.example index 3830b78..96ebe30 100644 --- a/stack/.env.example +++ b/stack/.env.example @@ -3,3 +3,8 @@ POSTGRES_PASSWORD=CHANGE_ME POSTGRES_DB=mcpctl MCPD_PORT=3100 MCPD_LOG_LEVEL=info + +# --- websearch profile (docker compose --profile websearch up -d) --- +# The SearXNG engine only. The MCP servers in front of it are mcpctl +# resources, not compose services. No API key needed. +SEARXNG_SECRET=CHANGE_ME diff --git a/stack/docker-compose.yml b/stack/docker-compose.yml index 4d70a48..9809bc3 100644 --- a/stack/docker-compose.yml +++ b/stack/docker-compose.yml @@ -48,6 +48,39 @@ services: retries: 3 start_period: 15s + # --- websearch profile ------------------------------------------------- + # Opt-in: `docker compose --profile websearch up -d`. + # + # Only the SearXNG *engine* lives here — it is plain infrastructure, not an + # MCP server, so mcpctl has nothing to manage it with. The MCP servers that + # sit in front of it (`searxng`, `duckduckgo`, `docs-mcp`) are mcpctl + # resources created from templates. Needs no API key. + + searxng: + image: docker.io/searxng/searxng:latest + container_name: mcpctl-searxng + profiles: ["websearch"] + restart: unless-stopped + environment: + SEARXNG_BASE_URL: http://mcpctl-searxng:8080/ + SEARXNG_SECRET: ${SEARXNG_SECRET:-mcpctl-searxng-internal-only} + # The limiter guards public instances from bots and needs Valkey to work. + # This one binds no host port and is reachable only from mcp-servers. + SEARXNG_LIMITER: "false" + volumes: + - ./searxng/settings.yml:/etc/searxng/settings.yml:ro + - mcpctl-searxng-cache:/var/cache/searxng + networks: + - mcp-servers + healthcheck: + # Probes the JSON API specifically — a healthy HTML UI with `json` missing + # from search.formats is exactly the failure mode worth catching here. + test: ["CMD-SHELL", "wget -q -O /dev/null 'http://localhost:8080/search?q=ping&format=json' || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 20s + networks: mcpctl: driver: bridge @@ -60,3 +93,4 @@ networks: volumes: mcpctl-pgdata: mcpctl-backup: + mcpctl-searxng-cache: diff --git a/stack/searxng/settings.yml b/stack/searxng/settings.yml new file mode 100644 index 0000000..bce22a5 --- /dev/null +++ b/stack/searxng/settings.yml @@ -0,0 +1,14 @@ +# Minimal SearXNG config for API/MCP use. Everything not set here inherits the +# image defaults (engines, locales, categories) via use_default_settings. +# +# secret_key and limiter come from $SEARXNG_SECRET / $SEARXNG_LIMITER in the +# environment — see stack/docker-compose.yml. `formats` has no env override, +# which is the only reason this file has to exist. +use_default_settings: true + +search: + # The upstream default is `[html]` only. Without `json` here every + # /search?format=json request 403s and mcp-searxng returns nothing. + formats: + - html + - json diff --git a/templates/docs-mcp.yaml b/templates/docs-mcp.yaml new file mode 100644 index 0000000..ddb5693 --- /dev/null +++ b/templates/docs-mcp.yaml @@ -0,0 +1,39 @@ +name: docs-mcp +version: "1.0.0" +description: Self-hosted library documentation index — version-pinned docs scraped from sites, GitHub, npm and PyPI +dockerImage: "ghcr.io/arabold/docs-mcp-server:latest" +transport: SSE +containerPort: 6280 +repositoryUrl: https://github.com/arabold/docs-mcp-server +command: + - --protocol + - http + - --host + - 0.0.0.0 + - --port + - "6280" +# The scraped index is a SQLite file (better-sqlite3 + sqlite-vec) under /data. +# There is no external-database mode, so the volume below is what makes the +# index outlive instance recreation. Its claim is named after the server +# (mcpctl--data), not the instance, so a server edit re-attaches to the +# same data instead of starting empty. +# +# Health check omitted deliberately: `search_docs` requires a library argument, +# so a synthetic probe would need a library that is only present after a scrape +# and would report unhealthy on a fresh instance. +volumes: + - name: data + mountPath: /data + sizeGb: 20 +env: + - name: DOCS_MCP_EMBEDDING_MODEL + description: >- + Embedding model as provider:name. Unset = keyword search only. For local + embeddings use openai:nomic-embed-text with OPENAI_API_BASE on Ollama. + required: false + - name: OPENAI_API_BASE + description: OpenAI-compatible embeddings endpoint, e.g. http://ollama:11434/v1 + required: false + - name: OPENAI_API_KEY + description: Embeddings API key. Literal "ollama" when pointing at a local Ollama. + required: false diff --git a/templates/duckduckgo.yaml b/templates/duckduckgo.yaml new file mode 100644 index 0000000..c21f44c --- /dev/null +++ b/templates/duckduckgo.yaml @@ -0,0 +1,24 @@ +name: duckduckgo +version: "1.0.0" +description: DuckDuckGo web search and page-to-markdown fetching — no API key, no backing service +packageName: "duckduckgo-mcp-server" +runtime: python +transport: STDIO +repositoryUrl: https://github.com/nickclyde/duckduckgo-mcp-server +healthCheck: + tool: search + arguments: + query: "mcp" + max_results: 1 +env: + - name: DDG_SAFE_SEARCH + description: Result filtering — STRICT, MODERATE or OFF + required: false + defaultValue: "MODERATE" + - name: DDG_REGION + description: Default region/language code (e.g. us-en, pl-pl, uk-en) + required: false + - name: DDG_SEARCH_BACKEND + description: Fetch backend — auto, httpx or curl. `curl` survives more bot checks. + required: false + defaultValue: "auto" diff --git a/templates/searxng.yaml b/templates/searxng.yaml new file mode 100644 index 0000000..cb16d87 --- /dev/null +++ b/templates/searxng.yaml @@ -0,0 +1,25 @@ +name: searxng +version: "1.0.0" +description: SearXNG MCP server for private web search and URL-to-markdown reading +packageName: "mcp-searxng" +transport: STDIO +repositoryUrl: https://github.com/ihor-sokoliuk/mcp-searxng +healthCheck: + tool: searxng_web_search + arguments: + query: "mcp" +env: + - name: SEARXNG_URL + description: >- + SearXNG instance URL, or ';'-separated replicas + (e.g. http://mcpctl-searxng:8080). The instance must list `json` under + `search.formats` — see stack/searxng/settings.yml. + required: true + - name: SEARXNG_DEFAULT_RESPONSE_FORMAT + description: Result shape when a call omits response_format — "text" or "json" + required: false + defaultValue: "text" + - name: SEARXNG_HTML_FALLBACK + description: Scrape the HTML result page when the JSON API is unavailable + required: false + defaultValue: "false"