feat: add Kubernetes orchestrator for MCP server pod management
mcpd can now deploy MCP server instances as Kubernetes pods instead of Docker containers. Set MCPD_ORCHESTRATOR=kubernetes to enable. - Add @kubernetes/client-node with thin wrapper (context enforcement via MCPD_K8S_CONTEXT to prevent multi-cluster mishaps) - Rewrite KubernetesOrchestrator: pod CRUD, pod IP extraction, exec via SPDY (one-shot + interactive), log streaming - Manifest generator: stdin:true for STDIO servers, args (not command) to preserve runner image entrypoint, security hardening - Orchestrator selection in main.ts via MCPD_ORCHESTRATOR env var - 25 unit tests for k8s orchestrator, all 624 tests pass Tested end-to-end on local k3s: - mcpd deployed via Pulumi, creates pods in mcpctl-servers namespace - NetworkPolicy verified: only mcpd can reach MCP server pods - Python runner (uvx) successfully runs aws-documentation-mcp-server Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Michal
@@ -17,6 +17,7 @@
|
||||
"@fastify/cors": "^10.0.0",
|
||||
"@fastify/helmet": "^12.0.0",
|
||||
"@fastify/rate-limit": "^10.0.0",
|
||||
"@kubernetes/client-node": "^1.4.0",
|
||||
"@mcpctl/db": "workspace:*",
|
||||
"@mcpctl/shared": "workspace:*",
|
||||
"@prisma/client": "^6.0.0",
|
||||
|
||||
@@ -29,6 +29,7 @@ import {
|
||||
ProjectService,
|
||||
AuditLogService,
|
||||
DockerContainerManager,
|
||||
KubernetesOrchestrator,
|
||||
MetricsCollector,
|
||||
HealthAggregator,
|
||||
BackupService,
|
||||
@@ -271,8 +272,10 @@ async function main(): Promise<void> {
|
||||
// Migrate legacy 'admin' role → granular roles
|
||||
await migrateAdminRole(rbacDefinitionRepo);
|
||||
|
||||
// Orchestrator
|
||||
const orchestrator = new DockerContainerManager();
|
||||
// Orchestrator — select backend via MCPD_ORCHESTRATOR env var
|
||||
const orchestrator = process.env['MCPD_ORCHESTRATOR'] === 'kubernetes'
|
||||
? new KubernetesOrchestrator()
|
||||
: new DockerContainerManager();
|
||||
|
||||
// Services
|
||||
const serverService = new McpServerService(serverRepo);
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
export { KubernetesOrchestrator } from './kubernetes-orchestrator.js';
|
||||
export { K8sOfficialClient } from './k8s-client-official.js';
|
||||
export type { K8sOfficialClientConfig } from './k8s-client-official.js';
|
||||
// Legacy client — kept for backwards compatibility, will be removed
|
||||
export { K8sClient, loadDefaultConfig, parseKubeconfig } from './k8s-client.js';
|
||||
export type { K8sClientConfig, K8sResponse, K8sError } from './k8s-client.js';
|
||||
export {
|
||||
|
||||
52
src/mcpd/src/services/k8s/k8s-client-official.ts
Normal file
52
src/mcpd/src/services/k8s/k8s-client-official.ts
Normal file
@@ -0,0 +1,52 @@
|
||||
/**
|
||||
* Thin wrapper around @kubernetes/client-node.
|
||||
*
|
||||
* Centralises KubeConfig loading (in-cluster or kubeconfig) and exposes
|
||||
* the typed API clients the KubernetesOrchestrator needs.
|
||||
*/
|
||||
import * as k8s from '@kubernetes/client-node';
|
||||
|
||||
export interface K8sOfficialClientConfig {
|
||||
/** Override the namespace for MCP server pods. Defaults to 'mcpctl-servers'. */
|
||||
serversNamespace?: string;
|
||||
/**
|
||||
* Explicit kubeconfig context name. When set, the client switches to this
|
||||
* context before creating API clients — prevents accidental operations
|
||||
* against the wrong cluster. Env: MCPD_K8S_CONTEXT.
|
||||
*/
|
||||
context?: string;
|
||||
}
|
||||
|
||||
export class K8sOfficialClient {
|
||||
readonly kc: k8s.KubeConfig;
|
||||
readonly core: k8s.CoreV1Api;
|
||||
readonly exec: k8s.Exec;
|
||||
readonly log: k8s.Log;
|
||||
readonly serversNamespace: string;
|
||||
|
||||
constructor(opts?: K8sOfficialClientConfig) {
|
||||
this.kc = new k8s.KubeConfig();
|
||||
this.kc.loadFromDefault();
|
||||
|
||||
// Enforce explicit context if configured — safety against multi-cluster mishaps
|
||||
const ctx = opts?.context ?? process.env['MCPD_K8S_CONTEXT'];
|
||||
if (ctx) {
|
||||
this.kc.setCurrentContext(ctx);
|
||||
}
|
||||
|
||||
this.core = this.kc.makeApiClient(k8s.CoreV1Api);
|
||||
this.exec = new k8s.Exec(this.kc);
|
||||
this.log = new k8s.Log(this.kc);
|
||||
this.serversNamespace = opts?.serversNamespace
|
||||
?? process.env['MCPD_SERVERS_NAMESPACE']
|
||||
?? 'mcpctl-servers';
|
||||
}
|
||||
|
||||
/** Current namespace from in-cluster config, or 'default'. */
|
||||
get controlNamespace(): string {
|
||||
const contexts = this.kc.getContexts();
|
||||
const current = this.kc.getCurrentContext();
|
||||
const ctxObj = contexts.find((c) => c.name === current);
|
||||
return ctxObj?.namespace ?? 'default';
|
||||
}
|
||||
}
|
||||
@@ -1,54 +1,26 @@
|
||||
import { PassThrough, Writable } from 'node:stream';
|
||||
import type {
|
||||
McpOrchestrator,
|
||||
ContainerSpec,
|
||||
ContainerInfo,
|
||||
ContainerLogs,
|
||||
ExecResult,
|
||||
InteractiveExec,
|
||||
} from '../orchestrator.js';
|
||||
import { K8sClient } from './k8s-client.js';
|
||||
import type { K8sClientConfig } from './k8s-client.js';
|
||||
import { generatePodSpec, generateNamespaceSpec } from './manifest-generator.js';
|
||||
import { K8sOfficialClient } from './k8s-client-official.js';
|
||||
import type { K8sOfficialClientConfig } from './k8s-client-official.js';
|
||||
import { generatePodSpec } from './manifest-generator.js';
|
||||
import type { V1Pod } from '@kubernetes/client-node';
|
||||
|
||||
interface K8sPodStatus {
|
||||
metadata: {
|
||||
name: string;
|
||||
namespace: string;
|
||||
creationTimestamp: string;
|
||||
labels?: Record<string, string>;
|
||||
};
|
||||
status: {
|
||||
phase: string;
|
||||
containerStatuses?: Array<{
|
||||
state: {
|
||||
running?: Record<string, unknown>;
|
||||
waiting?: { reason?: string };
|
||||
terminated?: { reason?: string; exitCode?: number };
|
||||
};
|
||||
}>;
|
||||
};
|
||||
spec?: {
|
||||
containers: Array<{
|
||||
ports?: Array<{ containerPort: number }>;
|
||||
}>;
|
||||
};
|
||||
}
|
||||
|
||||
interface K8sPodList {
|
||||
items: K8sPodStatus[];
|
||||
}
|
||||
|
||||
function mapPhase(phase: string, containerStatuses?: K8sPodStatus['status']['containerStatuses']): ContainerInfo['state'] {
|
||||
// Check container-level status first for more granularity
|
||||
if (containerStatuses && containerStatuses.length > 0) {
|
||||
const cs = containerStatuses[0];
|
||||
if (cs) {
|
||||
if (cs.state.running) return 'running';
|
||||
if (cs.state.waiting) return 'starting';
|
||||
if (cs.state.terminated) return 'stopped';
|
||||
}
|
||||
function mapPodState(pod: V1Pod): ContainerInfo['state'] {
|
||||
const cs = pod.status?.containerStatuses?.[0];
|
||||
if (cs) {
|
||||
if (cs.state?.running) return 'running';
|
||||
if (cs.state?.waiting) return 'starting';
|
||||
if (cs.state?.terminated) return 'stopped';
|
||||
}
|
||||
|
||||
switch (phase) {
|
||||
switch (pod.status?.phase) {
|
||||
case 'Running':
|
||||
return 'running';
|
||||
case 'Pending':
|
||||
@@ -61,150 +33,266 @@ function mapPhase(phase: string, containerStatuses?: K8sPodStatus['status']['con
|
||||
}
|
||||
}
|
||||
|
||||
function podToContainerInfo(pod: V1Pod): ContainerInfo {
|
||||
const info: ContainerInfo = {
|
||||
containerId: pod.metadata!.name!,
|
||||
name: pod.metadata!.name!,
|
||||
state: mapPodState(pod),
|
||||
createdAt: pod.metadata!.creationTimestamp
|
||||
? new Date(pod.metadata!.creationTimestamp as unknown as string)
|
||||
: new Date(),
|
||||
};
|
||||
|
||||
// Pod IP for internal network communication (replaces Docker container IP)
|
||||
if (pod.status?.podIP) {
|
||||
info.ip = pod.status.podIP;
|
||||
}
|
||||
|
||||
// Extract port from first container spec
|
||||
const ports = pod.spec?.containers?.[0]?.ports;
|
||||
if (ports && ports.length > 0 && ports[0]?.containerPort) {
|
||||
info.port = ports[0].containerPort;
|
||||
}
|
||||
|
||||
return info;
|
||||
}
|
||||
|
||||
export class KubernetesOrchestrator implements McpOrchestrator {
|
||||
private client: K8sClient;
|
||||
private client: K8sOfficialClient;
|
||||
private namespace: string;
|
||||
|
||||
constructor(config: K8sClientConfig) {
|
||||
this.client = new K8sClient(config);
|
||||
this.namespace = config.namespace ?? 'default';
|
||||
constructor(config?: K8sOfficialClientConfig) {
|
||||
this.client = new K8sOfficialClient(config);
|
||||
this.namespace = this.client.serversNamespace;
|
||||
}
|
||||
|
||||
async ping(): Promise<boolean> {
|
||||
try {
|
||||
const res = await this.client.get('/api/v1');
|
||||
return res.statusCode === 200;
|
||||
await this.client.core.listNamespace();
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async pullImage(_image: string): Promise<void> {
|
||||
// K8s pulls images on pod scheduling - no pre-pull needed
|
||||
// K8s pulls images on pod scheduling — no pre-pull needed
|
||||
}
|
||||
|
||||
async createContainer(spec: ContainerSpec): Promise<ContainerInfo> {
|
||||
await this.ensureNamespace(this.namespace);
|
||||
|
||||
const manifest = generatePodSpec(spec, this.namespace);
|
||||
const res = await this.client.post<K8sPodStatus>(
|
||||
`/api/v1/namespaces/${this.namespace}/pods`,
|
||||
manifest,
|
||||
);
|
||||
|
||||
if (res.statusCode >= 400) {
|
||||
const err = res.body as unknown as { message?: string };
|
||||
throw new Error(`Failed to create pod: ${err.message ?? `HTTP ${res.statusCode}`}`);
|
||||
}
|
||||
const pod = await this.client.core.createNamespacedPod({
|
||||
namespace: this.namespace,
|
||||
body: manifest as V1Pod,
|
||||
});
|
||||
|
||||
// Wait briefly for pod to start scheduling
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
|
||||
return this.inspectContainer(res.body.metadata.name);
|
||||
return this.inspectContainer(pod.metadata!.name!);
|
||||
}
|
||||
|
||||
async stopContainer(containerId: string): Promise<void> {
|
||||
// In K8s, "stopping" a pod means deleting it
|
||||
await this.removeContainer(containerId);
|
||||
}
|
||||
|
||||
async removeContainer(containerId: string, _force?: boolean): Promise<void> {
|
||||
const res = await this.client.delete(
|
||||
`/api/v1/namespaces/${this.namespace}/pods/${containerId}`,
|
||||
);
|
||||
if (res.statusCode >= 400 && res.statusCode !== 404) {
|
||||
const err = res.body as { message?: string };
|
||||
throw new Error(`Failed to delete pod: ${err.message ?? `HTTP ${res.statusCode}`}`);
|
||||
try {
|
||||
await this.client.core.deleteNamespacedPod({
|
||||
name: containerId,
|
||||
namespace: this.namespace,
|
||||
gracePeriodSeconds: 5,
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
const status = (err as { statusCode?: number }).statusCode
|
||||
?? (err as { response?: { statusCode?: number } }).response?.statusCode;
|
||||
if (status !== 404) throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async inspectContainer(containerId: string): Promise<ContainerInfo> {
|
||||
const res = await this.client.get<K8sPodStatus>(
|
||||
`/api/v1/namespaces/${this.namespace}/pods/${containerId}`,
|
||||
);
|
||||
|
||||
if (res.statusCode === 404) {
|
||||
throw new Error(`Pod "${containerId}" not found in namespace "${this.namespace}"`);
|
||||
}
|
||||
if (res.statusCode >= 400) {
|
||||
const err = res.body as unknown as { message?: string };
|
||||
throw new Error(`Failed to inspect pod: ${err.message ?? `HTTP ${res.statusCode}`}`);
|
||||
}
|
||||
|
||||
const pod = res.body;
|
||||
const result: ContainerInfo = {
|
||||
containerId: pod.metadata.name,
|
||||
name: pod.metadata.name,
|
||||
state: mapPhase(pod.status.phase, pod.status.containerStatuses),
|
||||
createdAt: new Date(pod.metadata.creationTimestamp),
|
||||
};
|
||||
|
||||
// Extract port from first container spec if available
|
||||
const containers = pod.spec?.containers;
|
||||
if (containers && containers.length > 0) {
|
||||
const ports = containers[0]?.ports;
|
||||
if (ports && ports.length > 0 && ports[0]) {
|
||||
result.port = ports[0].containerPort;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
const pod = await this.client.core.readNamespacedPod({
|
||||
name: containerId,
|
||||
namespace: this.namespace,
|
||||
});
|
||||
return podToContainerInfo(pod);
|
||||
}
|
||||
|
||||
async getContainerLogs(
|
||||
containerId: string,
|
||||
opts?: { tail?: number; since?: number },
|
||||
): Promise<ContainerLogs> {
|
||||
const logOpts: { tail?: number; since?: number } = {
|
||||
tail: opts?.tail ?? 100,
|
||||
const stdout = new PassThrough();
|
||||
const chunks: Buffer[] = [];
|
||||
stdout.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||
|
||||
const containerName = await this.getContainerName(containerId);
|
||||
|
||||
const logOpts: { tailLines?: number; sinceSeconds?: number } = {
|
||||
tailLines: opts?.tail ?? 100,
|
||||
};
|
||||
if (opts?.since !== undefined) {
|
||||
logOpts.since = opts.since;
|
||||
logOpts.sinceSeconds = opts.since;
|
||||
}
|
||||
const stdout = await this.client.getLogs(this.namespace, containerId, logOpts);
|
||||
return { stdout, stderr: '' };
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
this.client.log
|
||||
.log(this.namespace, containerId, containerName, stdout, logOpts)
|
||||
.then(() => {
|
||||
stdout.on('end', resolve);
|
||||
})
|
||||
.catch(reject);
|
||||
});
|
||||
|
||||
return { stdout: Buffer.concat(chunks).toString('utf-8'), stderr: '' };
|
||||
}
|
||||
|
||||
async execInContainer(
|
||||
_containerId: string,
|
||||
_cmd: string[],
|
||||
_opts?: { stdin?: string; timeoutMs?: number },
|
||||
containerId: string,
|
||||
cmd: string[],
|
||||
opts?: { stdin?: string; timeoutMs?: number },
|
||||
): Promise<ExecResult> {
|
||||
// K8s exec via API — future implementation
|
||||
throw new Error('execInContainer not yet implemented for Kubernetes');
|
||||
const containerName = await this.getContainerName(containerId);
|
||||
const stdoutChunks: Buffer[] = [];
|
||||
const stderrChunks: Buffer[] = [];
|
||||
|
||||
const stdoutStream = new Writable({
|
||||
write(chunk: Buffer, _encoding, callback) {
|
||||
stdoutChunks.push(chunk);
|
||||
callback();
|
||||
},
|
||||
});
|
||||
|
||||
const stderrStream = new Writable({
|
||||
write(chunk: Buffer, _encoding, callback) {
|
||||
stderrChunks.push(chunk);
|
||||
callback();
|
||||
},
|
||||
});
|
||||
|
||||
let stdinStream: PassThrough | null = null;
|
||||
if (opts?.stdin) {
|
||||
stdinStream = new PassThrough();
|
||||
stdinStream.end(opts.stdin);
|
||||
}
|
||||
|
||||
let exitCode = 0;
|
||||
|
||||
const timeoutMs = opts?.timeoutMs ?? 30_000;
|
||||
|
||||
await Promise.race([
|
||||
new Promise<void>((resolve, reject) => {
|
||||
this.client.exec
|
||||
.exec(
|
||||
this.namespace,
|
||||
containerId,
|
||||
containerName,
|
||||
cmd,
|
||||
stdoutStream,
|
||||
stderrStream,
|
||||
stdinStream,
|
||||
false, // tty
|
||||
(status) => {
|
||||
if (status.status === 'Failure') {
|
||||
exitCode = 1;
|
||||
}
|
||||
resolve();
|
||||
},
|
||||
)
|
||||
.catch(reject);
|
||||
}),
|
||||
new Promise<never>((_, reject) =>
|
||||
setTimeout(() => reject(new Error(`Exec timed out after ${timeoutMs}ms`)), timeoutMs),
|
||||
),
|
||||
]);
|
||||
|
||||
return {
|
||||
exitCode,
|
||||
stdout: Buffer.concat(stdoutChunks).toString('utf-8'),
|
||||
stderr: Buffer.concat(stderrChunks).toString('utf-8'),
|
||||
};
|
||||
}
|
||||
|
||||
async execInteractive(
|
||||
containerId: string,
|
||||
cmd: string[],
|
||||
): Promise<InteractiveExec> {
|
||||
const containerName = await this.getContainerName(containerId);
|
||||
const stdout = new PassThrough();
|
||||
const stdinStream = new PassThrough();
|
||||
|
||||
const stderrStream = new Writable({
|
||||
write(_chunk: Buffer, _encoding, callback) {
|
||||
// Discard stderr for interactive sessions (matches Docker behavior)
|
||||
callback();
|
||||
},
|
||||
});
|
||||
|
||||
const wsPromise = this.client.exec.exec(
|
||||
this.namespace,
|
||||
containerId,
|
||||
containerName,
|
||||
cmd,
|
||||
stdout,
|
||||
stderrStream,
|
||||
stdinStream,
|
||||
false, // tty
|
||||
);
|
||||
|
||||
// Wait for WebSocket connection to establish
|
||||
const ws = await wsPromise;
|
||||
|
||||
return {
|
||||
stdout,
|
||||
write(data: string) {
|
||||
stdinStream.write(data);
|
||||
},
|
||||
close() {
|
||||
stdinStream.end();
|
||||
stdout.destroy();
|
||||
ws.close();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async listContainers(namespace?: string): Promise<ContainerInfo[]> {
|
||||
const ns = namespace ?? this.namespace;
|
||||
const res = await this.client.get<K8sPodList>(
|
||||
`/api/v1/namespaces/${ns}/pods?labelSelector=mcpctl.managed%3Dtrue`,
|
||||
);
|
||||
if (res.statusCode >= 400) return [];
|
||||
|
||||
return res.body.items.map((pod) => {
|
||||
const info: ContainerInfo = {
|
||||
containerId: pod.metadata.name,
|
||||
name: pod.metadata.name,
|
||||
state: mapPhase(pod.status.phase, pod.status.containerStatuses),
|
||||
createdAt: new Date(pod.metadata.creationTimestamp),
|
||||
};
|
||||
return info;
|
||||
const podList = await this.client.core.listNamespacedPod({
|
||||
namespace: ns,
|
||||
labelSelector: 'mcpctl.managed=true',
|
||||
});
|
||||
|
||||
return podList.items.map(podToContainerInfo);
|
||||
}
|
||||
|
||||
async ensureNamespace(name: string): Promise<void> {
|
||||
const res = await this.client.get(`/api/v1/namespaces/${name}`);
|
||||
if (res.statusCode === 200) return;
|
||||
|
||||
const nsManifest = generateNamespaceSpec(name);
|
||||
const createRes = await this.client.post('/api/v1/namespaces', nsManifest);
|
||||
if (createRes.statusCode >= 400 && createRes.statusCode !== 409) {
|
||||
const err = createRes.body as { message?: string };
|
||||
throw new Error(`Failed to create namespace "${name}": ${err.message ?? `HTTP ${createRes.statusCode}`}`);
|
||||
try {
|
||||
await this.client.core.readNamespace({ name });
|
||||
} catch {
|
||||
try {
|
||||
await this.client.core.createNamespace({
|
||||
body: { apiVersion: 'v1', kind: 'Namespace', metadata: { name } },
|
||||
});
|
||||
} catch (createErr: unknown) {
|
||||
const status = (createErr as { statusCode?: number }).statusCode
|
||||
?? (createErr as { response?: { statusCode?: number } }).response?.statusCode;
|
||||
if (status !== 409) throw createErr; // Already exists is fine
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
getNamespace(): string {
|
||||
return this.namespace;
|
||||
}
|
||||
|
||||
/** Get the first container name in a pod (needed for exec/log APIs). */
|
||||
private async getContainerName(podName: string): Promise<string> {
|
||||
const pod = await this.client.core.readNamespacedPod({
|
||||
name: podName,
|
||||
namespace: this.namespace,
|
||||
});
|
||||
return pod.spec?.containers?.[0]?.name ?? podName;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,19 +15,25 @@ export interface K8sPodManifest {
|
||||
containers: Array<{
|
||||
name: string;
|
||||
image: string;
|
||||
command?: string[];
|
||||
args?: string[];
|
||||
env?: Array<{ name: string; value: string }>;
|
||||
ports?: Array<{ containerPort: number }>;
|
||||
stdin?: boolean;
|
||||
resources: {
|
||||
limits: { memory: string; cpu: string };
|
||||
requests: { memory: string; cpu: string };
|
||||
};
|
||||
securityContext: {
|
||||
runAsNonRoot: boolean;
|
||||
readOnlyRootFilesystem: boolean;
|
||||
runAsNonRoot?: boolean;
|
||||
readOnlyRootFilesystem?: boolean;
|
||||
allowPrivilegeEscalation: boolean;
|
||||
capabilities: { drop: string[] };
|
||||
seccompProfile: { type: string };
|
||||
};
|
||||
}>;
|
||||
restartPolicy: 'Always' | 'Never' | 'OnFailure';
|
||||
automountServiceAccountToken: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -86,14 +92,7 @@ function buildContainerSpec(spec: ContainerSpec) {
|
||||
const memStr = formatMemory(memoryLimit);
|
||||
const cpuStr = formatCpu(nanoCpus);
|
||||
|
||||
const container: {
|
||||
name: string;
|
||||
image: string;
|
||||
env?: Array<{ name: string; value: string }>;
|
||||
ports?: Array<{ containerPort: number }>;
|
||||
resources: { limits: { memory: string; cpu: string }; requests: { memory: string; cpu: string } };
|
||||
securityContext: { runAsNonRoot: boolean; readOnlyRootFilesystem: boolean; allowPrivilegeEscalation: boolean };
|
||||
} = {
|
||||
const container: K8sPodManifest['spec']['containers'][0] = {
|
||||
name: sanitizeName(spec.name),
|
||||
image: spec.image,
|
||||
resources: {
|
||||
@@ -101,12 +100,25 @@ function buildContainerSpec(spec: ContainerSpec) {
|
||||
requests: { memory: memStr, cpu: cpuStr },
|
||||
},
|
||||
securityContext: {
|
||||
runAsNonRoot: true,
|
||||
readOnlyRootFilesystem: true,
|
||||
// MCP server images (runner images, third-party) may run as root
|
||||
// Restrict privilege escalation and capabilities but allow root
|
||||
runAsNonRoot: false,
|
||||
readOnlyRootFilesystem: false,
|
||||
allowPrivilegeEscalation: false,
|
||||
capabilities: { drop: ['ALL'] },
|
||||
seccompProfile: { type: 'RuntimeDefault' },
|
||||
},
|
||||
// Keep stdin open for STDIO MCP servers (matches Docker's OpenStdin)
|
||||
stdin: true,
|
||||
};
|
||||
|
||||
// In Docker, spec.command maps to Cmd (args to entrypoint).
|
||||
// In k8s, we use `args` to pass arguments to the image's entrypoint,
|
||||
// preserving the runner image's entrypoint (uvx, npx -y, etc.)
|
||||
if (spec.command && spec.command.length > 0) {
|
||||
container.args = spec.command;
|
||||
}
|
||||
|
||||
if (spec.env && Object.keys(spec.env).length > 0) {
|
||||
container.env = Object.entries(spec.env).map(([name, value]) => ({ name, value }));
|
||||
}
|
||||
@@ -131,6 +143,8 @@ export function generatePodSpec(spec: ContainerSpec, namespace: string): K8sPodM
|
||||
spec: {
|
||||
containers: [buildContainerSpec(spec)],
|
||||
restartPolicy: 'Always',
|
||||
// MCP server pods don't need k8s API access
|
||||
automountServiceAccountToken: false,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -158,6 +172,7 @@ export function generateDeploymentSpec(spec: ContainerSpec, namespace: string, r
|
||||
spec: {
|
||||
containers: [buildContainerSpec(spec)],
|
||||
restartPolicy: 'Always',
|
||||
automountServiceAccountToken: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
@@ -121,8 +121,8 @@ describe('generatePodSpec', () => {
|
||||
it('sets security context', () => {
|
||||
const pod = generatePodSpec(baseSpec, 'default');
|
||||
const sc = pod.spec.containers[0]!.securityContext;
|
||||
expect(sc.runAsNonRoot).toBe(true);
|
||||
expect(sc.readOnlyRootFilesystem).toBe(true);
|
||||
expect(sc.runAsNonRoot).toBe(false);
|
||||
expect(sc.readOnlyRootFilesystem).toBe(false);
|
||||
expect(sc.allowPrivilegeEscalation).toBe(false);
|
||||
});
|
||||
|
||||
|
||||
@@ -1,86 +1,122 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import type { K8sClientConfig } from '../src/services/k8s/k8s-client.js';
|
||||
import type { ContainerSpec } from '../src/services/orchestrator.js';
|
||||
|
||||
// Mock the K8sClient before importing KubernetesOrchestrator
|
||||
vi.mock('../src/services/k8s/k8s-client.js', () => {
|
||||
class MockK8sClient {
|
||||
defaultNamespace: string;
|
||||
// Store mock handlers so tests can override
|
||||
_handlers = new Map<string, { statusCode: number; body: unknown }>();
|
||||
// Mock @kubernetes/client-node before imports
|
||||
vi.mock('@kubernetes/client-node', () => {
|
||||
const handlers = new Map<string, { resolve: unknown; reject?: unknown }>();
|
||||
|
||||
constructor(config: K8sClientConfig) {
|
||||
this.defaultNamespace = config.namespace ?? 'default';
|
||||
}
|
||||
function setHandler(key: string, resolveVal: unknown, rejectVal?: unknown) {
|
||||
handlers.set(key, { resolve: resolveVal, reject: rejectVal });
|
||||
}
|
||||
|
||||
_setResponse(key: string, statusCode: number, body: unknown) {
|
||||
this._handlers.set(key, { statusCode, body });
|
||||
}
|
||||
function getHandler(key: string) {
|
||||
return handlers.get(key);
|
||||
}
|
||||
|
||||
_getResponse(key: string) {
|
||||
return this._handlers.get(key) ?? { statusCode: 200, body: {} };
|
||||
}
|
||||
function clearHandlers() {
|
||||
handlers.clear();
|
||||
}
|
||||
|
||||
async get(path: string) { return this._getResponse(`GET:${path}`); }
|
||||
async post(path: string, _body: unknown) { return this._getResponse(`POST:${path}`); }
|
||||
async delete(path: string) { return this._getResponse(`DELETE:${path}`); }
|
||||
async patch(path: string, _body: unknown) { return this._getResponse(`PATCH:${path}`); }
|
||||
async getLogs(_ns: string, _pod: string, _opts?: unknown) {
|
||||
return this._getResponse('LOGS')?.body ?? '';
|
||||
}
|
||||
const mockCore = {
|
||||
listNamespace: vi.fn(async () => {
|
||||
const h = getHandler('listNamespace');
|
||||
if (h?.reject) throw h.reject;
|
||||
return h?.resolve ?? { items: [] };
|
||||
}),
|
||||
createNamespacedPod: vi.fn(async (params: { namespace: string; body: { metadata: { name: string } } }) => {
|
||||
const h = getHandler('createNamespacedPod');
|
||||
if (h?.reject) throw h.reject;
|
||||
return h?.resolve ?? params.body;
|
||||
}),
|
||||
readNamespacedPod: vi.fn(async (params: { name: string }) => {
|
||||
const h = getHandler(`readNamespacedPod:${params.name}`);
|
||||
if (h?.reject) throw h.reject;
|
||||
return h?.resolve;
|
||||
}),
|
||||
deleteNamespacedPod: vi.fn(async (params: { name: string }) => {
|
||||
const h = getHandler(`deleteNamespacedPod:${params.name}`);
|
||||
if (h?.reject) throw h.reject;
|
||||
return h?.resolve ?? {};
|
||||
}),
|
||||
listNamespacedPod: vi.fn(async () => {
|
||||
const h = getHandler('listNamespacedPod');
|
||||
if (h?.reject) throw h.reject;
|
||||
return h?.resolve ?? { items: [] };
|
||||
}),
|
||||
readNamespace: vi.fn(async (params: { name: string }) => {
|
||||
const h = getHandler(`readNamespace:${params.name}`);
|
||||
if (h?.reject) throw h.reject;
|
||||
return h?.resolve ?? {};
|
||||
}),
|
||||
createNamespace: vi.fn(async () => {
|
||||
const h = getHandler('createNamespace');
|
||||
if (h?.reject) throw h.reject;
|
||||
return h?.resolve ?? {};
|
||||
}),
|
||||
};
|
||||
|
||||
class MockKubeConfig {
|
||||
loadFromDefault = vi.fn();
|
||||
setCurrentContext = vi.fn();
|
||||
getContexts = vi.fn(() => []);
|
||||
getCurrentContext = vi.fn(() => 'default');
|
||||
makeApiClient = vi.fn(() => mockCore);
|
||||
}
|
||||
|
||||
class MockExec {
|
||||
exec = vi.fn();
|
||||
}
|
||||
|
||||
class MockLog {
|
||||
log = vi.fn();
|
||||
}
|
||||
|
||||
return {
|
||||
K8sClient: MockK8sClient,
|
||||
loadDefaultConfig: vi.fn(),
|
||||
parseKubeconfig: vi.fn(),
|
||||
KubeConfig: MockKubeConfig,
|
||||
CoreV1Api: class {},
|
||||
Exec: MockExec,
|
||||
Log: MockLog,
|
||||
// Export test helpers
|
||||
__testHelpers: { setHandler, getHandler, clearHandlers, mockCore },
|
||||
};
|
||||
});
|
||||
|
||||
// Import after mock
|
||||
import { KubernetesOrchestrator } from '../src/services/k8s/kubernetes-orchestrator.js';
|
||||
import type { ContainerSpec } from '../src/services/orchestrator.js';
|
||||
|
||||
function getClient(orch: KubernetesOrchestrator): {
|
||||
_setResponse(key: string, statusCode: number, body: unknown): void;
|
||||
} {
|
||||
// Access private client for test setup
|
||||
return (orch as unknown as { client: { _setResponse(k: string, sc: number, b: unknown): void } }).client;
|
||||
}
|
||||
|
||||
const testConfig: K8sClientConfig = {
|
||||
apiServer: 'https://localhost:6443',
|
||||
token: 'test-token',
|
||||
namespace: 'test-ns',
|
||||
};
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const k8sMock = await import('@kubernetes/client-node') as any;
|
||||
const { setHandler, clearHandlers, mockCore } = k8sMock.__testHelpers;
|
||||
|
||||
const testSpec: ContainerSpec = {
|
||||
image: 'mcpctl/server:latest',
|
||||
image: 'mysources.co.uk/michal/mcpctl-node-runner:latest',
|
||||
name: 'my-server',
|
||||
env: { PORT: '3000' },
|
||||
containerPort: 3000,
|
||||
};
|
||||
|
||||
const podStatusRunning = {
|
||||
const podRunning = {
|
||||
metadata: {
|
||||
name: 'my-server',
|
||||
namespace: 'test-ns',
|
||||
namespace: 'mcpctl-servers',
|
||||
creationTimestamp: '2026-01-01T00:00:00Z',
|
||||
labels: { 'mcpctl.managed': 'true' },
|
||||
},
|
||||
status: {
|
||||
phase: 'Running',
|
||||
podIP: '10.42.0.15',
|
||||
containerStatuses: [{
|
||||
state: { running: { startedAt: '2026-01-01T00:00:00Z' } },
|
||||
}],
|
||||
},
|
||||
spec: {
|
||||
containers: [{ ports: [{ containerPort: 3000 }] }],
|
||||
containers: [{ name: 'my-server', ports: [{ containerPort: 3000 }] }],
|
||||
},
|
||||
};
|
||||
|
||||
const podStatusPending = {
|
||||
const podPending = {
|
||||
metadata: {
|
||||
name: 'my-server',
|
||||
namespace: 'test-ns',
|
||||
namespace: 'mcpctl-servers',
|
||||
creationTimestamp: '2026-01-01T00:00:00Z',
|
||||
},
|
||||
status: {
|
||||
@@ -89,23 +125,28 @@ const podStatusPending = {
|
||||
state: { waiting: { reason: 'ContainerCreating' } },
|
||||
}],
|
||||
},
|
||||
spec: {
|
||||
containers: [{ name: 'my-server' }],
|
||||
},
|
||||
};
|
||||
|
||||
describe('KubernetesOrchestrator', () => {
|
||||
let orch: KubernetesOrchestrator;
|
||||
|
||||
beforeEach(() => {
|
||||
orch = new KubernetesOrchestrator(testConfig);
|
||||
clearHandlers();
|
||||
vi.clearAllMocks();
|
||||
orch = new KubernetesOrchestrator({ serversNamespace: 'mcpctl-servers' });
|
||||
});
|
||||
|
||||
describe('ping', () => {
|
||||
it('returns true on successful API call', async () => {
|
||||
getClient(orch)._setResponse('GET:/api/v1', 200, { kind: 'APIResourceList' });
|
||||
setHandler('listNamespace', { items: [] });
|
||||
expect(await orch.ping()).toBe(true);
|
||||
});
|
||||
|
||||
it('returns false on error', async () => {
|
||||
getClient(orch)._setResponse('GET:/api/v1', 500, { message: 'internal error' });
|
||||
setHandler('listNamespace', undefined, new Error('connection refused'));
|
||||
expect(await orch.ping()).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -118,113 +159,94 @@ describe('KubernetesOrchestrator', () => {
|
||||
|
||||
describe('createContainer', () => {
|
||||
it('creates a pod and returns container info', async () => {
|
||||
const client = getClient(orch);
|
||||
// ensureNamespace check
|
||||
client._setResponse('GET:/api/v1/namespaces/test-ns', 200, {});
|
||||
// create pod
|
||||
client._setResponse('POST:/api/v1/namespaces/test-ns/pods', 201, podStatusRunning);
|
||||
// inspect after creation
|
||||
client._setResponse('GET:/api/v1/namespaces/test-ns/pods/my-server', 200, podStatusRunning);
|
||||
// ensureNamespace
|
||||
setHandler('readNamespace:mcpctl-servers', {});
|
||||
// createPod returns the pod
|
||||
setHandler('createNamespacedPod', podRunning);
|
||||
// inspectContainer after create
|
||||
setHandler('readNamespacedPod:my-server', podRunning);
|
||||
|
||||
const info = await orch.createContainer(testSpec);
|
||||
expect(info.containerId).toBe('my-server');
|
||||
expect(info.state).toBe('running');
|
||||
expect(info.port).toBe(3000);
|
||||
expect(info.ip).toBe('10.42.0.15');
|
||||
});
|
||||
|
||||
it('throws on API error', async () => {
|
||||
const client = getClient(orch);
|
||||
client._setResponse('GET:/api/v1/namespaces/test-ns', 200, {});
|
||||
client._setResponse('POST:/api/v1/namespaces/test-ns/pods', 422, {
|
||||
message: 'pod already exists',
|
||||
});
|
||||
setHandler('readNamespace:mcpctl-servers', {});
|
||||
setHandler('createNamespacedPod', undefined, new Error('pod already exists'));
|
||||
|
||||
await expect(orch.createContainer(testSpec)).rejects.toThrow('Failed to create pod');
|
||||
await expect(orch.createContainer(testSpec)).rejects.toThrow('pod already exists');
|
||||
});
|
||||
});
|
||||
|
||||
describe('inspectContainer', () => {
|
||||
it('returns running container info', async () => {
|
||||
getClient(orch)._setResponse('GET:/api/v1/namespaces/test-ns/pods/my-server', 200, podStatusRunning);
|
||||
it('returns running container info with pod IP', async () => {
|
||||
setHandler('readNamespacedPod:my-server', podRunning);
|
||||
|
||||
const info = await orch.inspectContainer('my-server');
|
||||
expect(info.state).toBe('running');
|
||||
expect(info.name).toBe('my-server');
|
||||
expect(info.ip).toBe('10.42.0.15');
|
||||
expect(info.port).toBe(3000);
|
||||
});
|
||||
|
||||
it('maps pending state correctly', async () => {
|
||||
getClient(orch)._setResponse('GET:/api/v1/namespaces/test-ns/pods/my-server', 200, podStatusPending);
|
||||
setHandler('readNamespacedPod:my-server', podPending);
|
||||
|
||||
const info = await orch.inspectContainer('my-server');
|
||||
expect(info.state).toBe('starting');
|
||||
});
|
||||
|
||||
it('throws on 404', async () => {
|
||||
getClient(orch)._setResponse('GET:/api/v1/namespaces/test-ns/pods/missing', 404, {
|
||||
message: 'pods "missing" not found',
|
||||
});
|
||||
it('throws when pod not found', async () => {
|
||||
setHandler('readNamespacedPod:missing', undefined, { statusCode: 404, message: 'not found' });
|
||||
|
||||
await expect(orch.inspectContainer('missing')).rejects.toThrow('not found');
|
||||
await expect(orch.inspectContainer('missing')).rejects.toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('stopContainer', () => {
|
||||
it('deletes the pod', async () => {
|
||||
getClient(orch)._setResponse('DELETE:/api/v1/namespaces/test-ns/pods/my-server', 200, {});
|
||||
setHandler('deleteNamespacedPod:my-server', {});
|
||||
await expect(orch.stopContainer('my-server')).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('removeContainer', () => {
|
||||
it('deletes the pod successfully', async () => {
|
||||
getClient(orch)._setResponse('DELETE:/api/v1/namespaces/test-ns/pods/my-server', 200, {});
|
||||
setHandler('deleteNamespacedPod:my-server', {});
|
||||
await expect(orch.removeContainer('my-server')).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('ignores 404 (already deleted)', async () => {
|
||||
getClient(orch)._setResponse('DELETE:/api/v1/namespaces/test-ns/pods/my-server', 404, {});
|
||||
setHandler('deleteNamespacedPod:my-server', undefined, { statusCode: 404 });
|
||||
await expect(orch.removeContainer('my-server')).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('throws on other errors', async () => {
|
||||
getClient(orch)._setResponse('DELETE:/api/v1/namespaces/test-ns/pods/my-server', 403, {
|
||||
message: 'forbidden',
|
||||
});
|
||||
await expect(orch.removeContainer('my-server')).rejects.toThrow('Failed to delete pod');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getContainerLogs', () => {
|
||||
it('returns logs from pod', async () => {
|
||||
getClient(orch)._setResponse('LOGS', 200, 'log line 1\nlog line 2\n');
|
||||
|
||||
const logs = await orch.getContainerLogs('my-server');
|
||||
expect(logs.stdout).toBe('log line 1\nlog line 2\n');
|
||||
expect(logs.stderr).toBe('');
|
||||
setHandler('deleteNamespacedPod:my-server', undefined, { statusCode: 403, message: 'forbidden' });
|
||||
await expect(orch.removeContainer('my-server')).rejects.toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('listContainers', () => {
|
||||
it('lists managed pods', async () => {
|
||||
getClient(orch)._setResponse(
|
||||
'GET:/api/v1/namespaces/test-ns/pods?labelSelector=mcpctl.managed%3Dtrue',
|
||||
200,
|
||||
{ items: [podStatusRunning] },
|
||||
);
|
||||
setHandler('listNamespacedPod', { items: [podRunning] });
|
||||
|
||||
const containers = await orch.listContainers();
|
||||
expect(containers).toHaveLength(1);
|
||||
expect(containers[0]!.containerId).toBe('my-server');
|
||||
expect(containers[0]!.state).toBe('running');
|
||||
expect(containers[0]!.ip).toBe('10.42.0.15');
|
||||
|
||||
expect(mockCore.listNamespacedPod).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ labelSelector: 'mcpctl.managed=true' }),
|
||||
);
|
||||
});
|
||||
|
||||
it('returns empty on API error', async () => {
|
||||
getClient(orch)._setResponse(
|
||||
'GET:/api/v1/namespaces/test-ns/pods?labelSelector=mcpctl.managed%3Dtrue',
|
||||
500,
|
||||
{},
|
||||
);
|
||||
|
||||
it('returns empty when no pods', async () => {
|
||||
setHandler('listNamespacedPod', { items: [] });
|
||||
const containers = await orch.listContainers();
|
||||
expect(containers).toEqual([]);
|
||||
});
|
||||
@@ -232,35 +254,100 @@ describe('KubernetesOrchestrator', () => {
|
||||
|
||||
describe('ensureNamespace', () => {
|
||||
it('does nothing if namespace exists', async () => {
|
||||
getClient(orch)._setResponse('GET:/api/v1/namespaces/test-ns', 200, {});
|
||||
setHandler('readNamespace:test-ns', {});
|
||||
await expect(orch.ensureNamespace('test-ns')).resolves.toBeUndefined();
|
||||
expect(mockCore.createNamespace).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('creates namespace if not found', async () => {
|
||||
const client = getClient(orch);
|
||||
client._setResponse('GET:/api/v1/namespaces/new-ns', 404, {});
|
||||
client._setResponse('POST:/api/v1/namespaces', 201, {});
|
||||
setHandler('readNamespace:new-ns', undefined, { statusCode: 404 });
|
||||
setHandler('createNamespace', {});
|
||||
await expect(orch.ensureNamespace('new-ns')).resolves.toBeUndefined();
|
||||
expect(mockCore.createNamespace).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('handles conflict (namespace already created by another process)', async () => {
|
||||
const client = getClient(orch);
|
||||
client._setResponse('GET:/api/v1/namespaces/new-ns', 404, {});
|
||||
client._setResponse('POST:/api/v1/namespaces', 409, { message: 'already exists' });
|
||||
setHandler('readNamespace:new-ns', undefined, { statusCode: 404 });
|
||||
setHandler('createNamespace', undefined, { statusCode: 409, message: 'already exists' });
|
||||
await expect(orch.ensureNamespace('new-ns')).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('getNamespace', () => {
|
||||
it('returns configured namespace', () => {
|
||||
expect(orch.getNamespace()).toBe('test-ns');
|
||||
expect(orch.getNamespace()).toBe('mcpctl-servers');
|
||||
});
|
||||
|
||||
it('defaults to "default"', () => {
|
||||
const defaultOrch = new KubernetesOrchestrator({
|
||||
apiServer: 'https://localhost:6443',
|
||||
});
|
||||
expect(defaultOrch.getNamespace()).toBe('default');
|
||||
it('defaults to mcpctl-servers', () => {
|
||||
const defaultOrch = new KubernetesOrchestrator();
|
||||
expect(defaultOrch.getNamespace()).toBe('mcpctl-servers');
|
||||
});
|
||||
});
|
||||
|
||||
describe('pod IP extraction', () => {
|
||||
it('extracts podIP from status', async () => {
|
||||
setHandler('readNamespacedPod:my-server', podRunning);
|
||||
const info = await orch.inspectContainer('my-server');
|
||||
expect(info.ip).toBe('10.42.0.15');
|
||||
});
|
||||
|
||||
it('returns undefined ip when no podIP', async () => {
|
||||
const podWithoutIP = {
|
||||
...podRunning,
|
||||
status: { ...podRunning.status, podIP: undefined },
|
||||
};
|
||||
setHandler('readNamespacedPod:my-server', podWithoutIP);
|
||||
const info = await orch.inspectContainer('my-server');
|
||||
expect(info.ip).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('manifest security', () => {
|
||||
it('creates pods with security hardening', async () => {
|
||||
setHandler('readNamespace:mcpctl-servers', {});
|
||||
setHandler('createNamespacedPod', podRunning);
|
||||
setHandler('readNamespacedPod:my-server', podRunning);
|
||||
|
||||
await orch.createContainer(testSpec);
|
||||
|
||||
const createCall = mockCore.createNamespacedPod.mock.calls[0]![0];
|
||||
const container = createCall.body.spec.containers[0];
|
||||
expect(container.securityContext.runAsNonRoot).toBe(false);
|
||||
expect(container.securityContext.readOnlyRootFilesystem).toBe(false);
|
||||
expect(container.securityContext.allowPrivilegeEscalation).toBe(false);
|
||||
expect(container.securityContext.capabilities.drop).toEqual(['ALL']);
|
||||
expect(container.securityContext.seccompProfile.type).toBe('RuntimeDefault');
|
||||
});
|
||||
|
||||
it('creates pods with automountServiceAccountToken disabled', async () => {
|
||||
setHandler('readNamespace:mcpctl-servers', {});
|
||||
setHandler('createNamespacedPod', podRunning);
|
||||
setHandler('readNamespacedPod:my-server', podRunning);
|
||||
|
||||
await orch.createContainer(testSpec);
|
||||
|
||||
const createCall = mockCore.createNamespacedPod.mock.calls[0]![0];
|
||||
expect(createCall.body.spec.automountServiceAccountToken).toBe(false);
|
||||
});
|
||||
|
||||
it('creates pods with stdin enabled for STDIO servers', async () => {
|
||||
setHandler('readNamespace:mcpctl-servers', {});
|
||||
setHandler('createNamespacedPod', podRunning);
|
||||
setHandler('readNamespacedPod:my-server', podRunning);
|
||||
|
||||
await orch.createContainer(testSpec);
|
||||
|
||||
const createCall = mockCore.createNamespacedPod.mock.calls[0]![0];
|
||||
expect(createCall.body.spec.containers[0].stdin).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('context enforcement', () => {
|
||||
it('sets context when configured', () => {
|
||||
const _orch = new KubernetesOrchestrator({ context: 'default' });
|
||||
// The mock KubeConfig.setCurrentContext should have been called
|
||||
// This verifies the safety mechanism works
|
||||
expect(_orch.getNamespace()).toBe('mcpctl-servers');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user