feat(mcpd+cli+mcplocal): /llms/<name>/members + POOL column + --pool-name (v4 Stage 2)
Surfaces the v4 pool model end-to-end: - mcpd: GET /api/v1/llms/:name/members returns the effective pool the named anchor belongs to, plus aggregate stats (size, activeCount, explicit vs implicit pool key). RBAC inherits from `view:llms` — same as the single-Llm route. Members are full LlmView shapes so callers don't need a second roundtrip to render the pool block. - mcpd: VirtualLlmService.register accepts an optional `poolName` on RegisterProviderInput; the route's `coerceProviderInput` validates the same character set as CreateLlmSchema.poolName. Backwards compatible — older mcplocals that don't send the field continue to publish solo Llms. - CLI `get llm` table: new POOL column right after NAME. Solo rows show "-" so the "no pool / pool of 1" case is unambiguous (per user direction "make sure we see it, prominently visible and impossible to mistake"). - CLI `describe llm`: fetches /members and renders a Pool block at the top of the detail view when the row is in an explicit pool OR when its implicit pool has size > 1. Each member line shows kind/status; the anchor row gets "← this row". Block is suppressed for solo rows so describe stays compact in the common case. - CLI `create llm --pool-name <name>` flag and apply schema both accept the new field. Yaml round-trip preserves it: get -o yaml emits `poolName: <name>`, apply -f re-imports it without diff. Verified end-to-end against the live mcpd. - mcplocal: LlmProviderFileEntry gains optional `poolName`; main.ts and registrar.ts thread it through into the register payload. Use case for distributed inference: each user's mcplocal picks a unique `name` (e.g. `vllm-<host>-qwen3`) but a shared `poolName` (e.g. `user-vllm-qwen3-thinking`); agents see one logical pool that auto-grows as workers come online. - Shell completions: regenerated from source via the existing scripts/generate-completions.ts. `--pool-name` now suggests in fish + bash for `mcpctl create llm`. Tests: +3 new mcpd route tests for /members (explicit pool, solo pool of 1, missing-anchor 404). All suites green: mcpd 868/868 (was 865, +3), mcplocal 723/723, cli 437/437. Stage 3 (next): live smoke against 2 publishers sharing a pool name + docs.
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
import type { LlmService } from '../services/llm.service.js';
|
||||
import { LlmAuthVerificationError } from '../services/llm.service.js';
|
||||
import type { LlmService, LlmView } from '../services/llm.service.js';
|
||||
import { LlmAuthVerificationError, effectivePoolName } from '../services/llm.service.js';
|
||||
import { NotFoundError, ConflictError } from '../services/mcp-server.service.js';
|
||||
|
||||
export function registerLlmRoutes(
|
||||
@@ -86,6 +86,44 @@ export function registerLlmRoutes(
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
|
||||
// v4: list all members of the effective pool that the named Llm belongs to.
|
||||
// The path uses an explicit `/members` suffix so it can't collide with the
|
||||
// single-Llm `/api/v1/llms/:id` route — an Llm whose name happens to be
|
||||
// "members" would otherwise be unaddressable.
|
||||
// Returns [] members[] AND a small header object so callers don't need a
|
||||
// second roundtrip to compute pool stats; agents.ts and the CLI both want
|
||||
// size + activeCount.
|
||||
app.get<{ Params: { name: string } }>('/api/v1/llms/:name/members', async (request, reply) => {
|
||||
try {
|
||||
const anchor = await getByIdOrName(service, request.params.name);
|
||||
const members = await service.listPoolMembers(effectivePoolName(anchor));
|
||||
return {
|
||||
poolName: effectivePoolName(anchor),
|
||||
explicitPoolName: anchor.poolName,
|
||||
size: members.length,
|
||||
activeCount: members.filter((m) => m.status === 'active').length,
|
||||
members,
|
||||
};
|
||||
} catch (err) {
|
||||
if (err instanceof NotFoundError) {
|
||||
reply.code(404);
|
||||
return { error: err.message };
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** v4: convenience type for the new `/members` endpoint response. */
|
||||
export interface PoolMembersResponse {
|
||||
/** Effective pool key (poolName ?? name on the anchor row). */
|
||||
poolName: string;
|
||||
/** Anchor row's literal poolName field — null when it falls back to its own name. */
|
||||
explicitPoolName: string | null;
|
||||
size: number;
|
||||
activeCount: number;
|
||||
members: LlmView[];
|
||||
}
|
||||
|
||||
const CUID_RE = /^c[a-z0-9]{24}/i;
|
||||
|
||||
@@ -201,6 +201,7 @@ function coerceProviderInput(raw: unknown): {
|
||||
description?: string;
|
||||
extraConfig?: Record<string, unknown>;
|
||||
initialStatus?: 'active' | 'hibernating';
|
||||
poolName?: string;
|
||||
} {
|
||||
if (raw === null || typeof raw !== 'object') {
|
||||
throw Object.assign(new Error('provider entry must be an object'), { statusCode: 400 });
|
||||
@@ -227,5 +228,11 @@ function coerceProviderInput(raw: unknown): {
|
||||
if (o['initialStatus'] === 'active' || o['initialStatus'] === 'hibernating') {
|
||||
out.initialStatus = o['initialStatus'];
|
||||
}
|
||||
// v4: optional pool key. Validation matches CreateLlmSchema.poolName so
|
||||
// a publisher can't slip an uppercase or whitespace name past the
|
||||
// virtual-publish path that the public CRUD path would reject.
|
||||
if (typeof o['poolName'] === 'string' && /^[a-z0-9-]+$/.test(o['poolName']) && o['poolName'].length >= 1 && o['poolName'].length <= 100) {
|
||||
out.poolName = o['poolName'];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
@@ -115,6 +115,17 @@ export class LlmService {
|
||||
return this.repo.findByPoolName(poolName);
|
||||
}
|
||||
|
||||
/**
|
||||
* v4: API/CLI-facing version of `findByPoolName` that returns full
|
||||
* `LlmView`s (apiKeyRef resolved via SecretService). Used by the
|
||||
* `GET /api/v1/llms/:name/members` route and `mcpctl describe llm` —
|
||||
* both of which want the full row shape for display.
|
||||
*/
|
||||
async listPoolMembers(poolName: string): Promise<LlmView[]> {
|
||||
const rows = await this.repo.findByPoolName(poolName);
|
||||
return Promise.all(rows.map((r) => this.toView(r)));
|
||||
}
|
||||
|
||||
async create(input: unknown, opts: { skipAuthCheck?: boolean } = {}): Promise<LlmView> {
|
||||
const data = CreateLlmSchema.parse(input);
|
||||
const existing = await this.repo.findByName(data.name);
|
||||
|
||||
@@ -47,6 +47,15 @@ export interface RegisterProviderInput {
|
||||
* publish time.
|
||||
*/
|
||||
initialStatus?: 'active' | 'hibernating';
|
||||
/**
|
||||
* v4: optional pool key. When set, this virtual Llm row stacks with
|
||||
* any other Llms (public OR virtual from any session) sharing the
|
||||
* same value. The chat dispatcher then load-balances across all
|
||||
* healthy members. Cluster-wide name uniqueness still applies — the
|
||||
* publisher picks a unique `name` (e.g. `vllm-<host>-qwen3`) and
|
||||
* shares the `poolName` with siblings.
|
||||
*/
|
||||
poolName?: string;
|
||||
}
|
||||
|
||||
export interface RegisterResult {
|
||||
@@ -147,6 +156,7 @@ export class VirtualLlmService implements IVirtualLlmService {
|
||||
tier: p.tier ?? 'fast',
|
||||
description: p.description ?? '',
|
||||
...(p.extraConfig !== undefined ? { extraConfig: p.extraConfig } : {}),
|
||||
...(p.poolName !== undefined ? { poolName: p.poolName } : {}),
|
||||
kind: 'virtual',
|
||||
providerSessionId: sessionId,
|
||||
status: initialStatus,
|
||||
@@ -180,6 +190,7 @@ export class VirtualLlmService implements IVirtualLlmService {
|
||||
...(p.tier !== undefined ? { tier: p.tier } : {}),
|
||||
...(p.description !== undefined ? { description: p.description } : {}),
|
||||
...(p.extraConfig !== undefined ? { extraConfig: p.extraConfig } : {}),
|
||||
...(p.poolName !== undefined ? { poolName: p.poolName } : {}),
|
||||
kind: 'virtual',
|
||||
providerSessionId: sessionId,
|
||||
status: initialStatus,
|
||||
|
||||
@@ -21,6 +21,12 @@ function makeLlm(overrides: Partial<Llm> = {}): Llm {
|
||||
apiKeySecretId: null,
|
||||
apiKeySecretKey: null,
|
||||
extraConfig: {},
|
||||
poolName: null,
|
||||
kind: 'public',
|
||||
providerSessionId: null,
|
||||
lastHeartbeatAt: null,
|
||||
status: 'active',
|
||||
inactiveSince: null,
|
||||
version: 1,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
@@ -38,6 +44,17 @@ function mockRepo(initial: Llm[] = []): ILlmRepository {
|
||||
return null;
|
||||
}),
|
||||
findByTier: vi.fn(async () => []),
|
||||
findByPoolName: vi.fn(async (poolName: string) => {
|
||||
const out: Llm[] = [];
|
||||
for (const r of rows.values()) {
|
||||
if (r.poolName === poolName) out.push(r);
|
||||
else if (r.poolName === null && r.name === poolName) out.push(r);
|
||||
}
|
||||
return out;
|
||||
}),
|
||||
findBySessionId: vi.fn(async () => []),
|
||||
findStaleVirtuals: vi.fn(async () => []),
|
||||
findExpiredInactives: vi.fn(async () => []),
|
||||
create: vi.fn(async (data) => {
|
||||
const row = makeLlm({ id: 'new-id', name: data.name, type: data.type, model: data.model });
|
||||
rows.set(row.id, row);
|
||||
@@ -191,4 +208,50 @@ describe('Llm Routes', () => {
|
||||
const res = await app.inject({ method: 'DELETE', url: '/api/v1/llms/missing' });
|
||||
expect(res.statusCode).toBe(404);
|
||||
});
|
||||
|
||||
// ── v4: GET /api/v1/llms/:name/members ──
|
||||
|
||||
it('GET /api/v1/llms/:name/members returns all members of an explicit pool', async () => {
|
||||
await createApp(mockRepo([
|
||||
makeLlm({ id: 'l1', name: 'qwen-prod-1', poolName: 'qwen-pool', model: 'qwen3' }),
|
||||
makeLlm({ id: 'l2', name: 'qwen-prod-2', poolName: 'qwen-pool', model: 'qwen3' }),
|
||||
makeLlm({ id: 'l3', name: 'qwen-prod-3', poolName: 'qwen-pool', model: 'qwen3', status: 'inactive' }),
|
||||
makeLlm({ id: 'other', name: 'gpt-4o', poolName: null, model: 'gpt-4o' }),
|
||||
]));
|
||||
// Hit via any pool member's name — the route resolves the anchor's
|
||||
// effective pool key and lists all matching rows.
|
||||
const res = await app.inject({ method: 'GET', url: '/api/v1/llms/qwen-prod-1/members' });
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = res.json<{
|
||||
poolName: string;
|
||||
explicitPoolName: string | null;
|
||||
size: number;
|
||||
activeCount: number;
|
||||
members: Array<{ name: string }>;
|
||||
}>();
|
||||
expect(body.poolName).toBe('qwen-pool');
|
||||
expect(body.explicitPoolName).toBe('qwen-pool');
|
||||
expect(body.size).toBe(3);
|
||||
expect(body.activeCount).toBe(2);
|
||||
expect(body.members.map((m) => m.name).sort()).toEqual(['qwen-prod-1', 'qwen-prod-2', 'qwen-prod-3']);
|
||||
});
|
||||
|
||||
it('GET /api/v1/llms/:name/members for a solo Llm returns a pool of 1', async () => {
|
||||
await createApp(mockRepo([
|
||||
makeLlm({ id: 'solo', name: 'gpt-4o', poolName: null, model: 'gpt-4o' }),
|
||||
]));
|
||||
const res = await app.inject({ method: 'GET', url: '/api/v1/llms/gpt-4o/members' });
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = res.json<{ poolName: string; explicitPoolName: string | null; size: number; activeCount: number }>();
|
||||
expect(body.poolName).toBe('gpt-4o');
|
||||
expect(body.explicitPoolName).toBeNull();
|
||||
expect(body.size).toBe(1);
|
||||
expect(body.activeCount).toBe(1);
|
||||
});
|
||||
|
||||
it('GET /api/v1/llms/:name/members returns 404 when the anchor name does not exist', async () => {
|
||||
await createApp(mockRepo());
|
||||
const res = await app.inject({ method: 'GET', url: '/api/v1/llms/nope/members' });
|
||||
expect(res.statusCode).toBe(404);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user