feat(mcpd): VirtualLlmService rewires through durable queue (v5 Stage 2)

The in-memory `tasksById` map for inference tasks is gone. Every
inference call lands as a row in `InferenceTask`; the result POST
updates the row + emits a wakeup; the in-flight HTTP handler unblocks
on the wake. mcpd surviving a restart no longer drops in-flight tasks,
and a worker disconnecting mid-task no longer fails the caller — the
row reverts to pending and a sibling worker on the same pool drains it.

Wake tasks (publisher control messages, not inference) keep their own
small in-memory map (`wakeTasks`). They're millisecond-scoped and
don't benefit from durability — a missed wake on restart just means
the next infer fires a fresh wake.

Behavioral changes worth flagging:

- Worker disconnect mid-task: WAS reject ref.done with "publisher
  disconnected"; NOW revert claimed/running rows to pending. Original
  caller's ref.done keeps waiting up to INFER_AWAIT_TIMEOUT_MS (10
  min); whichever worker delivers the result fulfills it.

- bindSession drains pending tasks for the session's pool keys. So
  tasks queued while no worker was up automatically get dispatched
  when one shows up. The drain matches by *effective pool key*
  (poolName ?? name) — tasks queued against vllm-alice get drained
  by any session whose owned Llms share alice's pool.

- New `failFast: true` option on enqueueInferTask (default: false).
  Existing callers that NEED fast-fail get it explicitly:
    - Direct `/api/v1/llms/<name>/infer` route: caller pinned a
      specific Llm and wants 503 immediately if the publisher is
      offline; queueing for an unknown future worker would surprise.
    - chat.service pool failover loop: it iterates pool candidates
      and needs each candidate's transport failure to surface fast.
      Without failFast, a downed pool member would absorb the call
      into the queue and the loop would wait 10 min before trying
      the next.
  The async API route (Stage 3) leaves failFast=false — that's the
  whole point of the durable queue path.

- VirtualLlmService now requires an InferenceTaskService dep at
  construction. Older test wirings that didn't pass it get a clear
  "InferenceTaskService not wired" error from enqueueInferTask
  rather than a confusing in-memory stub.

Tests:

- 12 existing virtual-llm-service tests updated for the new
  semantics: "rejects when no session" → "queues durably"; "rejects
  when row inactive" → "still queues (pool may have a sibling)";
  "unbindSession rejects in-flight tasks" → "reverts to pending".
  Wake-task probing now uses `wakeTasks` instead of `tasksById`.

- 3 new v5-specific tests: drain-on-bind matches by effective pool
  key (not just name); enqueue without a session keeps the row
  pending; completeTask via the result-route updates the DB and
  emits the wakeup that resolves ref.done.

- chat-service-virtual-llm + llm-infer-route assertions updated to
  expect the new {failFast: true} option arg.

mcpd 884/884 (was 881; +3 v5 cases). mcplocal 723/723. Full smoke
suite 144/144 against the deployed queue-backed mcpd.

Stage 3 (next): expose the durable queue via async API endpoints.
POST /api/v1/inference-tasks (enqueue with failFast=false), GET
/api/v1/inference-tasks/:id (poll), GET /api/v1/inference-tasks/:id/stream
(SSE), DELETE /api/v1/inference-tasks/:id (cancel). New `tasks` RBAC
resource.
This commit is contained in:
Michal
2026-04-28 02:33:26 +01:00
parent ed21ad1b5a
commit 7b18bb6d6b
8 changed files with 580 additions and 130 deletions

View File

@@ -193,6 +193,10 @@ describe('ChatService — kind=virtual branch (v3 Stage 1)', () => {
'vllm-local',
expect.objectContaining({ messages: expect.any(Array) }),
false,
// v5: chat.service passes failFast:true so its pool failover loop
// surfaces transport errors quickly instead of waiting on the
// durable queue's 10-min timeout.
{ failFast: true },
);
});
@@ -224,6 +228,7 @@ describe('ChatService — kind=virtual branch (v3 Stage 1)', () => {
'vllm-local',
expect.objectContaining({ messages: expect.any(Array), stream: true }),
true,
{ failFast: true },
);
});

View File

@@ -244,6 +244,9 @@ describe('POST /api/v1/llms/:name/infer', () => {
'claude',
expect.objectContaining({ messages: expect.any(Array) }),
false,
// v5: direct infer route passes failFast:true so a downed publisher
// returns 503 immediately instead of queueing the task durably.
{ failFast: true },
);
});

View File

@@ -1,7 +1,10 @@
import { describe, it, expect, vi } from 'vitest';
import { VirtualLlmService, type VirtualSessionHandle } from '../src/services/virtual-llm.service.js';
import { InferenceTaskService } from '../src/services/inference-task.service.js';
import type { IInferenceTaskService } from '../src/services/inference-task.service.js';
import type { IInferenceTaskRepository } from '../src/repositories/inference-task.repository.js';
import type { ILlmRepository } from '../src/repositories/llm.repository.js';
import type { Llm } from '@prisma/client';
import type { Llm, InferenceTask, InferenceTaskStatus } from '@prisma/client';
function makeLlm(overrides: Partial<Llm> = {}): Llm {
return {
@@ -15,6 +18,7 @@ function makeLlm(overrides: Partial<Llm> = {}): Llm {
apiKeySecretId: null,
apiKeySecretKey: null,
extraConfig: {} as Llm['extraConfig'],
poolName: null,
kind: 'virtual',
providerSessionId: 's-1',
lastHeartbeatAt: new Date(),
@@ -27,6 +31,105 @@ function makeLlm(overrides: Partial<Llm> = {}): Llm {
};
}
/**
* Drop-in mock of `InferenceTaskService` backed by a Map. We mirror just
* enough of the real service's signaling — events for terminal/chunk —
* so VirtualLlmService's enqueue/result flows behave the same way they
* do in production. Nothing here talks to Postgres.
*/
function mockTaskService(): IInferenceTaskService {
// Build a minimal repo for the real InferenceTaskService — that way we
// exercise the actual event-emitter wakeup logic, just without a DB.
const rows = new Map<string, InferenceTask>();
let n = 0;
const repo: IInferenceTaskRepository = {
create: vi.fn(async (data) => {
n += 1;
const row: InferenceTask = {
id: `task-${String(n)}`,
status: 'pending',
poolName: data.poolName,
llmName: data.llmName,
model: data.model,
tier: data.tier ?? null,
claimedBy: null,
requestBody: data.requestBody as InferenceTask['requestBody'],
responseBody: null,
errorMessage: null,
streaming: data.streaming,
createdAt: new Date(),
claimedAt: null,
streamStartedAt: null,
completedAt: null,
ownerId: data.ownerId,
agentId: data.agentId ?? null,
};
rows.set(row.id, row);
return row;
}),
findById: vi.fn(async (id) => rows.get(id) ?? null),
findPendingForPools: vi.fn(async (poolNames: string[]) =>
[...rows.values()].filter((r) => r.status === 'pending' && poolNames.includes(r.poolName)),
),
findHeldBy: vi.fn(async (claimedBy: string) =>
[...rows.values()].filter((r) =>
r.claimedBy === claimedBy
&& (r.status === 'claimed' || r.status === 'running')),
),
list: vi.fn(async () => [...rows.values()]),
tryClaim: vi.fn(async (id, claimedBy, claimedAt) => {
const r = rows.get(id);
if (r === undefined || r.status !== 'pending') return null;
const next = { ...r, status: 'claimed' as InferenceTaskStatus, claimedBy, claimedAt };
rows.set(id, next);
return next;
}),
markRunning: vi.fn(async (id, at) => {
const r = rows.get(id);
if (r === undefined || (r.status !== 'claimed' && r.status !== 'running')) return null;
const next = { ...r, status: 'running' as InferenceTaskStatus, streamStartedAt: at };
rows.set(id, next);
return next;
}),
markCompleted: vi.fn(async (id, body, at) => {
const r = rows.get(id);
if (r === undefined || r.status === 'completed' || r.status === 'error' || r.status === 'cancelled') return null;
const next = { ...r, status: 'completed' as InferenceTaskStatus, responseBody: (body ?? null) as InferenceTask['responseBody'], completedAt: at };
rows.set(id, next);
return next;
}),
markError: vi.fn(async (id, errorMessage, at) => {
const r = rows.get(id);
if (r === undefined || r.status === 'completed' || r.status === 'error' || r.status === 'cancelled') return null;
const next = { ...r, status: 'error' as InferenceTaskStatus, errorMessage, completedAt: at };
rows.set(id, next);
return next;
}),
markCancelled: vi.fn(async (id, at) => {
const r = rows.get(id);
if (r === undefined || r.status === 'completed' || r.status === 'error' || r.status === 'cancelled') return null;
const next = { ...r, status: 'cancelled' as InferenceTaskStatus, completedAt: at };
rows.set(id, next);
return next;
}),
revertToPending: vi.fn(async (id) => {
const r = rows.get(id);
if (r === undefined || (r.status !== 'claimed' && r.status !== 'running')) return null;
const next = { ...r, status: 'pending' as InferenceTaskStatus, claimedBy: null, claimedAt: null, streamStartedAt: null };
rows.set(id, next);
return next;
}),
findStalePending: vi.fn(async () => []),
findExpiredTerminal: vi.fn(async () => []),
deleteMany: vi.fn(async (ids) => {
let c = 0;
for (const id of ids) if (rows.delete(id)) c += 1;
return c;
}),
};
return new InferenceTaskService(repo);
}
function mockRepo(initial: Llm[] = []): ILlmRepository {
const rows = new Map<string, Llm>(initial.map((l) => [l.id, l]));
let counter = rows.size;
@@ -38,6 +141,14 @@ function mockRepo(initial: Llm[] = []): ILlmRepository {
return null;
}),
findByTier: vi.fn(async () => []),
findByPoolName: vi.fn(async (poolName: string) => {
const out: Llm[] = [];
for (const l of rows.values()) {
if (l.poolName === poolName) out.push(l);
else if (l.poolName === null && l.name === poolName) out.push(l);
}
return out;
}),
findBySessionId: vi.fn(async (sid: string) =>
[...rows.values()].filter((l) => l.providerSessionId === sid)),
findStaleVirtuals: vi.fn(async (cutoff: Date) =>
@@ -105,7 +216,7 @@ function fakeSession(): VirtualSessionHandle & { tasks: Array<unknown>; alive: b
describe('VirtualLlmService', () => {
it('register inserts new virtual rows with active status + sessionId', async () => {
const repo = mockRepo();
const svc = new VirtualLlmService(repo);
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
const { providerSessionId, llms } = await svc.register({
providerSessionId: null,
providers: [
@@ -122,7 +233,7 @@ describe('VirtualLlmService', () => {
it('register reuses the same row on sticky reconnect (same name + sessionId)', async () => {
const repo = mockRepo();
const svc = new VirtualLlmService(repo);
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
const first = await svc.register({
providerSessionId: 'fixed-id',
providers: [{ name: 'vllm-local', type: 'openai', model: 'm' }],
@@ -140,7 +251,7 @@ describe('VirtualLlmService', () => {
it('register refuses to overwrite a public LLM with the same name', async () => {
const repo = mockRepo([makeLlm({ name: 'qwen3-thinking', kind: 'public', providerSessionId: null })]);
const svc = new VirtualLlmService(repo);
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
await expect(svc.register({
providerSessionId: 'sess-x',
providers: [{ name: 'qwen3-thinking', type: 'openai', model: 'm' }],
@@ -149,7 +260,7 @@ describe('VirtualLlmService', () => {
it('register refuses if another active session owns the name', async () => {
const repo = mockRepo([makeLlm({ name: 'vllm-local', providerSessionId: 'other', status: 'active' })]);
const svc = new VirtualLlmService(repo);
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
await expect(svc.register({
providerSessionId: 'mine',
providers: [{ name: 'vllm-local', type: 'openai', model: 'm' }],
@@ -161,7 +272,7 @@ describe('VirtualLlmService', () => {
name: 'vllm-local', providerSessionId: 'old-session',
status: 'inactive', inactiveSince: new Date(),
})]);
const svc = new VirtualLlmService(repo);
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
const { llms } = await svc.register({
providerSessionId: 'new-session',
providers: [{ name: 'vllm-local', type: 'openai', model: 'm' }],
@@ -177,7 +288,7 @@ describe('VirtualLlmService', () => {
name: 'vllm-local', providerSessionId: 'sess', status: 'inactive',
lastHeartbeatAt: past, inactiveSince: past,
})]);
const svc = new VirtualLlmService(repo);
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
await svc.heartbeat('sess');
const row = await repo.findByName('vllm-local');
expect(row?.status).toBe('active');
@@ -191,7 +302,7 @@ describe('VirtualLlmService', () => {
makeLlm({ name: 'b', providerSessionId: 'sess' }),
makeLlm({ name: 'c', providerSessionId: 'other' }),
]);
const svc = new VirtualLlmService(repo);
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
svc.bindSession('sess', fakeSession());
await svc.unbindSession('sess');
expect((await repo.findByName('a'))?.status).toBe('inactive');
@@ -201,7 +312,7 @@ describe('VirtualLlmService', () => {
it('enqueueInferTask pushes a task frame to the SSE session', async () => {
const repo = mockRepo([makeLlm({ name: 'vllm-local', providerSessionId: 'sess' })]);
const svc = new VirtualLlmService(repo);
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
const session = fakeSession();
svc.bindSession('sess', session);
@@ -218,26 +329,41 @@ describe('VirtualLlmService', () => {
expect(t.streaming).toBe(false);
});
it('enqueueInferTask rejects when the publisher is offline (no session bound)', async () => {
it('enqueueInferTask queues the task when no session is bound (durable, drains on bind)', async () => {
// v5 semantic change: with a durable queue underneath, "no worker
// up" no longer rejects — the row stays pending and a future
// bindSession drains it. Caller's HTTP handler awaits on ref.done
// and bounds itself with INFER_AWAIT_TIMEOUT_MS; from the service's
// POV the enqueue itself succeeds.
const repo = mockRepo([makeLlm({ name: 'vllm-local', providerSessionId: 'sess' })]);
const svc = new VirtualLlmService(repo);
await expect(
svc.enqueueInferTask('vllm-local', { model: 'm', messages: [] }, false),
).rejects.toThrow(/no live SSE session|publisher offline/);
const tasks = mockTaskService();
const svc = new VirtualLlmService(repo, undefined, tasks);
const ref = await svc.enqueueInferTask('vllm-local', { model: 'm', messages: [] }, false);
expect(ref.taskId).toMatch(/^task-/);
// The row exists and is still pending — no claim happened.
const row = await tasks.findById(ref.taskId);
expect(row?.status).toBe('pending');
expect(row?.claimedBy).toBeNull();
});
it('enqueueInferTask rejects when the row is inactive', async () => {
it('enqueueInferTask still queues against an inactive row (pool may have a sibling worker)', async () => {
// v5 semantic change: status=inactive on a specific Llm doesn't
// mean the pool is dead — another mcplocal publishing the same
// poolName might be active. The dispatcher's bindSession drain
// matches by poolName, so even a "dead" pin queues correctly.
const repo = mockRepo([makeLlm({ name: 'vllm-local', providerSessionId: 'sess', status: 'inactive', inactiveSince: new Date() })]);
const svc = new VirtualLlmService(repo);
svc.bindSession('sess', fakeSession());
await expect(
svc.enqueueInferTask('vllm-local', { model: 'm', messages: [] }, false),
).rejects.toThrow(/inactive|publisher offline/);
const tasks = mockTaskService();
const svc = new VirtualLlmService(repo, undefined, tasks);
const ref = await svc.enqueueInferTask('vllm-local', { model: 'm', messages: [] }, false);
const row = await tasks.findById(ref.taskId);
expect(row?.status).toBe('pending');
// No frame pushed because no session is bound.
expect(row?.claimedBy).toBeNull();
});
it('enqueueInferTask rejects when the LLM is public (not virtual)', async () => {
const repo = mockRepo([makeLlm({ name: 'qwen3-thinking', kind: 'public', providerSessionId: null })]);
const svc = new VirtualLlmService(repo);
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
await expect(
svc.enqueueInferTask('qwen3-thinking', { model: 'm', messages: [] }, false),
).rejects.toThrow(/not a virtual provider/);
@@ -245,7 +371,7 @@ describe('VirtualLlmService', () => {
it('completeTask resolves the pending non-streaming promise', async () => {
const repo = mockRepo([makeLlm({ name: 'vllm-local', providerSessionId: 'sess' })]);
const svc = new VirtualLlmService(repo);
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
svc.bindSession('sess', fakeSession());
const ref = await svc.enqueueInferTask(
'vllm-local',
@@ -258,7 +384,7 @@ describe('VirtualLlmService', () => {
it('streaming: pushTaskChunk fans chunks to subscribers; done resolves the ref', async () => {
const repo = mockRepo([makeLlm({ name: 'vllm-local', providerSessionId: 'sess' })]);
const svc = new VirtualLlmService(repo);
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
svc.bindSession('sess', fakeSession());
const ref = await svc.enqueueInferTask(
'vllm-local',
@@ -278,7 +404,7 @@ describe('VirtualLlmService', () => {
it('failTask rejects the pending promise with a clear error', async () => {
const repo = mockRepo([makeLlm({ name: 'vllm-local', providerSessionId: 'sess' })]);
const svc = new VirtualLlmService(repo);
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
svc.bindSession('sess', fakeSession());
const ref = await svc.enqueueInferTask(
'vllm-local',
@@ -289,17 +415,34 @@ describe('VirtualLlmService', () => {
await expect(ref.done).rejects.toThrow(/upstream blew up/);
});
it('unbindSession rejects in-flight tasks for that session', async () => {
it('unbindSession reverts claimed inference tasks to pending (durable re-queue, not reject)', async () => {
// v5 semantic change: a worker disconnecting mid-task no longer
// *rejects* the task. The row goes back to pending so another
// worker on the same pool can pick it up. The original caller's
// ref.done keeps waiting up to its 10-min INFER_AWAIT_TIMEOUT_MS;
// the same caller is what gets the result whichever worker
// ultimately delivers it.
const repo = mockRepo([makeLlm({ name: 'vllm-local', providerSessionId: 'sess' })]);
const svc = new VirtualLlmService(repo);
const tasks = mockTaskService();
const svc = new VirtualLlmService(repo, undefined, tasks);
svc.bindSession('sess', fakeSession());
const ref = await svc.enqueueInferTask(
'vllm-local',
{ model: 'm', messages: [{ role: 'user', content: 'hi' }] },
false,
);
// After enqueue with a session up, the task is claimed.
let row = await tasks.findById(ref.taskId);
expect(row?.status).toBe('claimed');
expect(row?.claimedBy).toBe('sess');
await svc.unbindSession('sess');
await expect(ref.done).rejects.toThrow(/publisher disconnected/);
// After disconnect, claimed/running rows revert to pending — ready
// for the next worker to drain.
row = await tasks.findById(ref.taskId);
expect(row?.status).toBe('pending');
expect(row?.claimedBy).toBeNull();
});
it('gcSweep flips heartbeat-stale active virtuals to inactive', async () => {
@@ -309,7 +452,7 @@ describe('VirtualLlmService', () => {
makeLlm({ name: 'stale', providerSessionId: 'a', status: 'active', lastHeartbeatAt: long }),
makeLlm({ name: 'fresh', providerSessionId: 'b', status: 'active', lastHeartbeatAt: recent }),
]);
const svc = new VirtualLlmService(repo);
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
const result = await svc.gcSweep();
expect(result.markedInactive).toBe(1);
expect((await repo.findByName('stale'))?.status).toBe('inactive');
@@ -324,7 +467,7 @@ describe('VirtualLlmService', () => {
makeLlm({ name: 'recent', providerSessionId: 'b', status: 'inactive', inactiveSince: fresh }),
makeLlm({ name: 'public-survivor', providerSessionId: null, kind: 'public' }),
]);
const svc = new VirtualLlmService(repo);
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
const result = await svc.gcSweep();
expect(result.deleted).toBe(1);
expect(await repo.findByName('old')).toBeNull();
@@ -336,7 +479,7 @@ describe('VirtualLlmService', () => {
it('hibernating: dispatches a wake task first and waits for it to complete before infer', async () => {
const repo = mockRepo([makeLlm({ name: 'sleeping', providerSessionId: 'sess', status: 'hibernating' })]);
const svc = new VirtualLlmService(repo);
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
const session = fakeSession();
svc.bindSession('sess', session);
@@ -370,7 +513,7 @@ describe('VirtualLlmService', () => {
it('hibernating: concurrent infer requests share a single wake task', async () => {
const repo = mockRepo([makeLlm({ name: 'sleeping', providerSessionId: 'sess', status: 'hibernating' })]);
const svc = new VirtualLlmService(repo);
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
const session = fakeSession();
svc.bindSession('sess', session);
@@ -398,7 +541,7 @@ describe('VirtualLlmService', () => {
it('hibernating: rejects when the wake task fails', async () => {
const repo = mockRepo([makeLlm({ name: 'broken', providerSessionId: 'sess', status: 'hibernating' })]);
const svc = new VirtualLlmService(repo);
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
svc.bindSession('sess', fakeSession());
const inferPromise = svc.enqueueInferTask(
@@ -408,12 +551,12 @@ describe('VirtualLlmService', () => {
);
await new Promise((r) => setTimeout(r, 0));
// Get the wake task id from the in-flight tasks map (its only entry).
// We test the failure path via failTask which is part of the public
// surface used by the result-POST route handler.
// v5: wake tasks live in `wakeTasks` (in-memory). Inference tasks
// moved to the DB-backed queue but wake is publisher-control work
// that doesn't need durability — we kept the in-memory map for it.
const taskIds: string[] = [];
// eslint-disable-next-line @typescript-eslint/no-explicit-any
for (const id of (svc as any).tasksById.keys()) taskIds.push(id);
for (const id of (svc as any).wakeTasks.keys()) taskIds.push(id);
expect(taskIds).toHaveLength(1);
expect(svc.failTask(taskIds[0]!, new Error('wake recipe failed'))).toBe(true);
@@ -424,14 +567,28 @@ describe('VirtualLlmService', () => {
expect(row?.status).toBe('hibernating');
});
it('inactive: still rejects with 503 (publisher offline) — wake path only fires for hibernating', async () => {
it('inactive: queues without firing the wake path — wake only triggers on status=hibernating', async () => {
// Coverage for the v5 inactive-vs-hibernating distinction.
// hibernating = "publisher told us the backend is asleep, ask
// them to wake it"; inactive = "publisher itself is offline".
// For inactive rows, queueing is the right behavior (wait for a
// worker on the pool to come online and drain). The wake path
// must NOT fire — wake is opt-in via the publisher's register
// payload, not a generic "row is down" recovery.
//
// No session bind here: an "inactive" row in production means
// unbindSession already flipped it after SSE close. Binding a
// session for the same providerSessionId would be a contradictory
// setup that the test wouldn't model anything real about.
const repo = mockRepo([makeLlm({ name: 'gone', providerSessionId: 'sess', status: 'inactive', inactiveSince: new Date() })]);
const svc = new VirtualLlmService(repo);
svc.bindSession('sess', fakeSession());
const tasks = mockTaskService();
const svc = new VirtualLlmService(repo, undefined, tasks);
await expect(
svc.enqueueInferTask('gone', { model: 'm', messages: [] }, false),
).rejects.toThrow(/inactive|publisher offline/);
const ref = await svc.enqueueInferTask('gone', { model: 'm', messages: [] }, false);
// Task queued in pending; no claim, no frame.
const row = await tasks.findById(ref.taskId);
expect(row?.status).toBe('pending');
expect(row?.claimedBy).toBeNull();
});
it('gcSweep is idempotent — running twice in a row is a no-op the second time', async () => {
@@ -439,11 +596,75 @@ describe('VirtualLlmService', () => {
const repo = mockRepo([
makeLlm({ name: 'stale', providerSessionId: 'a', status: 'active', lastHeartbeatAt: long }),
]);
const svc = new VirtualLlmService(repo);
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
const first = await svc.gcSweep();
const second = await svc.gcSweep();
expect(first.markedInactive).toBe(1);
expect(second.markedInactive).toBe(0);
expect(second.deleted).toBe(0);
});
// ── v5: durable queue + drain-on-bind ──
it('bindSession drains pending inference tasks owned by the session\'s pool keys', async () => {
// Two enqueues land while no session is bound. Each row is created
// with status=pending; no SSE frame goes anywhere. When the worker
// finally binds, the drain loop claims both and pushes the frames.
const repo = mockRepo([makeLlm({ name: 'vllm-local', providerSessionId: 'sess', poolName: 'qwen-pool' })]);
const tasks = mockTaskService();
const svc = new VirtualLlmService(repo, undefined, tasks);
const ref1 = await svc.enqueueInferTask('vllm-local', { model: 'm', messages: [{ role: 'user', content: 'one' }] }, false);
const ref2 = await svc.enqueueInferTask('vllm-local', { model: 'm', messages: [{ role: 'user', content: 'two' }] }, false);
// Both rows are still pending — no worker bound yet.
expect((await tasks.findById(ref1.taskId))?.status).toBe('pending');
expect((await tasks.findById(ref2.taskId))?.status).toBe('pending');
// Worker shows up. Drain runs synchronously enough that we just
// need to flush the microtask queue before checking SSE frames.
const session = fakeSession();
svc.bindSession('sess', session);
// drainPendingForSession is fired with `void` so let microtasks
// settle before asserting.
await new Promise((r) => setTimeout(r, 0));
expect((session.tasks as Array<{ kind: string; taskId: string }>).map((t) => t.taskId).sort())
.toEqual([ref1.taskId, ref2.taskId].sort());
// Rows are now claimed by this session.
expect((await tasks.findById(ref1.taskId))?.status).toBe('claimed');
expect((await tasks.findById(ref1.taskId))?.claimedBy).toBe('sess');
});
it('drain-on-bind matches the effective pool key, not just llm.name', async () => {
// The pinned Llm has name=vllm-alice but poolName=qwen-pool.
// Enqueue against vllm-alice → row.poolName=qwen-pool.
// Worker binds with a session that owns vllm-alice (same pool key).
// Drain must surface this row even though poolName != name.
const repo = mockRepo([makeLlm({ name: 'vllm-alice', providerSessionId: 'sess', poolName: 'qwen-pool' })]);
const tasks = mockTaskService();
const svc = new VirtualLlmService(repo, undefined, tasks);
const ref = await svc.enqueueInferTask('vllm-alice', { model: 'm', messages: [] }, false);
expect((await tasks.findById(ref.taskId))?.poolName).toBe('qwen-pool');
const session = fakeSession();
svc.bindSession('sess', session);
await new Promise((r) => setTimeout(r, 0));
expect((session.tasks as Array<{ taskId: string }>).map((t) => t.taskId)).toEqual([ref.taskId]);
});
it('completeTask via the result-route updates the DB row + emits the wakeup', async () => {
// End-to-end through the public surface: enqueue → claim happens
// because session is bound → worker POSTs result → completeTask
// routes to InferenceTaskService.complete → ref.done resolves.
const repo = mockRepo([makeLlm({ name: 'vllm-local', providerSessionId: 'sess' })]);
const tasks = mockTaskService();
const svc = new VirtualLlmService(repo, undefined, tasks);
svc.bindSession('sess', fakeSession());
const ref = await svc.enqueueInferTask('vllm-local', { model: 'm', messages: [] }, false);
expect(svc.completeTask(ref.taskId, { status: 200, body: { ok: true } })).toBe(true);
await expect(ref.done).resolves.toEqual({ status: 200, body: { ok: true } });
const row = await tasks.findById(ref.taskId);
expect(row?.status).toBe('completed');
expect(row?.responseBody).toEqual({ ok: true });
});
});