feat(mcpd): VirtualLlmService rewires through durable queue (v5 Stage 2)
The in-memory `tasksById` map for inference tasks is gone. Every
inference call lands as a row in `InferenceTask`; the result POST
updates the row + emits a wakeup; the in-flight HTTP handler unblocks
on the wake. mcpd surviving a restart no longer drops in-flight tasks,
and a worker disconnecting mid-task no longer fails the caller — the
row reverts to pending and a sibling worker on the same pool drains it.
Wake tasks (publisher control messages, not inference) keep their own
small in-memory map (`wakeTasks`). They're millisecond-scoped and
don't benefit from durability — a missed wake on restart just means
the next infer fires a fresh wake.
Behavioral changes worth flagging:
- Worker disconnect mid-task: WAS reject ref.done with "publisher
disconnected"; NOW revert claimed/running rows to pending. Original
caller's ref.done keeps waiting up to INFER_AWAIT_TIMEOUT_MS (10
min); whichever worker delivers the result fulfills it.
- bindSession drains pending tasks for the session's pool keys. So
tasks queued while no worker was up automatically get dispatched
when one shows up. The drain matches by *effective pool key*
(poolName ?? name) — tasks queued against vllm-alice get drained
by any session whose owned Llms share alice's pool.
- New `failFast: true` option on enqueueInferTask (default: false).
Existing callers that NEED fast-fail get it explicitly:
- Direct `/api/v1/llms/<name>/infer` route: caller pinned a
specific Llm and wants 503 immediately if the publisher is
offline; queueing for an unknown future worker would surprise.
- chat.service pool failover loop: it iterates pool candidates
and needs each candidate's transport failure to surface fast.
Without failFast, a downed pool member would absorb the call
into the queue and the loop would wait 10 min before trying
the next.
The async API route (Stage 3) leaves failFast=false — that's the
whole point of the durable queue path.
- VirtualLlmService now requires an InferenceTaskService dep at
construction. Older test wirings that didn't pass it get a clear
"InferenceTaskService not wired" error from enqueueInferTask
rather than a confusing in-memory stub.
Tests:
- 12 existing virtual-llm-service tests updated for the new
semantics: "rejects when no session" → "queues durably"; "rejects
when row inactive" → "still queues (pool may have a sibling)";
"unbindSession rejects in-flight tasks" → "reverts to pending".
Wake-task probing now uses `wakeTasks` instead of `tasksById`.
- 3 new v5-specific tests: drain-on-bind matches by effective pool
key (not just name); enqueue without a session keeps the row
pending; completeTask via the result-route updates the DB and
emits the wakeup that resolves ref.done.
- chat-service-virtual-llm + llm-infer-route assertions updated to
expect the new {failFast: true} option arg.
mcpd 884/884 (was 881; +3 v5 cases). mcplocal 723/723. Full smoke
suite 144/144 against the deployed queue-backed mcpd.
Stage 3 (next): expose the durable queue via async API endpoints.
POST /api/v1/inference-tasks (enqueue with failFast=false), GET
/api/v1/inference-tasks/:id (poll), GET /api/v1/inference-tasks/:id/stream
(SSE), DELETE /api/v1/inference-tasks/:id (cancel). New `tasks` RBAC
resource.
This commit is contained in:
@@ -193,6 +193,10 @@ describe('ChatService — kind=virtual branch (v3 Stage 1)', () => {
|
||||
'vllm-local',
|
||||
expect.objectContaining({ messages: expect.any(Array) }),
|
||||
false,
|
||||
// v5: chat.service passes failFast:true so its pool failover loop
|
||||
// surfaces transport errors quickly instead of waiting on the
|
||||
// durable queue's 10-min timeout.
|
||||
{ failFast: true },
|
||||
);
|
||||
});
|
||||
|
||||
@@ -224,6 +228,7 @@ describe('ChatService — kind=virtual branch (v3 Stage 1)', () => {
|
||||
'vllm-local',
|
||||
expect.objectContaining({ messages: expect.any(Array), stream: true }),
|
||||
true,
|
||||
{ failFast: true },
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
@@ -244,6 +244,9 @@ describe('POST /api/v1/llms/:name/infer', () => {
|
||||
'claude',
|
||||
expect.objectContaining({ messages: expect.any(Array) }),
|
||||
false,
|
||||
// v5: direct infer route passes failFast:true so a downed publisher
|
||||
// returns 503 immediately instead of queueing the task durably.
|
||||
{ failFast: true },
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import { VirtualLlmService, type VirtualSessionHandle } from '../src/services/virtual-llm.service.js';
|
||||
import { InferenceTaskService } from '../src/services/inference-task.service.js';
|
||||
import type { IInferenceTaskService } from '../src/services/inference-task.service.js';
|
||||
import type { IInferenceTaskRepository } from '../src/repositories/inference-task.repository.js';
|
||||
import type { ILlmRepository } from '../src/repositories/llm.repository.js';
|
||||
import type { Llm } from '@prisma/client';
|
||||
import type { Llm, InferenceTask, InferenceTaskStatus } from '@prisma/client';
|
||||
|
||||
function makeLlm(overrides: Partial<Llm> = {}): Llm {
|
||||
return {
|
||||
@@ -15,6 +18,7 @@ function makeLlm(overrides: Partial<Llm> = {}): Llm {
|
||||
apiKeySecretId: null,
|
||||
apiKeySecretKey: null,
|
||||
extraConfig: {} as Llm['extraConfig'],
|
||||
poolName: null,
|
||||
kind: 'virtual',
|
||||
providerSessionId: 's-1',
|
||||
lastHeartbeatAt: new Date(),
|
||||
@@ -27,6 +31,105 @@ function makeLlm(overrides: Partial<Llm> = {}): Llm {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop-in mock of `InferenceTaskService` backed by a Map. We mirror just
|
||||
* enough of the real service's signaling — events for terminal/chunk —
|
||||
* so VirtualLlmService's enqueue/result flows behave the same way they
|
||||
* do in production. Nothing here talks to Postgres.
|
||||
*/
|
||||
function mockTaskService(): IInferenceTaskService {
|
||||
// Build a minimal repo for the real InferenceTaskService — that way we
|
||||
// exercise the actual event-emitter wakeup logic, just without a DB.
|
||||
const rows = new Map<string, InferenceTask>();
|
||||
let n = 0;
|
||||
const repo: IInferenceTaskRepository = {
|
||||
create: vi.fn(async (data) => {
|
||||
n += 1;
|
||||
const row: InferenceTask = {
|
||||
id: `task-${String(n)}`,
|
||||
status: 'pending',
|
||||
poolName: data.poolName,
|
||||
llmName: data.llmName,
|
||||
model: data.model,
|
||||
tier: data.tier ?? null,
|
||||
claimedBy: null,
|
||||
requestBody: data.requestBody as InferenceTask['requestBody'],
|
||||
responseBody: null,
|
||||
errorMessage: null,
|
||||
streaming: data.streaming,
|
||||
createdAt: new Date(),
|
||||
claimedAt: null,
|
||||
streamStartedAt: null,
|
||||
completedAt: null,
|
||||
ownerId: data.ownerId,
|
||||
agentId: data.agentId ?? null,
|
||||
};
|
||||
rows.set(row.id, row);
|
||||
return row;
|
||||
}),
|
||||
findById: vi.fn(async (id) => rows.get(id) ?? null),
|
||||
findPendingForPools: vi.fn(async (poolNames: string[]) =>
|
||||
[...rows.values()].filter((r) => r.status === 'pending' && poolNames.includes(r.poolName)),
|
||||
),
|
||||
findHeldBy: vi.fn(async (claimedBy: string) =>
|
||||
[...rows.values()].filter((r) =>
|
||||
r.claimedBy === claimedBy
|
||||
&& (r.status === 'claimed' || r.status === 'running')),
|
||||
),
|
||||
list: vi.fn(async () => [...rows.values()]),
|
||||
tryClaim: vi.fn(async (id, claimedBy, claimedAt) => {
|
||||
const r = rows.get(id);
|
||||
if (r === undefined || r.status !== 'pending') return null;
|
||||
const next = { ...r, status: 'claimed' as InferenceTaskStatus, claimedBy, claimedAt };
|
||||
rows.set(id, next);
|
||||
return next;
|
||||
}),
|
||||
markRunning: vi.fn(async (id, at) => {
|
||||
const r = rows.get(id);
|
||||
if (r === undefined || (r.status !== 'claimed' && r.status !== 'running')) return null;
|
||||
const next = { ...r, status: 'running' as InferenceTaskStatus, streamStartedAt: at };
|
||||
rows.set(id, next);
|
||||
return next;
|
||||
}),
|
||||
markCompleted: vi.fn(async (id, body, at) => {
|
||||
const r = rows.get(id);
|
||||
if (r === undefined || r.status === 'completed' || r.status === 'error' || r.status === 'cancelled') return null;
|
||||
const next = { ...r, status: 'completed' as InferenceTaskStatus, responseBody: (body ?? null) as InferenceTask['responseBody'], completedAt: at };
|
||||
rows.set(id, next);
|
||||
return next;
|
||||
}),
|
||||
markError: vi.fn(async (id, errorMessage, at) => {
|
||||
const r = rows.get(id);
|
||||
if (r === undefined || r.status === 'completed' || r.status === 'error' || r.status === 'cancelled') return null;
|
||||
const next = { ...r, status: 'error' as InferenceTaskStatus, errorMessage, completedAt: at };
|
||||
rows.set(id, next);
|
||||
return next;
|
||||
}),
|
||||
markCancelled: vi.fn(async (id, at) => {
|
||||
const r = rows.get(id);
|
||||
if (r === undefined || r.status === 'completed' || r.status === 'error' || r.status === 'cancelled') return null;
|
||||
const next = { ...r, status: 'cancelled' as InferenceTaskStatus, completedAt: at };
|
||||
rows.set(id, next);
|
||||
return next;
|
||||
}),
|
||||
revertToPending: vi.fn(async (id) => {
|
||||
const r = rows.get(id);
|
||||
if (r === undefined || (r.status !== 'claimed' && r.status !== 'running')) return null;
|
||||
const next = { ...r, status: 'pending' as InferenceTaskStatus, claimedBy: null, claimedAt: null, streamStartedAt: null };
|
||||
rows.set(id, next);
|
||||
return next;
|
||||
}),
|
||||
findStalePending: vi.fn(async () => []),
|
||||
findExpiredTerminal: vi.fn(async () => []),
|
||||
deleteMany: vi.fn(async (ids) => {
|
||||
let c = 0;
|
||||
for (const id of ids) if (rows.delete(id)) c += 1;
|
||||
return c;
|
||||
}),
|
||||
};
|
||||
return new InferenceTaskService(repo);
|
||||
}
|
||||
|
||||
function mockRepo(initial: Llm[] = []): ILlmRepository {
|
||||
const rows = new Map<string, Llm>(initial.map((l) => [l.id, l]));
|
||||
let counter = rows.size;
|
||||
@@ -38,6 +141,14 @@ function mockRepo(initial: Llm[] = []): ILlmRepository {
|
||||
return null;
|
||||
}),
|
||||
findByTier: vi.fn(async () => []),
|
||||
findByPoolName: vi.fn(async (poolName: string) => {
|
||||
const out: Llm[] = [];
|
||||
for (const l of rows.values()) {
|
||||
if (l.poolName === poolName) out.push(l);
|
||||
else if (l.poolName === null && l.name === poolName) out.push(l);
|
||||
}
|
||||
return out;
|
||||
}),
|
||||
findBySessionId: vi.fn(async (sid: string) =>
|
||||
[...rows.values()].filter((l) => l.providerSessionId === sid)),
|
||||
findStaleVirtuals: vi.fn(async (cutoff: Date) =>
|
||||
@@ -105,7 +216,7 @@ function fakeSession(): VirtualSessionHandle & { tasks: Array<unknown>; alive: b
|
||||
describe('VirtualLlmService', () => {
|
||||
it('register inserts new virtual rows with active status + sessionId', async () => {
|
||||
const repo = mockRepo();
|
||||
const svc = new VirtualLlmService(repo);
|
||||
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
|
||||
const { providerSessionId, llms } = await svc.register({
|
||||
providerSessionId: null,
|
||||
providers: [
|
||||
@@ -122,7 +233,7 @@ describe('VirtualLlmService', () => {
|
||||
|
||||
it('register reuses the same row on sticky reconnect (same name + sessionId)', async () => {
|
||||
const repo = mockRepo();
|
||||
const svc = new VirtualLlmService(repo);
|
||||
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
|
||||
const first = await svc.register({
|
||||
providerSessionId: 'fixed-id',
|
||||
providers: [{ name: 'vllm-local', type: 'openai', model: 'm' }],
|
||||
@@ -140,7 +251,7 @@ describe('VirtualLlmService', () => {
|
||||
|
||||
it('register refuses to overwrite a public LLM with the same name', async () => {
|
||||
const repo = mockRepo([makeLlm({ name: 'qwen3-thinking', kind: 'public', providerSessionId: null })]);
|
||||
const svc = new VirtualLlmService(repo);
|
||||
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
|
||||
await expect(svc.register({
|
||||
providerSessionId: 'sess-x',
|
||||
providers: [{ name: 'qwen3-thinking', type: 'openai', model: 'm' }],
|
||||
@@ -149,7 +260,7 @@ describe('VirtualLlmService', () => {
|
||||
|
||||
it('register refuses if another active session owns the name', async () => {
|
||||
const repo = mockRepo([makeLlm({ name: 'vllm-local', providerSessionId: 'other', status: 'active' })]);
|
||||
const svc = new VirtualLlmService(repo);
|
||||
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
|
||||
await expect(svc.register({
|
||||
providerSessionId: 'mine',
|
||||
providers: [{ name: 'vllm-local', type: 'openai', model: 'm' }],
|
||||
@@ -161,7 +272,7 @@ describe('VirtualLlmService', () => {
|
||||
name: 'vllm-local', providerSessionId: 'old-session',
|
||||
status: 'inactive', inactiveSince: new Date(),
|
||||
})]);
|
||||
const svc = new VirtualLlmService(repo);
|
||||
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
|
||||
const { llms } = await svc.register({
|
||||
providerSessionId: 'new-session',
|
||||
providers: [{ name: 'vllm-local', type: 'openai', model: 'm' }],
|
||||
@@ -177,7 +288,7 @@ describe('VirtualLlmService', () => {
|
||||
name: 'vllm-local', providerSessionId: 'sess', status: 'inactive',
|
||||
lastHeartbeatAt: past, inactiveSince: past,
|
||||
})]);
|
||||
const svc = new VirtualLlmService(repo);
|
||||
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
|
||||
await svc.heartbeat('sess');
|
||||
const row = await repo.findByName('vllm-local');
|
||||
expect(row?.status).toBe('active');
|
||||
@@ -191,7 +302,7 @@ describe('VirtualLlmService', () => {
|
||||
makeLlm({ name: 'b', providerSessionId: 'sess' }),
|
||||
makeLlm({ name: 'c', providerSessionId: 'other' }),
|
||||
]);
|
||||
const svc = new VirtualLlmService(repo);
|
||||
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
|
||||
svc.bindSession('sess', fakeSession());
|
||||
await svc.unbindSession('sess');
|
||||
expect((await repo.findByName('a'))?.status).toBe('inactive');
|
||||
@@ -201,7 +312,7 @@ describe('VirtualLlmService', () => {
|
||||
|
||||
it('enqueueInferTask pushes a task frame to the SSE session', async () => {
|
||||
const repo = mockRepo([makeLlm({ name: 'vllm-local', providerSessionId: 'sess' })]);
|
||||
const svc = new VirtualLlmService(repo);
|
||||
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
|
||||
const session = fakeSession();
|
||||
svc.bindSession('sess', session);
|
||||
|
||||
@@ -218,26 +329,41 @@ describe('VirtualLlmService', () => {
|
||||
expect(t.streaming).toBe(false);
|
||||
});
|
||||
|
||||
it('enqueueInferTask rejects when the publisher is offline (no session bound)', async () => {
|
||||
it('enqueueInferTask queues the task when no session is bound (durable, drains on bind)', async () => {
|
||||
// v5 semantic change: with a durable queue underneath, "no worker
|
||||
// up" no longer rejects — the row stays pending and a future
|
||||
// bindSession drains it. Caller's HTTP handler awaits on ref.done
|
||||
// and bounds itself with INFER_AWAIT_TIMEOUT_MS; from the service's
|
||||
// POV the enqueue itself succeeds.
|
||||
const repo = mockRepo([makeLlm({ name: 'vllm-local', providerSessionId: 'sess' })]);
|
||||
const svc = new VirtualLlmService(repo);
|
||||
await expect(
|
||||
svc.enqueueInferTask('vllm-local', { model: 'm', messages: [] }, false),
|
||||
).rejects.toThrow(/no live SSE session|publisher offline/);
|
||||
const tasks = mockTaskService();
|
||||
const svc = new VirtualLlmService(repo, undefined, tasks);
|
||||
const ref = await svc.enqueueInferTask('vllm-local', { model: 'm', messages: [] }, false);
|
||||
expect(ref.taskId).toMatch(/^task-/);
|
||||
// The row exists and is still pending — no claim happened.
|
||||
const row = await tasks.findById(ref.taskId);
|
||||
expect(row?.status).toBe('pending');
|
||||
expect(row?.claimedBy).toBeNull();
|
||||
});
|
||||
|
||||
it('enqueueInferTask rejects when the row is inactive', async () => {
|
||||
it('enqueueInferTask still queues against an inactive row (pool may have a sibling worker)', async () => {
|
||||
// v5 semantic change: status=inactive on a specific Llm doesn't
|
||||
// mean the pool is dead — another mcplocal publishing the same
|
||||
// poolName might be active. The dispatcher's bindSession drain
|
||||
// matches by poolName, so even a "dead" pin queues correctly.
|
||||
const repo = mockRepo([makeLlm({ name: 'vllm-local', providerSessionId: 'sess', status: 'inactive', inactiveSince: new Date() })]);
|
||||
const svc = new VirtualLlmService(repo);
|
||||
svc.bindSession('sess', fakeSession());
|
||||
await expect(
|
||||
svc.enqueueInferTask('vllm-local', { model: 'm', messages: [] }, false),
|
||||
).rejects.toThrow(/inactive|publisher offline/);
|
||||
const tasks = mockTaskService();
|
||||
const svc = new VirtualLlmService(repo, undefined, tasks);
|
||||
const ref = await svc.enqueueInferTask('vllm-local', { model: 'm', messages: [] }, false);
|
||||
const row = await tasks.findById(ref.taskId);
|
||||
expect(row?.status).toBe('pending');
|
||||
// No frame pushed because no session is bound.
|
||||
expect(row?.claimedBy).toBeNull();
|
||||
});
|
||||
|
||||
it('enqueueInferTask rejects when the LLM is public (not virtual)', async () => {
|
||||
const repo = mockRepo([makeLlm({ name: 'qwen3-thinking', kind: 'public', providerSessionId: null })]);
|
||||
const svc = new VirtualLlmService(repo);
|
||||
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
|
||||
await expect(
|
||||
svc.enqueueInferTask('qwen3-thinking', { model: 'm', messages: [] }, false),
|
||||
).rejects.toThrow(/not a virtual provider/);
|
||||
@@ -245,7 +371,7 @@ describe('VirtualLlmService', () => {
|
||||
|
||||
it('completeTask resolves the pending non-streaming promise', async () => {
|
||||
const repo = mockRepo([makeLlm({ name: 'vllm-local', providerSessionId: 'sess' })]);
|
||||
const svc = new VirtualLlmService(repo);
|
||||
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
|
||||
svc.bindSession('sess', fakeSession());
|
||||
const ref = await svc.enqueueInferTask(
|
||||
'vllm-local',
|
||||
@@ -258,7 +384,7 @@ describe('VirtualLlmService', () => {
|
||||
|
||||
it('streaming: pushTaskChunk fans chunks to subscribers; done resolves the ref', async () => {
|
||||
const repo = mockRepo([makeLlm({ name: 'vllm-local', providerSessionId: 'sess' })]);
|
||||
const svc = new VirtualLlmService(repo);
|
||||
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
|
||||
svc.bindSession('sess', fakeSession());
|
||||
const ref = await svc.enqueueInferTask(
|
||||
'vllm-local',
|
||||
@@ -278,7 +404,7 @@ describe('VirtualLlmService', () => {
|
||||
|
||||
it('failTask rejects the pending promise with a clear error', async () => {
|
||||
const repo = mockRepo([makeLlm({ name: 'vllm-local', providerSessionId: 'sess' })]);
|
||||
const svc = new VirtualLlmService(repo);
|
||||
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
|
||||
svc.bindSession('sess', fakeSession());
|
||||
const ref = await svc.enqueueInferTask(
|
||||
'vllm-local',
|
||||
@@ -289,17 +415,34 @@ describe('VirtualLlmService', () => {
|
||||
await expect(ref.done).rejects.toThrow(/upstream blew up/);
|
||||
});
|
||||
|
||||
it('unbindSession rejects in-flight tasks for that session', async () => {
|
||||
it('unbindSession reverts claimed inference tasks to pending (durable re-queue, not reject)', async () => {
|
||||
// v5 semantic change: a worker disconnecting mid-task no longer
|
||||
// *rejects* the task. The row goes back to pending so another
|
||||
// worker on the same pool can pick it up. The original caller's
|
||||
// ref.done keeps waiting up to its 10-min INFER_AWAIT_TIMEOUT_MS;
|
||||
// the same caller is what gets the result whichever worker
|
||||
// ultimately delivers it.
|
||||
const repo = mockRepo([makeLlm({ name: 'vllm-local', providerSessionId: 'sess' })]);
|
||||
const svc = new VirtualLlmService(repo);
|
||||
const tasks = mockTaskService();
|
||||
const svc = new VirtualLlmService(repo, undefined, tasks);
|
||||
svc.bindSession('sess', fakeSession());
|
||||
const ref = await svc.enqueueInferTask(
|
||||
'vllm-local',
|
||||
{ model: 'm', messages: [{ role: 'user', content: 'hi' }] },
|
||||
false,
|
||||
);
|
||||
// After enqueue with a session up, the task is claimed.
|
||||
let row = await tasks.findById(ref.taskId);
|
||||
expect(row?.status).toBe('claimed');
|
||||
expect(row?.claimedBy).toBe('sess');
|
||||
|
||||
await svc.unbindSession('sess');
|
||||
await expect(ref.done).rejects.toThrow(/publisher disconnected/);
|
||||
|
||||
// After disconnect, claimed/running rows revert to pending — ready
|
||||
// for the next worker to drain.
|
||||
row = await tasks.findById(ref.taskId);
|
||||
expect(row?.status).toBe('pending');
|
||||
expect(row?.claimedBy).toBeNull();
|
||||
});
|
||||
|
||||
it('gcSweep flips heartbeat-stale active virtuals to inactive', async () => {
|
||||
@@ -309,7 +452,7 @@ describe('VirtualLlmService', () => {
|
||||
makeLlm({ name: 'stale', providerSessionId: 'a', status: 'active', lastHeartbeatAt: long }),
|
||||
makeLlm({ name: 'fresh', providerSessionId: 'b', status: 'active', lastHeartbeatAt: recent }),
|
||||
]);
|
||||
const svc = new VirtualLlmService(repo);
|
||||
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
|
||||
const result = await svc.gcSweep();
|
||||
expect(result.markedInactive).toBe(1);
|
||||
expect((await repo.findByName('stale'))?.status).toBe('inactive');
|
||||
@@ -324,7 +467,7 @@ describe('VirtualLlmService', () => {
|
||||
makeLlm({ name: 'recent', providerSessionId: 'b', status: 'inactive', inactiveSince: fresh }),
|
||||
makeLlm({ name: 'public-survivor', providerSessionId: null, kind: 'public' }),
|
||||
]);
|
||||
const svc = new VirtualLlmService(repo);
|
||||
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
|
||||
const result = await svc.gcSweep();
|
||||
expect(result.deleted).toBe(1);
|
||||
expect(await repo.findByName('old')).toBeNull();
|
||||
@@ -336,7 +479,7 @@ describe('VirtualLlmService', () => {
|
||||
|
||||
it('hibernating: dispatches a wake task first and waits for it to complete before infer', async () => {
|
||||
const repo = mockRepo([makeLlm({ name: 'sleeping', providerSessionId: 'sess', status: 'hibernating' })]);
|
||||
const svc = new VirtualLlmService(repo);
|
||||
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
|
||||
const session = fakeSession();
|
||||
svc.bindSession('sess', session);
|
||||
|
||||
@@ -370,7 +513,7 @@ describe('VirtualLlmService', () => {
|
||||
|
||||
it('hibernating: concurrent infer requests share a single wake task', async () => {
|
||||
const repo = mockRepo([makeLlm({ name: 'sleeping', providerSessionId: 'sess', status: 'hibernating' })]);
|
||||
const svc = new VirtualLlmService(repo);
|
||||
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
|
||||
const session = fakeSession();
|
||||
svc.bindSession('sess', session);
|
||||
|
||||
@@ -398,7 +541,7 @@ describe('VirtualLlmService', () => {
|
||||
|
||||
it('hibernating: rejects when the wake task fails', async () => {
|
||||
const repo = mockRepo([makeLlm({ name: 'broken', providerSessionId: 'sess', status: 'hibernating' })]);
|
||||
const svc = new VirtualLlmService(repo);
|
||||
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
|
||||
svc.bindSession('sess', fakeSession());
|
||||
|
||||
const inferPromise = svc.enqueueInferTask(
|
||||
@@ -408,12 +551,12 @@ describe('VirtualLlmService', () => {
|
||||
);
|
||||
await new Promise((r) => setTimeout(r, 0));
|
||||
|
||||
// Get the wake task id from the in-flight tasks map (its only entry).
|
||||
// We test the failure path via failTask which is part of the public
|
||||
// surface used by the result-POST route handler.
|
||||
// v5: wake tasks live in `wakeTasks` (in-memory). Inference tasks
|
||||
// moved to the DB-backed queue but wake is publisher-control work
|
||||
// that doesn't need durability — we kept the in-memory map for it.
|
||||
const taskIds: string[] = [];
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
for (const id of (svc as any).tasksById.keys()) taskIds.push(id);
|
||||
for (const id of (svc as any).wakeTasks.keys()) taskIds.push(id);
|
||||
expect(taskIds).toHaveLength(1);
|
||||
expect(svc.failTask(taskIds[0]!, new Error('wake recipe failed'))).toBe(true);
|
||||
|
||||
@@ -424,14 +567,28 @@ describe('VirtualLlmService', () => {
|
||||
expect(row?.status).toBe('hibernating');
|
||||
});
|
||||
|
||||
it('inactive: still rejects with 503 (publisher offline) — wake path only fires for hibernating', async () => {
|
||||
it('inactive: queues without firing the wake path — wake only triggers on status=hibernating', async () => {
|
||||
// Coverage for the v5 inactive-vs-hibernating distinction.
|
||||
// hibernating = "publisher told us the backend is asleep, ask
|
||||
// them to wake it"; inactive = "publisher itself is offline".
|
||||
// For inactive rows, queueing is the right behavior (wait for a
|
||||
// worker on the pool to come online and drain). The wake path
|
||||
// must NOT fire — wake is opt-in via the publisher's register
|
||||
// payload, not a generic "row is down" recovery.
|
||||
//
|
||||
// No session bind here: an "inactive" row in production means
|
||||
// unbindSession already flipped it after SSE close. Binding a
|
||||
// session for the same providerSessionId would be a contradictory
|
||||
// setup that the test wouldn't model anything real about.
|
||||
const repo = mockRepo([makeLlm({ name: 'gone', providerSessionId: 'sess', status: 'inactive', inactiveSince: new Date() })]);
|
||||
const svc = new VirtualLlmService(repo);
|
||||
svc.bindSession('sess', fakeSession());
|
||||
const tasks = mockTaskService();
|
||||
const svc = new VirtualLlmService(repo, undefined, tasks);
|
||||
|
||||
await expect(
|
||||
svc.enqueueInferTask('gone', { model: 'm', messages: [] }, false),
|
||||
).rejects.toThrow(/inactive|publisher offline/);
|
||||
const ref = await svc.enqueueInferTask('gone', { model: 'm', messages: [] }, false);
|
||||
// Task queued in pending; no claim, no frame.
|
||||
const row = await tasks.findById(ref.taskId);
|
||||
expect(row?.status).toBe('pending');
|
||||
expect(row?.claimedBy).toBeNull();
|
||||
});
|
||||
|
||||
it('gcSweep is idempotent — running twice in a row is a no-op the second time', async () => {
|
||||
@@ -439,11 +596,75 @@ describe('VirtualLlmService', () => {
|
||||
const repo = mockRepo([
|
||||
makeLlm({ name: 'stale', providerSessionId: 'a', status: 'active', lastHeartbeatAt: long }),
|
||||
]);
|
||||
const svc = new VirtualLlmService(repo);
|
||||
const svc = new VirtualLlmService(repo, undefined, mockTaskService());
|
||||
const first = await svc.gcSweep();
|
||||
const second = await svc.gcSweep();
|
||||
expect(first.markedInactive).toBe(1);
|
||||
expect(second.markedInactive).toBe(0);
|
||||
expect(second.deleted).toBe(0);
|
||||
});
|
||||
|
||||
// ── v5: durable queue + drain-on-bind ──
|
||||
|
||||
it('bindSession drains pending inference tasks owned by the session\'s pool keys', async () => {
|
||||
// Two enqueues land while no session is bound. Each row is created
|
||||
// with status=pending; no SSE frame goes anywhere. When the worker
|
||||
// finally binds, the drain loop claims both and pushes the frames.
|
||||
const repo = mockRepo([makeLlm({ name: 'vllm-local', providerSessionId: 'sess', poolName: 'qwen-pool' })]);
|
||||
const tasks = mockTaskService();
|
||||
const svc = new VirtualLlmService(repo, undefined, tasks);
|
||||
const ref1 = await svc.enqueueInferTask('vllm-local', { model: 'm', messages: [{ role: 'user', content: 'one' }] }, false);
|
||||
const ref2 = await svc.enqueueInferTask('vllm-local', { model: 'm', messages: [{ role: 'user', content: 'two' }] }, false);
|
||||
// Both rows are still pending — no worker bound yet.
|
||||
expect((await tasks.findById(ref1.taskId))?.status).toBe('pending');
|
||||
expect((await tasks.findById(ref2.taskId))?.status).toBe('pending');
|
||||
|
||||
// Worker shows up. Drain runs synchronously enough that we just
|
||||
// need to flush the microtask queue before checking SSE frames.
|
||||
const session = fakeSession();
|
||||
svc.bindSession('sess', session);
|
||||
// drainPendingForSession is fired with `void` so let microtasks
|
||||
// settle before asserting.
|
||||
await new Promise((r) => setTimeout(r, 0));
|
||||
|
||||
expect((session.tasks as Array<{ kind: string; taskId: string }>).map((t) => t.taskId).sort())
|
||||
.toEqual([ref1.taskId, ref2.taskId].sort());
|
||||
// Rows are now claimed by this session.
|
||||
expect((await tasks.findById(ref1.taskId))?.status).toBe('claimed');
|
||||
expect((await tasks.findById(ref1.taskId))?.claimedBy).toBe('sess');
|
||||
});
|
||||
|
||||
it('drain-on-bind matches the effective pool key, not just llm.name', async () => {
|
||||
// The pinned Llm has name=vllm-alice but poolName=qwen-pool.
|
||||
// Enqueue against vllm-alice → row.poolName=qwen-pool.
|
||||
// Worker binds with a session that owns vllm-alice (same pool key).
|
||||
// Drain must surface this row even though poolName != name.
|
||||
const repo = mockRepo([makeLlm({ name: 'vllm-alice', providerSessionId: 'sess', poolName: 'qwen-pool' })]);
|
||||
const tasks = mockTaskService();
|
||||
const svc = new VirtualLlmService(repo, undefined, tasks);
|
||||
const ref = await svc.enqueueInferTask('vllm-alice', { model: 'm', messages: [] }, false);
|
||||
expect((await tasks.findById(ref.taskId))?.poolName).toBe('qwen-pool');
|
||||
|
||||
const session = fakeSession();
|
||||
svc.bindSession('sess', session);
|
||||
await new Promise((r) => setTimeout(r, 0));
|
||||
expect((session.tasks as Array<{ taskId: string }>).map((t) => t.taskId)).toEqual([ref.taskId]);
|
||||
});
|
||||
|
||||
it('completeTask via the result-route updates the DB row + emits the wakeup', async () => {
|
||||
// End-to-end through the public surface: enqueue → claim happens
|
||||
// because session is bound → worker POSTs result → completeTask
|
||||
// routes to InferenceTaskService.complete → ref.done resolves.
|
||||
const repo = mockRepo([makeLlm({ name: 'vllm-local', providerSessionId: 'sess' })]);
|
||||
const tasks = mockTaskService();
|
||||
const svc = new VirtualLlmService(repo, undefined, tasks);
|
||||
svc.bindSession('sess', fakeSession());
|
||||
const ref = await svc.enqueueInferTask('vllm-local', { model: 'm', messages: [] }, false);
|
||||
|
||||
expect(svc.completeTask(ref.taskId, { status: 200, body: { ok: true } })).toBe(true);
|
||||
await expect(ref.done).resolves.toEqual({ status: 200, body: { ok: true } });
|
||||
const row = await tasks.findById(ref.taskId);
|
||||
expect(row?.status).toBe('completed');
|
||||
expect(row?.responseBody).toEqual({ ok: true });
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user