From a432e9e19f287c702183dbe70560f2b0fb7d9c6e Mon Sep 17 00:00:00 2001 From: Michal Date: Fri, 24 Jul 2026 21:38:08 +0100 Subject: [PATCH] feat(gate): terse selection prompt so a no-think model answers in ~2s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With the fast (no-think) route, gate-selection latency is now proportional to the ANSWER length (a no-think model runs ~10-13 tok/s), not hidden thinking. A verbose "reasoning" field pushed a selection to ~9s (over the 8s budget). Request compact JSON with a <=8-word reasoning → ~2s (vs ~9s prose, or ~1.2s with no reasoning). Validated live against vllm-fast (glm-4.6-reap-fast): 2.0s, valid selection JSON, zero reasoning tokens. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/mcplocal/src/gate/llm-selector.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/mcplocal/src/gate/llm-selector.ts b/src/mcplocal/src/gate/llm-selector.ts index c857351..45fe2b0 100644 --- a/src/mcplocal/src/gate/llm-selector.ts +++ b/src/mcplocal/src/gate/llm-selector.ts @@ -95,7 +95,11 @@ export class LlmPromptSelector { ): Promise { const { getSystemPromptFn, signal, serverInfer } = opts; - const DEFAULT_SYSTEM_PROMPT = `You are a context selection assistant. Given a developer's task keywords and a list of available project prompts, select which prompts are relevant to their work. Return a JSON object with "selectedNames" (array of prompt names) and "reasoning" (brief explanation). Priority 10 prompts must always be included.`; + // Keep the answer SHORT: latency is proportional to output length (a + // no-think model runs ~10-13 tok/s), and a verbose reasoning field pushes a + // selection past the gate's responsiveness budget. Ask for compact JSON with + // a <=8-word reasoning — ~2s vs ~9s for a prose reasoning. + const DEFAULT_SYSTEM_PROMPT = `You are a context selection assistant. Given a developer's task keywords and available project prompts, select the relevant ones. Return ONLY compact JSON: {"selectedNames":[...],"reasoning":"<=8 words"}. No prose outside the JSON. Priority 10 prompts must always be included.`; const systemPrompt = getSystemPromptFn ? await getSystemPromptFn('llm-gate-context-selector', DEFAULT_SYSTEM_PROMPT) : DEFAULT_SYSTEM_PROMPT; -- 2.49.1