From 27480181bfe4f4503b5da759ec73ca80ccc37b2d Mon Sep 17 00:00:00 2001 From: Michal Date: Fri, 24 Jul 2026 13:07:27 +0100 Subject: [PATCH 1/2] feat(gate): ask reasoning models for a fast, no-think selection response MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prompt selection is mechanical classification — it gains nothing from chain-of- thought, and a reasoning model (qwen3-thinking) otherwise burns its whole budget reasoning (~40s, 6k+ chars) and blows the gate's 8s timeout. The gate's server selection request now includes thinking-suppression hints, forwarded verbatim by mcpd's passthrough adapter to litellm/vLLM: chat_template_kwargs.enable_thinking=false (Qwen3 hard-off) reasoning_effort=low (o-series / newer vLLM) Harmless for models that ignore them; if a backend rejects them the selector falls back to the local provider. Override via MCPCTL_GATE_SELECT_EXTRA_BODY (JSON), or '' to disable. NOT yet validated live — the qwen3-thinking vLLM is crashlooping again (0/1, HTTP 500), and whether litellm forwards chat_template_kwargs to vLLM is unconfirmed (the /no_think prompt directive was NOT honored). Validate when the backend recovers. mcpld gate/selector tests green (75). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/mcplocal/src/proxymodel/plugins/gate.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/mcplocal/src/proxymodel/plugins/gate.ts b/src/mcplocal/src/proxymodel/plugins/gate.ts index 78ef5d9..eafd54e 100644 --- a/src/mcplocal/src/proxymodel/plugins/gate.ts +++ b/src/mcplocal/src/proxymodel/plugins/gate.ts @@ -21,6 +21,26 @@ import { withTimeout, TimeoutError } from '../../util/with-timeout.js'; * begin_session — on timeout we fall back to deterministic tag matching. */ const GATE_LLM_TIMEOUT_MS = Number(process.env['MCPCTL_GATE_LLM_TIMEOUT_MS'] ?? '8000'); +/** + * Extra request fields for the gate's server-Llm selection call. Prompt + * selection is mechanical classification that gains nothing from chain-of- + * thought, so we ask reasoning models to skip thinking (they otherwise burn the + * whole budget reasoning and blow the gate timeout). Sent verbatim by mcpd's + * passthrough adapter → litellm/vLLM. Harmless for models that ignore them, and + * if a backend rejects them the selector falls back to the local provider. + * - `chat_template_kwargs.enable_thinking:false` — Qwen3 hard-off. + * - `reasoning_effort:'low'` — OpenAI o-series / newer vLLM. + * Override with a JSON object in MCPCTL_GATE_SELECT_EXTRA_BODY, or '' to disable. + */ +const GATE_SELECT_EXTRA_BODY: Record = (() => { + const raw = process.env['MCPCTL_GATE_SELECT_EXTRA_BODY']; + if (raw === '') return {}; + if (raw !== undefined) { + try { return JSON.parse(raw) as Record; } catch { /* use default */ } + } + return { chat_template_kwargs: { enable_thinking: false }, reasoning_effort: 'low' }; +})(); + export interface GatePluginConfig { gated?: boolean; providerRegistry?: ProviderRegistry | null; @@ -308,6 +328,7 @@ async function handleBeginSession( temperature: o.temperature, max_tokens: o.maxTokens, stream: false, + ...GATE_SELECT_EXTRA_BODY, // ask reasoning models for a fast, no-think answer }); return pickCompletionText(resp); } From 5225b54901d91297277ee5a4054c56180f82b7da Mon Sep 17 00:00:00 2001 From: Michal Date: Fri, 24 Jul 2026 14:28:44 +0100 Subject: [PATCH 2/2] =?UTF-8?q?feat(gate):=20MCPCTL=5FGATE=5FSELECTION=5FL?= =?UTF-8?q?LM=20=E2=80=94=20pin=20a=20dedicated=20fast=20selection=20Llm?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lets ops route gate prompt-selection at a fast (no-think) server Llm independent of the project's chat llmProvider — so selection stays ~1-2s while chat keeps its thinking model. Overrides the project llmProvider for the gate's server-selection path; unset → prior behavior (use the project's llmProvider). Pairs with the litellm qwen3-fast no-think alias (kubernetes-deployment) + a `vllm-fast` mcpd Llm. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/mcplocal/src/http/project-mcp-endpoint.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/mcplocal/src/http/project-mcp-endpoint.ts b/src/mcplocal/src/http/project-mcp-endpoint.ts index 4a38d37..274b6b7 100644 --- a/src/mcplocal/src/http/project-mcp-endpoint.ts +++ b/src/mcplocal/src/http/project-mcp-endpoint.ts @@ -147,10 +147,14 @@ export function registerProjectMcpEndpoint(app: FastifyInstance, mcpdClient: Mcp providerRegistry: effectiveRegistry, }; if (resolvedModel) pluginConfig.modelOverride = resolvedModel; - // Route gate prompt-selection through this project's server Llm (mcpd - // inference proxy) so cloud/server keys stay at the k8s level; the local - // personal-token provider is the fallback. See credential-tiering. - if (mcpdConfig.llmProvider) pluginConfig.llmProvider = mcpdConfig.llmProvider; + // Route gate prompt-selection through a server Llm (mcpd inference proxy) + // so cloud/server keys stay at the k8s level; the local personal-token + // provider is the fallback. See credential-tiering. A dedicated fast + // (no-think) selection Llm can be pinned globally via + // MCPCTL_GATE_SELECTION_LLM — it overrides the project's chat llmProvider so + // selection stays fast while chat keeps its (thinking) model. + const gateSelectionLlm = process.env['MCPCTL_GATE_SELECTION_LLM'] || mcpdConfig.llmProvider; + if (gateSelectionLlm) pluginConfig.llmProvider = gateSelectionLlm; const basePlugin = createDefaultPlugin(pluginConfig); // Optional favourite-index presentation: curated favourite/ + full // all// + a "prefer favourite/" instruction. Composed AFTER