feat(gate): fast no-think selection + dedicated selection Llm #88
@@ -147,10 +147,14 @@ export function registerProjectMcpEndpoint(app: FastifyInstance, mcpdClient: Mcp
|
|||||||
providerRegistry: effectiveRegistry,
|
providerRegistry: effectiveRegistry,
|
||||||
};
|
};
|
||||||
if (resolvedModel) pluginConfig.modelOverride = resolvedModel;
|
if (resolvedModel) pluginConfig.modelOverride = resolvedModel;
|
||||||
// Route gate prompt-selection through this project's server Llm (mcpd
|
// Route gate prompt-selection through a server Llm (mcpd inference proxy)
|
||||||
// inference proxy) so cloud/server keys stay at the k8s level; the local
|
// so cloud/server keys stay at the k8s level; the local personal-token
|
||||||
// personal-token provider is the fallback. See credential-tiering.
|
// provider is the fallback. See credential-tiering. A dedicated fast
|
||||||
if (mcpdConfig.llmProvider) pluginConfig.llmProvider = mcpdConfig.llmProvider;
|
// (no-think) selection Llm can be pinned globally via
|
||||||
|
// MCPCTL_GATE_SELECTION_LLM — it overrides the project's chat llmProvider so
|
||||||
|
// selection stays fast while chat keeps its (thinking) model.
|
||||||
|
const gateSelectionLlm = process.env['MCPCTL_GATE_SELECTION_LLM'] || mcpdConfig.llmProvider;
|
||||||
|
if (gateSelectionLlm) pluginConfig.llmProvider = gateSelectionLlm;
|
||||||
const basePlugin = createDefaultPlugin(pluginConfig);
|
const basePlugin = createDefaultPlugin(pluginConfig);
|
||||||
// Optional favourite-index presentation: curated favourite/<tool> + full
|
// Optional favourite-index presentation: curated favourite/<tool> + full
|
||||||
// all/<server>/<tool> + a "prefer favourite/" instruction. Composed AFTER
|
// all/<server>/<tool> + a "prefer favourite/" instruction. Composed AFTER
|
||||||
|
|||||||
@@ -21,6 +21,26 @@ import { withTimeout, TimeoutError } from '../../util/with-timeout.js';
|
|||||||
* begin_session — on timeout we fall back to deterministic tag matching. */
|
* begin_session — on timeout we fall back to deterministic tag matching. */
|
||||||
const GATE_LLM_TIMEOUT_MS = Number(process.env['MCPCTL_GATE_LLM_TIMEOUT_MS'] ?? '8000');
|
const GATE_LLM_TIMEOUT_MS = Number(process.env['MCPCTL_GATE_LLM_TIMEOUT_MS'] ?? '8000');
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extra request fields for the gate's server-Llm selection call. Prompt
|
||||||
|
* selection is mechanical classification that gains nothing from chain-of-
|
||||||
|
* thought, so we ask reasoning models to skip thinking (they otherwise burn the
|
||||||
|
* whole budget reasoning and blow the gate timeout). Sent verbatim by mcpd's
|
||||||
|
* passthrough adapter → litellm/vLLM. Harmless for models that ignore them, and
|
||||||
|
* if a backend rejects them the selector falls back to the local provider.
|
||||||
|
* - `chat_template_kwargs.enable_thinking:false` — Qwen3 hard-off.
|
||||||
|
* - `reasoning_effort:'low'` — OpenAI o-series / newer vLLM.
|
||||||
|
* Override with a JSON object in MCPCTL_GATE_SELECT_EXTRA_BODY, or '' to disable.
|
||||||
|
*/
|
||||||
|
const GATE_SELECT_EXTRA_BODY: Record<string, unknown> = (() => {
|
||||||
|
const raw = process.env['MCPCTL_GATE_SELECT_EXTRA_BODY'];
|
||||||
|
if (raw === '') return {};
|
||||||
|
if (raw !== undefined) {
|
||||||
|
try { return JSON.parse(raw) as Record<string, unknown>; } catch { /* use default */ }
|
||||||
|
}
|
||||||
|
return { chat_template_kwargs: { enable_thinking: false }, reasoning_effort: 'low' };
|
||||||
|
})();
|
||||||
|
|
||||||
export interface GatePluginConfig {
|
export interface GatePluginConfig {
|
||||||
gated?: boolean;
|
gated?: boolean;
|
||||||
providerRegistry?: ProviderRegistry | null;
|
providerRegistry?: ProviderRegistry | null;
|
||||||
@@ -308,6 +328,7 @@ async function handleBeginSession(
|
|||||||
temperature: o.temperature,
|
temperature: o.temperature,
|
||||||
max_tokens: o.maxTokens,
|
max_tokens: o.maxTokens,
|
||||||
stream: false,
|
stream: false,
|
||||||
|
...GATE_SELECT_EXTRA_BODY, // ask reasoning models for a fast, no-think answer
|
||||||
});
|
});
|
||||||
return pickCompletionText(resp);
|
return pickCompletionText(resp);
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user