From f614e9bb98d94c79dac3a53328fd31ae6fa06ceb Mon Sep 17 00:00:00 2001 From: Michal Date: Thu, 23 Jul 2026 01:20:30 +0100 Subject: [PATCH] feat(proxy): favourite-index tool presentation (favourite/ + all/ + prefer instruction) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured winner from the DGX-Spark bake-off (toolsim.py, 145-tool catalog): a curated favourite/ shortlist + the full all// catalog + a load-bearing "prefer favourite/ first" instruction nearly halved wander (37→20) and 2.5x'd first-pick (2→5/8) vs a flat catalog. The instruction is load-bearing; enriching descriptions did not help. - New mcplocal plugin `favourite-index.ts`: composes AFTER gate (no-ops while gated), reshapes the ungated upstream catalog into favourite/ + all/, injects the instruction (onInitialize), and rewrites presented names back to canonical server/tool in onToolCallBefore so normal routing + content-pipeline still run. Gate/agent virtual tools pass through untouched; favourites are upstream-only. - compose.ts: onInitialize now concatenates plugin instructions (was first-non-null) so favindex can contribute its banner alongside the gate's. - Per-project config `Project.favouriteIndex` {enabled, tools[], maxFavourites}; surfaced to the proxy via discovery; wired at project-mcp-endpoint when enabled. - Usage derivation: mcpd tool-usage ranking over tool_call_trace events (normalizing presented names → canonical), GET /api/v1/audit/tool-usage, and `mcpctl favourites suggest|list`. - CLI: `create project` gains --favourite/--favourite-index/--max-favourites; favouriteIndex round-trips through get -o yaml | apply -f. Completions regenerated. - Tests: plugin unit (presentation, rewrite routing, gated no-op, collisions), compose merge, canonicalizeToolName, buildFavouriteIndex, + a live smoke test. - Docs: docs/tool-presentation.md. Co-Authored-By: Claude Opus 4.8 (1M context) --- completions/mcpctl.bash | 22 ++- completions/mcpctl.fish | 16 +- docs/tool-presentation.md | 86 ++++++++ scripts/generate-completions.ts | 2 +- src/cli/src/commands/apply.ts | 7 + src/cli/src/commands/create.ts | 37 ++++ src/cli/src/commands/favourites.ts | 105 ++++++++++ src/cli/src/index.ts | 2 + src/cli/tests/favourite-index-opts.test.ts | 31 +++ src/db/prisma/schema.prisma | 1 + .../repositories/audit-event.repository.ts | 50 ++++- src/mcpd/src/repositories/interfaces.ts | 9 + .../src/repositories/project.repository.ts | 5 +- src/mcpd/src/routes/audit-events.ts | 9 + src/mcpd/src/services/audit-event.service.ts | 11 ++ .../src/services/backup/yaml-serializer.ts | 4 +- src/mcpd/src/services/project.service.ts | 3 + src/mcpd/src/validation/project.schema.ts | 12 ++ src/mcpd/tests/tool-usage.test.ts | 30 +++ src/mcplocal/src/discovery.ts | 13 ++ src/mcplocal/src/http/project-mcp-endpoint.ts | 20 +- .../src/proxymodel/plugins/compose.ts | 9 +- .../src/proxymodel/plugins/favourite-index.ts | 139 +++++++++++++ .../tests/plugin-favourite-index.test.ts | 185 ++++++++++++++++++ .../tests/smoke/favourite-index.test.ts | 125 ++++++++++++ 25 files changed, 919 insertions(+), 14 deletions(-) create mode 100644 docs/tool-presentation.md create mode 100644 src/cli/src/commands/favourites.ts create mode 100644 src/cli/tests/favourite-index-opts.test.ts create mode 100644 src/mcpd/tests/tool-usage.test.ts create mode 100644 src/mcplocal/src/proxymodel/plugins/favourite-index.ts create mode 100644 src/mcplocal/tests/plugin-favourite-index.test.ts create mode 100644 src/mcplocal/tests/smoke/favourite-index.test.ts diff --git a/completions/mcpctl.bash b/completions/mcpctl.bash index e7ecdbc..b544435 100644 --- a/completions/mcpctl.bash +++ b/completions/mcpctl.bash @@ -6,7 +6,7 @@ _mcpctl() { _init_completion || return local commands="status login logout config get describe delete logs create edit apply chat chat-llm patch passwd errors backup approve review skills console cache provider test migrate rotate" - local project_commands="get describe delete logs create edit attach-server detach-server" + local project_commands="get describe delete logs create edit attach-server detach-server favourites" local global_opts="-v --version --daemon-url --direct -p --project -h --help" local resources="servers instances secrets secretbackends llms agents personalities templates projects users groups rbac prompts promptrequests serverattachments proxymodels inference-tasks all" local resource_aliases="servers instances secrets secretbackends llms agents personalities templates projects users groups rbac prompts promptrequests serverattachments proxymodels inference-tasks all server srv instance inst secret sec secretbackend sb llm agent personality template tpl project proj user group rbac-definition rbac-binding prompt promptrequest pr serverattachment sa proxymodel pm task tasks inference-task" @@ -194,7 +194,7 @@ _mcpctl() { COMPREPLY=($(compgen -W "--type --description --default --url --namespace --mount --path-prefix --auth --token-secret --role --auth-mount --sa-token-path --config --wizard --setup-token --policy-name --token-role --no-promote-default --force -h --help" -- "$cur")) ;; project) - COMPREPLY=($(compgen -W "-d --description --proxy-model --prompt --llm --llm-model --gated --no-gated --server --force -h --help" -- "$cur")) + COMPREPLY=($(compgen -W "-d --description --proxy-model --prompt --llm --llm-model --gated --no-gated --server --favourite --favourite-index --no-favourite-index --max-favourites --force -h --help" -- "$cur")) ;; user) COMPREPLY=($(compgen -W "--password --name --force -h --help" -- "$cur")) @@ -323,6 +323,24 @@ _mcpctl() { COMPREPLY=($(compgen -W "$names -h --help" -- "$cur")) fi return ;; + favourites) + local favourites_sub=$(_mcpctl_get_subcmd $subcmd_pos) + if [[ -z "$favourites_sub" ]]; then + COMPREPLY=($(compgen -W "suggest list help" -- "$cur")) + else + case "$favourites_sub" in + suggest) + COMPREPLY=($(compgen -W "--top --window -h --help" -- "$cur")) + ;; + list) + COMPREPLY=($(compgen -W "-h --help" -- "$cur")) + ;; + *) + COMPREPLY=($(compgen -W "-h --help" -- "$cur")) + ;; + esac + fi + return ;; review) local review_sub=$(_mcpctl_get_subcmd $subcmd_pos) if [[ -z "$review_sub" ]]; then diff --git a/completions/mcpctl.fish b/completions/mcpctl.fish index 78caa5f..ff85934 100644 --- a/completions/mcpctl.fish +++ b/completions/mcpctl.fish @@ -5,7 +5,7 @@ complete -c mcpctl -e set -l commands status login logout config get describe delete logs create edit apply chat chat-llm patch passwd errors backup approve review skills console cache provider test migrate rotate -set -l project_commands get describe delete logs create edit attach-server detach-server +set -l project_commands get describe delete logs create edit attach-server detach-server favourites # Disable file completions by default complete -c mcpctl -f @@ -256,6 +256,7 @@ complete -c mcpctl -n "__mcpctl_has_project; and not __fish_seen_subcommand_from complete -c mcpctl -n "__mcpctl_has_project; and not __fish_seen_subcommand_from $project_commands" -a edit -d 'Edit a resource in your default editor (server, project)' complete -c mcpctl -n "__mcpctl_has_project; and not __fish_seen_subcommand_from $project_commands" -a attach-server -d 'Attach a server to a project (requires --project)' complete -c mcpctl -n "__mcpctl_has_project; and not __fish_seen_subcommand_from $project_commands" -a detach-server -d 'Detach a server from a project (requires --project)' +complete -c mcpctl -n "__mcpctl_has_project; and not __fish_seen_subcommand_from $project_commands" -a favourites -d 'Inspect / derive a project\'s favourite-index tool shortlist (requires --project)' # Resource types — only when resource type not yet selected complete -c mcpctl -n "__fish_seen_subcommand_from get describe delete patch; and __mcpctl_needs_resource_type" -a "$resources" -d 'Resource type' @@ -395,6 +396,10 @@ complete -c mcpctl -n "__mcpctl_subcmd_active create project" -l llm-model -d 'O complete -c mcpctl -n "__mcpctl_subcmd_active create project" -l gated -d '[deprecated: use --proxy-model default]' complete -c mcpctl -n "__mcpctl_subcmd_active create project" -l no-gated -d '[deprecated: use --proxy-model content-pipeline]' complete -c mcpctl -n "__mcpctl_subcmd_active create project" -l server -d 'Server name (repeat for multiple)' -x +complete -c mcpctl -n "__mcpctl_subcmd_active create project" -l favourite -d 'Pin a tool to the favourite/ shortlist (repeat; enables favourite-index)' -x +complete -c mcpctl -n "__mcpctl_subcmd_active create project" -l favourite-index -d 'Enable the favourite/ + all/ tool presentation' +complete -c mcpctl -n "__mcpctl_subcmd_active create project" -l no-favourite-index -d 'Disable the favourite-index presentation' +complete -c mcpctl -n "__mcpctl_subcmd_active create project" -l max-favourites -d 'Cap how many favourites are presented' -x complete -c mcpctl -n "__mcpctl_subcmd_active create project" -l force -d 'Update if already exists' # create user options @@ -461,6 +466,15 @@ complete -c mcpctl -n "__fish_seen_subcommand_from backup; and not __fish_seen_s # backup log options complete -c mcpctl -n "__mcpctl_subcmd_active backup log" -s n -l limit -d 'number of commits to show' -x +# favourites subcommands +set -l favourites_cmds suggest list +complete -c mcpctl -n "__fish_seen_subcommand_from favourites; and not __fish_seen_subcommand_from $favourites_cmds" -a suggest -d 'Rank tools by usage to pick favourite/ pins' +complete -c mcpctl -n "__fish_seen_subcommand_from favourites; and not __fish_seen_subcommand_from $favourites_cmds" -a list -d 'Show the favourite-index config the proxy will present' + +# favourites suggest options +complete -c mcpctl -n "__mcpctl_subcmd_active favourites suggest" -l top -d 'How many to show (default 20)' -x +complete -c mcpctl -n "__mcpctl_subcmd_active favourites suggest" -l window -d 'Look-back window in days (default 30)' -x + # review subcommands set -l review_cmds pending next show approve reject diff complete -c mcpctl -n "__fish_seen_subcommand_from review; and not __fish_seen_subcommand_from $review_cmds" -a pending -d 'List pending proposals' diff --git a/docs/tool-presentation.md b/docs/tool-presentation.md new file mode 100644 index 0000000..ea45853 --- /dev/null +++ b/docs/tool-presentation.md @@ -0,0 +1,86 @@ +# Tool presentation: the favourite-index + +Big, flat MCP tool catalogs make LLMs **wander** — they call wrong tools before +the right one, or find it then keep exploring. mcpctl (the proxy) decides *what +tool list the model sees*, so presentation is a lever we can pull. + +We measured it. On a 145-tool catalog with a wandering model +(`kubernetes-deployment/scripts/model-eval/toolsim.py`), the winning presentation +— **favindex** — nearly halved wander (37→20) and 2.5×'d first-pick (2→5/8) vs the +flat catalog. Enriching descriptions did **not** help; scoping/indexing did. +(Full table + method in the `feedback_llm_tool_scoping` memory.) + +## What it does + +When enabled for a project, the proxy presents tools in **two namespaces** plus a +**load-bearing instruction**: + +- `favourite/` — a curated, usage-weighted shortlist of the commonly-used + tools, listed **first**. +- `all//` — the **full** catalog (the escape hatch). +- An `initialize` instruction: *"PREFER favourite/ … use all// + only if nothing in favourites fits."* + +The instruction is not optional — the same two namespaces **without** it scored +materially worse in the bake-off. Curate **and** tell the model to check +favourites first. + +Favourites are a *subset*: leave rare/cloud tools in `all/` only, so the model +falls back correctly for the long tail. + +## How it composes + +favourite-index is a proxymodel plugin +(`src/mcplocal/src/proxymodel/plugins/favourite-index.ts`) composed **after** the +gate/default plugin: + +- While a session is **gated**, only `begin_session` is visible — favourite-index + no-ops. After `begin_session` ungates, the catalog is reshaped into + `favourite/` + `all/`. +- Gate/agent virtual tools (`read_prompts`, `propose_prompt`, `agent-` + chat) pass through unchanged at the top level. Favourites are drawn only from + the upstream MCP catalog. +- Calls to `favourite/…` / `all/…` are rewritten back to the canonical + `server/tool` in `onToolCallBefore`, so normal upstream routing **and** the + content-pipeline (pagination/sectioning) still run. + +## Configuring it + +Per-project, stored on the project as `favouriteIndex` +(`{ enabled, tools: ["server/tool", …], maxFavourites? }`). Round-trips through +`get project -o yaml | apply -f -`. + +```bash +# enable + pin the common tools +mcpctl create project myproj --force --favourite-index \ + --favourite k8s/get_pods --favourite k8s/get_pod_logs \ + --favourite gitea/create_pull_request --max-favourites 15 + +# what the proxy will present +mcpctl favourites list --project myproj +``` + +### Deriving favourites from usage + +Every tool call is recorded (`AuditEvent` `tool_call_trace`). Rank them and pin +the common ones: + +```bash +mcpctl favourites suggest --project myproj --top 20 --window 30 +``` + +This normalizes presented names back to canonical `server/tool` (so `all/…` and +`favourite/…` calls collapse), ranks by call count, and prints a ready-to-run +`create project … --favourite …` line. `★` marks tools already pinned. + +## Validating a change first + +Before shipping a presentation change, run it through the fake-catalog harness — +it caught "enrichment doesn't help" in ~1h instead of a wasted build: + +```bash +toolsim.py favindex all # vs `terse all` (flat baseline) +``` + +Then confirm it reproduces through the **real** proxy on a tool-heavy project +(presentation + routing: `src/mcplocal/tests/smoke/favourite-index.test.ts`). diff --git a/scripts/generate-completions.ts b/scripts/generate-completions.ts index 8fe21e6..497b0a4 100644 --- a/scripts/generate-completions.ts +++ b/scripts/generate-completions.ts @@ -66,7 +66,7 @@ const FILE_OPTIONS = new Set([ ]); /** Commands shown ONLY when --project/-p is on the command line. */ -const PROJECT_ONLY_COMMANDS = new Set(['attach-server', 'detach-server']); +const PROJECT_ONLY_COMMANDS = new Set(['attach-server', 'detach-server', 'favourites']); /** Commands that appear in BOTH project and non-project context. */ const PROJECT_SCOPED_COMMANDS = new Set([ diff --git a/src/cli/src/commands/apply.ts b/src/cli/src/commands/apply.ts index ecb7b63..5360a1a 100644 --- a/src/cli/src/commands/apply.ts +++ b/src/cli/src/commands/apply.ts @@ -197,6 +197,13 @@ const ProjectSpecSchema = z.object({ // model when unset. llmModel: z.string().optional(), servers: z.array(z.string()).default([]), + // Two-namespace tool presentation: curated favourite/ + full + // all//. `tools` are canonical `server/tool` pins. + favouriteIndex: z.object({ + enabled: z.boolean().optional(), + tools: z.array(z.string()).optional(), + maxFavourites: z.number().int().positive().optional(), + }).nullable().optional(), }); const McpTokenSpecSchema = z.object({ diff --git a/src/cli/src/commands/create.ts b/src/cli/src/commands/create.ts index 0a93f26..6f64801 100644 --- a/src/cli/src/commands/create.ts +++ b/src/cli/src/commands/create.ts @@ -11,6 +11,37 @@ function collect(value: string, prev: string[]): string[] { return [...prev, value]; } +export interface FavouriteIndexOpts { + favourite?: string[]; + favouriteIndex?: boolean; + maxFavourites?: string; +} + +/** + * Build a project `favouriteIndex` config from CLI flags, or undefined when no + * favourite-index flag was given (so the field is only sent when intended). + */ +export function buildFavouriteIndex( + opts: FavouriteIndexOpts, +): { enabled: boolean; tools?: string[]; maxFavourites?: number } | undefined { + const tools = opts.favourite ?? []; + const hasFlag = opts.favouriteIndex !== undefined || tools.length > 0 || opts.maxFavourites !== undefined; + if (!hasFlag) return undefined; + // --no-favourite-index → false; any other favourite flag implies enabled. + const result: { enabled: boolean; tools?: string[]; maxFavourites?: number } = { + enabled: opts.favouriteIndex !== false, + }; + if (tools.length > 0) result.tools = tools; + if (opts.maxFavourites !== undefined) { + const n = Number(opts.maxFavourites); + if (!Number.isInteger(n) || n <= 0) { + throw new Error(`Invalid --max-favourites '${opts.maxFavourites}'. Expected a positive integer.`); + } + result.maxFavourites = n; + } + return result; +} + /** * Parse a `--ttl` value. * @@ -521,6 +552,10 @@ export function createCreateCommand(deps: CreateCommandDeps): Command { .option('--gated', '[deprecated: use --proxy-model default]') .option('--no-gated', '[deprecated: use --proxy-model content-pipeline]') .option('--server ', 'Server name (repeat for multiple)', collect, []) + .option('--favourite ', 'Pin a tool to the favourite/ shortlist (repeat; enables favourite-index)', collect, []) + .option('--favourite-index', 'Enable the favourite/ + all/ tool presentation') + .option('--no-favourite-index', 'Disable the favourite-index presentation') + .option('--max-favourites ', 'Cap how many favourites are presented') .option('--force', 'Update if already exists') .action(async (name: string, opts) => { const body: Record = { @@ -539,6 +574,8 @@ export function createCreateCommand(deps: CreateCommandDeps): Command { if (opts.server.length > 0) body.servers = opts.server; if (opts.llm) body.llmProvider = opts.llm; if (opts.llmModel) body.llmModel = opts.llmModel; + const favIndex = buildFavouriteIndex(opts as FavouriteIndexOpts); + if (favIndex) body.favouriteIndex = favIndex; try { const project = await client.post<{ id: string; name: string }>('/api/v1/projects', body); diff --git a/src/cli/src/commands/favourites.ts b/src/cli/src/commands/favourites.ts new file mode 100644 index 0000000..f1ee097 --- /dev/null +++ b/src/cli/src/commands/favourites.ts @@ -0,0 +1,105 @@ +import { Command } from 'commander'; +import type { ApiClient } from '../api-client.js'; +import { resolveNameOrId } from './shared.js'; + +export interface FavouritesCommandDeps { + client: ApiClient; + log: (...args: string[]) => void; + getProject: () => string | undefined; +} + +interface ToolUsageEntry { + tool: string; + server: string | null; + count: number; +} + +interface ProjectFavouriteIndex { + favouriteIndex?: { enabled?: boolean; tools?: string[]; maxFavourites?: number }; +} + +function requireProject(deps: FavouritesCommandDeps): string { + const project = deps.getProject(); + if (!project) { + deps.log('Error: --project is required for this command.'); + process.exitCode = 1; + throw new Error('--project required'); + } + return project; +} + +/** + * `mcpctl favourites` — inspect + derive the curated favourite/ shortlist used + * by the favourite-index tool presentation. `suggest` ranks tools by real usage + * so you can pick pins; `list` shows what the proxy will present. + */ +export function createFavouritesCommand(deps: FavouritesCommandDeps): Command { + const { client, log } = deps; + + const cmd = new Command('favourites') + .description('Inspect / derive a project\'s favourite-index tool shortlist (requires --project)'); + + cmd.command('suggest') + .description('Rank tools by usage to pick favourite/ pins') + .option('--top ', 'How many to show (default 20)', '20') + .option('--window ', 'Look-back window in days (default 30)', '30') + .action(async (opts: { top: string; window: string }) => { + const projectName = requireProject(deps); + const top = Number(opts.top) > 0 ? Number(opts.top) : 20; + const windowDays = Number(opts.window) > 0 ? Number(opts.window) : 30; + const res = await client.get<{ tools: ToolUsageEntry[] }>( + `/api/v1/audit/tool-usage?projectName=${encodeURIComponent(projectName)}&window=${windowDays}&limit=${top}`, + ); + const tools = res.tools ?? []; + + // Mark which candidates are already pinned. + const project = await client.get( + `/api/v1/projects/${await resolveNameOrId(client, 'projects', projectName)}`, + ); + const pinned = new Set(project.favouriteIndex?.tools ?? []); + + if (tools.length === 0) { + log(`No tool-call usage recorded for project '${projectName}' in the last ${windowDays}d.`); + return; + } + log(`Top ${tools.length} tools for '${projectName}' (last ${windowDays}d):`); + log(''); + log(` ${'#'.padStart(3)} ${'CALLS'.padStart(6)} ${'PIN'.padEnd(3)} TOOL`); + log(` ${'-'.repeat(3)} ${'-'.repeat(6)} ${'-'.repeat(3)} ${'-'.repeat(30)}`); + tools.forEach((t, i) => { + const mark = pinned.has(t.tool) ? '★' : ' '; + log(` ${String(i + 1).padStart(3)} ${String(t.count).padStart(6)} ${mark} ${t.tool}`); + }); + const unpinned = tools.filter((t) => !pinned.has(t.tool) && !t.tool.startsWith('favourite/')); + if (unpinned.length > 0) { + log(''); + log('Pin the common ones (★ = already pinned):'); + const flags = unpinned.slice(0, 10).map((t) => `--favourite ${t.tool}`).join(' '); + log(` mcpctl create project ${projectName} --force --favourite-index ${flags}`); + } + }); + + cmd.command('list') + .description('Show the favourite-index config the proxy will present') + .action(async () => { + const projectName = requireProject(deps); + const project = await client.get( + `/api/v1/projects/${await resolveNameOrId(client, 'projects', projectName)}`, + ); + const fav = project.favouriteIndex; + if (!fav || !fav.enabled) { + log(`favourite-index is DISABLED for project '${projectName}'.`); + log(`Enable it with: mcpctl create project ${projectName} --force --favourite-index --favourite ...`); + return; + } + const tools = fav.tools ?? []; + log(`favourite-index ENABLED for '${projectName}'${fav.maxFavourites ? ` (cap ${fav.maxFavourites})` : ''}:`); + if (tools.length === 0) { + log(' (no pins — nothing will be promoted to favourite/)'); + } else { + tools.forEach((t, i) => log(` ${String(i + 1).padStart(3)} ${t}`)); + } + }); + + return cmd; +} diff --git a/src/cli/src/index.ts b/src/cli/src/index.ts index 935153f..7227068 100644 --- a/src/cli/src/index.ts +++ b/src/cli/src/index.ts @@ -14,6 +14,7 @@ import { createEditCommand } from './commands/edit.js'; import { createBackupCommand } from './commands/backup.js'; import { createLoginCommand, createLogoutCommand } from './commands/auth.js'; import { createAttachServerCommand, createDetachServerCommand, createApproveCommand } from './commands/project-ops.js'; +import { createFavouritesCommand } from './commands/favourites.js'; import { createMcpCommand } from './commands/mcp.js'; import { createPatchCommand } from './commands/patch.js'; import { createConsoleCommand } from './commands/console/index.js'; @@ -290,6 +291,7 @@ export function createProgram(): Command { program.addCommand(createAttachServerCommand(projectOpsDeps), { hidden: true }); program.addCommand(createDetachServerCommand(projectOpsDeps), { hidden: true }); program.addCommand(createApproveCommand(projectOpsDeps)); + program.addCommand(createFavouritesCommand(projectOpsDeps)); // PR-4: reviewer queue for proposed prompts + skills. program.addCommand(createReviewCommand({ client, diff --git a/src/cli/tests/favourite-index-opts.test.ts b/src/cli/tests/favourite-index-opts.test.ts new file mode 100644 index 0000000..c1d114d --- /dev/null +++ b/src/cli/tests/favourite-index-opts.test.ts @@ -0,0 +1,31 @@ +import { describe, it, expect } from 'vitest'; +import { buildFavouriteIndex } from '../src/commands/create.js'; + +describe('buildFavouriteIndex — CLI flags → project favouriteIndex', () => { + it('returns undefined when no favourite flag is given', () => { + expect(buildFavouriteIndex({})).toBeUndefined(); + }); + + it('enables and sets the pins when --favourite is given', () => { + expect(buildFavouriteIndex({ favourite: ['k8s/get_pods', 'vault/read_secret'] })).toEqual({ + enabled: true, + tools: ['k8s/get_pods', 'vault/read_secret'], + }); + }); + + it('--no-favourite-index disables', () => { + expect(buildFavouriteIndex({ favouriteIndex: false })).toEqual({ enabled: false }); + }); + + it('parses --max-favourites into an integer', () => { + expect(buildFavouriteIndex({ favouriteIndex: true, maxFavourites: '15' })).toEqual({ + enabled: true, + maxFavourites: 15, + }); + }); + + it('throws on a non-positive-integer --max-favourites', () => { + expect(() => buildFavouriteIndex({ maxFavourites: 'abc' })).toThrow(/max-favourites/); + expect(() => buildFavouriteIndex({ maxFavourites: '0' })).toThrow(/max-favourites/); + }); +}); diff --git a/src/db/prisma/schema.prisma b/src/db/prisma/schema.prisma index aae073d..b8de194 100644 --- a/src/db/prisma/schema.prisma +++ b/src/db/prisma/schema.prisma @@ -323,6 +323,7 @@ model Project { llmProvider String? llmModel String? serverOverrides Json? + favouriteIndex Json? ownerId String version Int @default(1) createdAt DateTime @default(now()) diff --git a/src/mcpd/src/repositories/audit-event.repository.ts b/src/mcpd/src/repositories/audit-event.repository.ts index 4fe1bb7..f5e1aa9 100644 --- a/src/mcpd/src/repositories/audit-event.repository.ts +++ b/src/mcpd/src/repositories/audit-event.repository.ts @@ -1,5 +1,29 @@ import type { PrismaClient, AuditEvent, Prisma } from '@prisma/client'; -import type { IAuditEventRepository, AuditEventFilter, AuditEventCreateInput, AuditSessionSummary } from './interfaces.js'; +import type { IAuditEventRepository, AuditEventFilter, AuditEventCreateInput, AuditSessionSummary, ToolUsageEntry } from './interfaces.js'; + +/** + * Normalize a called tool name to its canonical `server/tool` identity so + * usage counts collapse across the presentation namespaces. The favourite-index + * mode logs the *presented* name (`all//` or `favourite/`) + * in the audit trace, so `all/…` is stripped back to canonical; `favourite/…` + * is already-pinned and kept distinct. `serverName` is the trace's resolved + * server (null for presented names). + */ +export function canonicalizeToolName( + toolName: string, + serverName: string | null, +): { key: string; server: string | null } { + if (toolName.startsWith('all/')) { + const rest = toolName.slice('all/'.length); + const i = rest.indexOf('/'); + return { key: rest, server: i < 0 ? null : rest.slice(0, i) }; + } + if (toolName.startsWith('favourite/')) { + return { key: toolName, server: serverName }; + } + const i = toolName.indexOf('/'); + return { key: toolName, server: i < 0 ? serverName : toolName.slice(0, i) }; +} export class AuditEventRepository implements IAuditEventRepository { constructor(private readonly prisma: PrismaClient) {} @@ -105,6 +129,30 @@ export class AuditEventRepository implements IAuditEventRepository { })); } + async toolUsage(projectName: string, from: Date, sampleLimit = 10000): Promise { + // toolName lives in the payload JSON (un-indexed), so aggregate in-app over + // a bounded recent sample rather than a JSON groupBy. + const rows = await this.prisma.auditEvent.findMany({ + where: { projectName, eventKind: 'tool_call_trace', timestamp: { gte: from } }, + select: { serverName: true, payload: true }, + orderBy: { timestamp: 'desc' }, + take: sampleLimit, + }); + const counts = new Map(); + for (const r of rows) { + const toolName = (r.payload as { toolName?: unknown } | null)?.toolName; + if (typeof toolName !== 'string' || toolName === '') continue; + const { key, server } = canonicalizeToolName(toolName, r.serverName); + const cur = counts.get(key) ?? { server, count: 0 }; + cur.count += 1; + if (cur.server === null && server !== null) cur.server = server; + counts.set(key, cur); + } + return [...counts.entries()] + .map(([tool, v]) => ({ tool, server: v.server, count: v.count })) + .sort((a, b) => b.count - a.count); + } + async countSessions(filter?: { projectName?: string; userName?: string; from?: Date; to?: Date }): Promise { const where: Prisma.AuditEventWhereInput = {}; if (filter?.projectName !== undefined) where.projectName = filter.projectName; diff --git a/src/mcpd/src/repositories/interfaces.ts b/src/mcpd/src/repositories/interfaces.ts index 919e3ed..6d94852 100644 --- a/src/mcpd/src/repositories/interfaces.ts +++ b/src/mcpd/src/repositories/interfaces.ts @@ -92,6 +92,13 @@ export interface AuditSessionSummary { eventKinds: string[]; } +/** A tool ranked by how often it was invoked (canonical `server/tool`). */ +export interface ToolUsageEntry { + tool: string; + server: string | null; + count: number; +} + export interface IAuditEventRepository { findAll(filter?: AuditEventFilter): Promise; findById(id: string): Promise; @@ -99,6 +106,8 @@ export interface IAuditEventRepository { count(filter?: AuditEventFilter): Promise; listSessions(filter?: { projectName?: string; userName?: string; from?: Date; to?: Date; limit?: number; offset?: number }): Promise; countSessions(filter?: { projectName?: string; userName?: string; from?: Date; to?: Date }): Promise; + /** Rank tools by invocation count for a project (from tool_call_trace events). */ + toolUsage(projectName: string, from: Date, sampleLimit?: number): Promise; } // ── MCP Tokens ── diff --git a/src/mcpd/src/repositories/project.repository.ts b/src/mcpd/src/repositories/project.repository.ts index fd086fd..0c6ca04 100644 --- a/src/mcpd/src/repositories/project.repository.ts +++ b/src/mcpd/src/repositories/project.repository.ts @@ -12,7 +12,7 @@ export interface IProjectRepository { findAll(ownerId?: string): Promise; findById(id: string): Promise; findByName(name: string): Promise; - create(data: { name: string; description: string; prompt?: string; ownerId: string; proxyModel?: string; gated?: boolean; llmProvider?: string; llmModel?: string; serverOverrides?: Record }): Promise; + create(data: { name: string; description: string; prompt?: string; ownerId: string; proxyModel?: string; gated?: boolean; llmProvider?: string; llmModel?: string; serverOverrides?: Record; favouriteIndex?: Record }): Promise; update(id: string, data: Record): Promise; delete(id: string): Promise; setServers(projectId: string, serverIds: string[]): Promise; @@ -36,7 +36,7 @@ export class ProjectRepository implements IProjectRepository { return this.prisma.project.findUnique({ where: { name }, include: PROJECT_INCLUDE }) as unknown as Promise; } - async create(data: { name: string; description: string; prompt?: string; ownerId: string; proxyModel?: string; gated?: boolean; llmProvider?: string; llmModel?: string; serverOverrides?: Record }): Promise { + async create(data: { name: string; description: string; prompt?: string; ownerId: string; proxyModel?: string; gated?: boolean; llmProvider?: string; llmModel?: string; serverOverrides?: Record; favouriteIndex?: Record }): Promise { const createData: Record = { name: data.name, description: data.description, @@ -48,6 +48,7 @@ export class ProjectRepository implements IProjectRepository { if (data.llmProvider !== undefined) createData['llmProvider'] = data.llmProvider; if (data.llmModel !== undefined) createData['llmModel'] = data.llmModel; if (data.serverOverrides !== undefined) createData['serverOverrides'] = data.serverOverrides; + if (data.favouriteIndex !== undefined) createData['favouriteIndex'] = data.favouriteIndex; return this.prisma.project.create({ data: createData as Parameters[0]['data'], diff --git a/src/mcpd/src/routes/audit-events.ts b/src/mcpd/src/routes/audit-events.ts index 4b72121..1223ce1 100644 --- a/src/mcpd/src/routes/audit-events.ts +++ b/src/mcpd/src/routes/audit-events.ts @@ -58,6 +58,15 @@ export function registerAuditEventRoutes(app: FastifyInstance, service: AuditEve return service.getById(request.params.id); }); + // GET /api/v1/audit/tool-usage — rank tools by invocation count (for favourites) + app.get<{ Querystring: { projectName?: string; window?: string; limit?: string } }>('/api/v1/audit/tool-usage', async (request, reply) => { + const q = request.query; + if (!q.projectName) return reply.code(400).send({ error: 'projectName is required' }); + const windowDays = q.window !== undefined ? parseInt(q.window, 10) : 30; + const limit = q.limit !== undefined ? parseInt(q.limit, 10) : 50; + return { tools: await service.toolUsage(q.projectName, windowDays, limit) }; + }); + // GET /api/v1/audit/sessions — list sessions with aggregates app.get<{ Querystring: { projectName?: string; userName?: string; from?: string; to?: string; limit?: string; offset?: string } }>('/api/v1/audit/sessions', async (request) => { const q = request.query; diff --git a/src/mcpd/src/services/audit-event.service.ts b/src/mcpd/src/services/audit-event.service.ts index e5b3ec7..488a6a1 100644 --- a/src/mcpd/src/services/audit-event.service.ts +++ b/src/mcpd/src/services/audit-event.service.ts @@ -63,6 +63,17 @@ export class AuditEventService { return { sessions, total }; } + /** + * Rank a project's tools by invocation count over a recent window, to derive + * a usage-weighted favourite shortlist. Presented names are normalized to + * canonical `server/tool` in the repository. + */ + async toolUsage(projectName: string, windowDays = 30, limit = 50): Promise>> { + const from = new Date(Date.now() - windowDays * 86400_000); + const ranked = await this.repo.toolUsage(projectName, from); + return limit > 0 ? ranked.slice(0, limit) : ranked; + } + private buildFilter(params?: AuditEventQueryParams): AuditEventFilter | undefined { if (!params) return undefined; const filter: AuditEventFilter = {}; diff --git a/src/mcpd/src/services/backup/yaml-serializer.ts b/src/mcpd/src/services/backup/yaml-serializer.ts index 60c5f69..b0ba369 100644 --- a/src/mcpd/src/services/backup/yaml-serializer.ts +++ b/src/mcpd/src/services/backup/yaml-serializer.ts @@ -70,8 +70,8 @@ function toApplyDoc(kind: string, raw: Record): Record 0) { result[key] = value; } diff --git a/src/mcpd/src/services/project.service.ts b/src/mcpd/src/services/project.service.ts index 712d4f3..d1d982c 100644 --- a/src/mcpd/src/services/project.service.ts +++ b/src/mcpd/src/services/project.service.ts @@ -91,6 +91,7 @@ export class ProjectService { ...(data.llmProvider !== undefined ? { llmProvider: data.llmProvider } : {}), ...(data.llmModel !== undefined ? { llmModel: data.llmModel } : {}), ...(data.serverOverrides !== undefined ? { serverOverrides: data.serverOverrides } : {}), + ...(data.favouriteIndex !== undefined ? { favouriteIndex: data.favouriteIndex } : {}), }); // Link servers @@ -115,6 +116,7 @@ export class ProjectService { if (data.llmModel !== undefined) updateData['llmModel'] = data.llmModel; if (data.gated !== undefined) updateData['gated'] = data.gated; if (data.serverOverrides !== undefined) updateData['serverOverrides'] = data.serverOverrides; + if (data.favouriteIndex !== undefined) updateData['favouriteIndex'] = data.favouriteIndex; // Update scalar fields if any changed if (Object.keys(updateData).length > 0) { @@ -188,6 +190,7 @@ export class ProjectService { if (data['llmProvider'] !== undefined) scalarFields['llmProvider'] = data['llmProvider']; if (data['llmModel'] !== undefined) scalarFields['llmModel'] = data['llmModel']; if (data['serverOverrides'] !== undefined) scalarFields['serverOverrides'] = data['serverOverrides']; + if (data['favouriteIndex'] !== undefined) scalarFields['favouriteIndex'] = data['favouriteIndex']; if (existing !== null) { if (Object.keys(scalarFields).length > 0) { diff --git a/src/mcpd/src/validation/project.schema.ts b/src/mcpd/src/validation/project.schema.ts index 382a2d6..25f33c8 100644 --- a/src/mcpd/src/validation/project.schema.ts +++ b/src/mcpd/src/validation/project.schema.ts @@ -1,5 +1,15 @@ import { z } from 'zod'; +/** + * Two-namespace `favourite/` + `all/` tool presentation (per-project). + * `tools` are canonical `server/tool` pins surfaced as favourites, in order. + */ +export const FavouriteIndexSchema = z.object({ + enabled: z.boolean().optional(), + tools: z.array(z.string().min(1)).optional(), + maxFavourites: z.number().int().positive().max(200).optional(), +}); + export const CreateProjectSchema = z.object({ name: z.string().min(1).max(100).regex(/^[a-z0-9-]+$/, 'Name must be lowercase alphanumeric with hyphens'), description: z.string().max(1000).default(''), @@ -12,6 +22,7 @@ export const CreateProjectSchema = z.object({ serverOverrides: z.record(z.string(), z.object({ proxyModel: z.string().optional(), })).optional(), + favouriteIndex: FavouriteIndexSchema.optional(), // Backward compat: accept but ignore proxyMode from old configs proxyMode: z.string().optional(), }).transform(({ proxyMode: _ignored, ...rest }) => rest); @@ -27,6 +38,7 @@ export const UpdateProjectSchema = z.object({ serverOverrides: z.record(z.string(), z.object({ proxyModel: z.string().optional(), })).optional(), + favouriteIndex: FavouriteIndexSchema.nullable().optional(), // Backward compat: accept but ignore proxyMode from old configs proxyMode: z.string().optional(), }).transform(({ proxyMode: _ignored, ...rest }) => rest); diff --git a/src/mcpd/tests/tool-usage.test.ts b/src/mcpd/tests/tool-usage.test.ts new file mode 100644 index 0000000..6c2b5f7 --- /dev/null +++ b/src/mcpd/tests/tool-usage.test.ts @@ -0,0 +1,30 @@ +import { describe, it, expect } from 'vitest'; +import { canonicalizeToolName } from '../src/repositories/audit-event.repository.js'; + +describe('canonicalizeToolName — normalize presented tool names for usage ranking', () => { + it('strips all/ back to canonical server/tool and derives the server', () => { + expect(canonicalizeToolName('all/k8s/get_pods', null)).toEqual({ key: 'k8s/get_pods', server: 'k8s' }); + }); + + it('keeps favourite/ distinct (already a pin), using the trace server if any', () => { + expect(canonicalizeToolName('favourite/get_pods', null)).toEqual({ key: 'favourite/get_pods', server: null }); + }); + + it('passes a canonical server/tool through, deriving the server from the name', () => { + expect(canonicalizeToolName('vault/read_secret', 'vault')).toEqual({ key: 'vault/read_secret', server: 'vault' }); + }); + + it('handles a bare (non-namespaced) tool name', () => { + expect(canonicalizeToolName('read_prompts', null)).toEqual({ key: 'read_prompts', server: null }); + }); + + it('handles all/ where the tool name itself contains slashes', () => { + expect(canonicalizeToolName('all/srv/a/b', null)).toEqual({ key: 'srv/a/b', server: 'srv' }); + }); + + it('all// and / collapse to the same key', () => { + const a = canonicalizeToolName('all/gitea/create_pull_request', null); + const b = canonicalizeToolName('gitea/create_pull_request', 'gitea'); + expect(a.key).toBe(b.key); + }); +}); diff --git a/src/mcplocal/src/discovery.ts b/src/mcplocal/src/discovery.ts index 6ee8181..2cf8970 100644 --- a/src/mcplocal/src/discovery.ts +++ b/src/mcplocal/src/discovery.ts @@ -65,6 +65,16 @@ export async function refreshProjectUpstreams( * in `~/.mcpctl/config.json` still take priority, and unknown names fall * through to the registry default. */ +/** + * Two-namespace `favourite/` + `all/` tool presentation config (per-project). + * `tools` are canonical `server/tool` pins shown as favourites, in order. + */ +export interface FavouriteIndexConfig { + enabled?: boolean; + tools?: string[]; + maxFavourites?: number; +} + export interface ProjectLlmConfig { /** Name of an `Llm` resource on mcpd, or 'none' to disable LLM features. */ llmProvider?: string; @@ -72,6 +82,7 @@ export interface ProjectLlmConfig { proxyModel?: string; gated?: boolean; serverOverrides?: Record; + favouriteIndex?: FavouriteIndexConfig; } /** @@ -110,6 +121,7 @@ export async function fetchProjectLlmConfig( proxyModel?: string; gated?: boolean; serverOverrides?: Record; + favouriteIndex?: FavouriteIndexConfig; }>(`/api/v1/projects/${encodeURIComponent(projectName)}`); const config: ProjectLlmConfig = {}; if (project.llmProvider) config.llmProvider = project.llmProvider; @@ -125,6 +137,7 @@ export async function fetchProjectLlmConfig( } config.gated = config.proxyModel === 'default' || config.proxyModel === 'gate'; if (project.serverOverrides) config.serverOverrides = project.serverOverrides; + if (project.favouriteIndex) config.favouriteIndex = project.favouriteIndex; return config; } catch { return {}; diff --git a/src/mcplocal/src/http/project-mcp-endpoint.ts b/src/mcplocal/src/http/project-mcp-endpoint.ts index 4f53d11..94b8b91 100644 --- a/src/mcplocal/src/http/project-mcp-endpoint.ts +++ b/src/mcplocal/src/http/project-mcp-endpoint.ts @@ -23,7 +23,9 @@ import { LLMProviderAdapter } from '../proxymodel/llm-adapter.js'; import { FileCache } from '../proxymodel/file-cache.js'; import { createDefaultPlugin } from '../proxymodel/plugins/default.js'; import { createAgentsPlugin } from '../proxymodel/plugins/agents.js'; +import { createFavouriteIndexPlugin } from '../proxymodel/plugins/favourite-index.js'; import { composePlugins } from '../proxymodel/plugins/compose.js'; +import type { ProxyModelPlugin } from '../proxymodel/plugin.js'; import { AuditCollector } from '../audit/collector.js'; interface ProjectCacheEntry { @@ -146,11 +148,25 @@ export function registerProjectMcpEndpoint(app: FastifyInstance, mcpdClient: Mcp }; if (resolvedModel) pluginConfig.modelOverride = resolvedModel; const basePlugin = createDefaultPlugin(pluginConfig); + // Optional favourite-index presentation: curated favourite/ + full + // all// + a "prefer favourite/" instruction. Composed AFTER + // the base (gate) plugin so it reshapes the ungated catalog and no-ops + // while gated. Only when the project opts in and has pinned tools. + const fav = mcpdConfig.favouriteIndex; + const chain: ProxyModelPlugin[] = [basePlugin]; + if (fav?.enabled && (fav.tools?.length ?? 0) > 0) { + chain.push( + createFavouriteIndexPlugin({ + favourites: fav.tools ?? [], + ...(fav.maxFavourites !== undefined ? { maxFavourites: fav.maxFavourites } : {}), + }), + ); + } // Always compose the agents plugin on top so Agents attached to the // project show up as virtual MCP servers in tools/list, regardless of // which proxymodel the project is using. - const plugin = composePlugins([basePlugin, createAgentsPlugin()]); - router.setPlugin(plugin); + chain.push(createAgentsPlugin()); + router.setPlugin(composePlugins(chain)); // Fetch project instructions and set on router try { diff --git a/src/mcplocal/src/proxymodel/plugins/compose.ts b/src/mcplocal/src/proxymodel/plugins/compose.ts index abbaba3..9cffcce 100644 --- a/src/mcplocal/src/proxymodel/plugins/compose.ts +++ b/src/mcplocal/src/proxymodel/plugins/compose.ts @@ -10,7 +10,8 @@ * * Hook semantics: * - onSessionCreate / onSessionDestroy: every plugin's hook runs in order. - * - onInitialize: first non-null result wins (instructions don't merge). + * - onInitialize: instructions from every plugin are concatenated (in order, + * blank-line separated) so orthogonal plugins can each contribute a banner. * - onToolsList / onResourcesList / onPromptsList: results pipeline through * the plugins, each transforming the previous step's output. * - onToolCallBefore / onResourceRead / onPromptGet: first non-null wins @@ -50,13 +51,15 @@ export function composePlugins(plugins: ProxyModelPlugin[]): ProxyModelPlugin { } if (plugins.some((p) => p.onInitialize)) { out.onInitialize = async (request, ctx) => { + const parts: string[] = []; for (const p of plugins) { if (p.onInitialize) { const res = await p.onInitialize(request, ctx); - if (res !== null) return res; + const instr = res?.instructions?.trim(); + if (instr) parts.push(instr); } } - return null; + return parts.length > 0 ? { instructions: parts.join('\n\n') } : null; }; } if (plugins.some((p) => p.onToolsList)) { diff --git a/src/mcplocal/src/proxymodel/plugins/favourite-index.ts b/src/mcplocal/src/proxymodel/plugins/favourite-index.ts new file mode 100644 index 0000000..e737319 --- /dev/null +++ b/src/mcplocal/src/proxymodel/plugins/favourite-index.ts @@ -0,0 +1,139 @@ +/** + * favourite-index plugin — two-namespace tool presentation. + * + * Measured on a 145-tool catalog (scripts/model-eval/toolsim.py) to be the tool + * presentation that most reduces wander and lifts first-pick for a wandering + * model: expose a curated `favourite/` shortlist FIRST, the full + * `all//` catalog second, plus a load-bearing instruction telling + * the model to prefer favourites and fall back to `all/` only when nothing fits. + * (The instruction is not optional — the same two namespaces without it scored + * materially worse. See memory `feedback_llm_tool_scoping`.) + * + * Composes AFTER the gate/default plugin: it transforms the *ungated* upstream + * catalog and no-ops while a session is still gated (only `begin_session` + * visible). Gate/agent virtual tools (read_prompts, propose_prompt, + * agent- chat) pass through unchanged at the top level. + * + * Routing: the presented names are re-mapped back to their canonical + * `server/tool` in `onToolCallBefore` (a name rewrite, then the chain + * continues) so the normal upstream routing AND the content-pipeline + * `onToolCallAfter` still run — unlike a virtual-tool alias, which would + * short-circuit before content processing. + */ +import type { ProxyModelPlugin, PluginSessionContext } from '../plugin.js'; +import type { ToolDefinition } from '../types.js'; + +/** Per-session state key holding the presented-name → canonical-name map. */ +const RESOLVER_KEY = 'favourite-index:resolver'; + +/** The load-bearing "prefer favourite/" instruction (see module doc). */ +export const FAVOURITE_INDEX_INSTRUCTION = + 'Tools are indexed in two namespaces. PREFER favourite/ — a short ' + + 'curated list of the common tools that covers most tasks; reach for these ' + + 'first. Use the full catalog under all// only if nothing in ' + + "favourites fits. (read_prompts gives this project's own guidance.)"; + +export interface FavouriteIndexConfig { + /** Canonical `server/tool` names to surface as favourites, in display order. */ + favourites: string[]; + /** Cap on favourites presented (default: all provided). Enforces "not everything". */ + maxFavourites?: number; +} + +function humanize(name: string): string { + return name.replace(/_/g, ' '); +} + +/** Split a canonical `server/tool` on the FIRST slash (server names have none). */ +function splitServer(canonical: string): { server: string; short: string } { + const i = canonical.indexOf('/'); + return i < 0 + ? { server: '', short: canonical } + : { server: canonical.slice(0, i), short: canonical.slice(i + 1) }; +} + +export function createFavouriteIndexPlugin(config: FavouriteIndexConfig): ProxyModelPlugin { + const favourites = config.favourites ?? []; + const cap = config.maxFavourites && config.maxFavourites > 0 ? config.maxFavourites : undefined; + + return { + name: 'favourite-index', + description: + 'Two-namespace tool presentation: curated favourite/ + full all//, prefer favourites.', + + async onInitialize() { + return { instructions: FAVOURITE_INDEX_INSTRUCTION }; + }, + + async onToolsList(tools: ToolDefinition[], ctx: PluginSessionContext): Promise { + // Distinguish upstream tools (in the discovered catalog) from virtual + // tools (gate/agent) that must pass through untouched. + const upstream = await ctx.discoverTools(); + const upstreamNames = new Set(upstream.map((t) => t.name)); + const upstreamTools = tools.filter((t) => upstreamNames.has(t.name)); + const passthrough = tools.filter((t) => !upstreamNames.has(t.name)); + + // Gated / empty catalog → no-op (leave the list exactly as gate produced it). + if (upstreamTools.length === 0) { + ctx.state.delete(RESOLVER_KEY); + return tools; + } + + const byName = new Map(upstreamTools.map((t) => [t.name, t])); + const resolver = new Map(); // presented → canonical + const usedPresented = new Set(); + const seenCanonical = new Set(); + + // Favourites first: configured order, only those present, capped, with + // short-name collision disambiguation. + const limit = cap ?? favourites.length; + const favTools: ToolDefinition[] = []; + for (const canonical of favourites) { + if (favTools.length >= limit) break; + if (seenCanonical.has(canonical)) continue; + const t = byName.get(canonical); + if (!t) continue; // stale/missing pin — skip (all/ still exposes it) + const { server, short } = splitServer(canonical); + let presented = `favourite/${short}`; + if (usedPresented.has(presented)) presented = `favourite/${server}-${short}`; + if (usedPresented.has(presented)) continue; // still collides (rare) — skip + seenCanonical.add(canonical); + usedPresented.add(presented); + resolver.set(presented, canonical); + favTools.push({ + name: presented, + description: `${humanize(short)} — common (${server})`, + ...(t.inputSchema !== undefined ? { inputSchema: t.inputSchema } : {}), + }); + } + + // Then the full catalog under all//. + const allTools: ToolDefinition[] = upstreamTools.map((t) => { + const presented = `all/${t.name}`; + resolver.set(presented, t.name); + return { + name: presented, + ...(t.description !== undefined ? { description: t.description } : {}), + ...(t.inputSchema !== undefined ? { inputSchema: t.inputSchema } : {}), + }; + }); + + ctx.state.set(RESOLVER_KEY, resolver); + return [...favTools, ...allTools, ...passthrough]; + }, + + async onToolCallBefore(toolName, _args, request, ctx) { + const resolver = ctx.state.get(RESOLVER_KEY) as Map | undefined; + const canonical = + resolver?.get(toolName) ?? + (toolName.startsWith('all/') ? toolName.slice('all/'.length) : undefined); + if (canonical && canonical !== toolName) { + // Rewrite presented name → canonical `server/tool`, then continue the + // chain (return null) so normal routing + content-pipeline still run. + const params = request.params as Record | undefined; + if (params) params['name'] = canonical; + } + return null; + }, + }; +} diff --git a/src/mcplocal/tests/plugin-favourite-index.test.ts b/src/mcplocal/tests/plugin-favourite-index.test.ts new file mode 100644 index 0000000..2e78e69 --- /dev/null +++ b/src/mcplocal/tests/plugin-favourite-index.test.ts @@ -0,0 +1,185 @@ +/** + * favourite-index plugin tests — presentation (favourite/ + all/), rewrite + * routing back to canonical server/tool, gated no-op, and instruction merge. + * + * Driven through a real McpRouter (gate composed with favourite-index, mirroring + * project-mcp-endpoint wiring) so routing is exercised end-to-end. + */ +import { describe, it, expect, vi } from 'vitest'; +import { McpRouter } from '../src/router.js'; +import type { UpstreamConnection, JsonRpcRequest, JsonRpcResponse } from '../src/types.js'; +import type { McpdClient } from '../src/http/mcpd-client.js'; +import { createGatePlugin } from '../src/proxymodel/plugins/gate.js'; +import { composePlugins } from '../src/proxymodel/plugins/compose.js'; +import { + createFavouriteIndexPlugin, + FAVOURITE_INDEX_INSTRUCTION, +} from '../src/proxymodel/plugins/favourite-index.js'; +import { LLMProviderAdapter } from '../src/proxymodel/llm-adapter.js'; +import { MemoryCache } from '../src/proxymodel/cache.js'; + +function mockUpstream( + name: string, + tools: Array<{ name: string; description?: string }>, +): UpstreamConnection { + return { + name, + isAlive: vi.fn(() => true), + close: vi.fn(async () => {}), + onNotification: vi.fn(), + send: vi.fn(async (req: JsonRpcRequest): Promise => { + if (req.method === 'tools/list') { + return { jsonrpc: '2.0', id: req.id, result: { tools } }; + } + if (req.method === 'tools/call') { + // Echo the name the UPSTREAM received (proves canonical routing). + return { + jsonrpc: '2.0', + id: req.id, + result: { content: [{ type: 'text', text: `Called ${(req.params as Record)?.['name']}` }] }, + }; + } + if (req.method === 'resources/list') return { jsonrpc: '2.0', id: req.id, result: { resources: [] } }; + if (req.method === 'prompts/list') return { jsonrpc: '2.0', id: req.id, result: { prompts: [] } }; + return { jsonrpc: '2.0', id: req.id, error: { code: -32601, message: 'Not found' } }; + }), + } as UpstreamConnection; +} + +function mockMcpdClient(): McpdClient { + return { + get: vi.fn(async () => []), + post: vi.fn(async () => ({})), + put: vi.fn(async () => ({})), + delete: vi.fn(async () => {}), + forward: vi.fn(async () => ({ status: 200, body: {} })), + withHeaders: vi.fn(function (this: McpdClient) { return this; }), + } as unknown as McpdClient; +} + +function setup(opts: { gated?: boolean; favourites?: string[]; maxFavourites?: number } = {}) { + const router = new McpRouter(); + router.setPromptConfig(mockMcpdClient(), 'test-project'); + const plugin = composePlugins([ + createGatePlugin({ gated: opts.gated ?? false, providerRegistry: null }), + createFavouriteIndexPlugin({ + favourites: opts.favourites ?? [], + ...(opts.maxFavourites !== undefined ? { maxFavourites: opts.maxFavourites } : {}), + }), + ]); + router.setPlugin(plugin); + router.setProxyModel('default', { complete: async () => '', available: () => false } as unknown as LLMProviderAdapter, new MemoryCache()); + return router; +} + +async function initAndList(router: McpRouter, sessionId = 's1'): Promise> { + await router.route({ jsonrpc: '2.0', id: 1, method: 'initialize' }, { sessionId }); + const res = await router.route({ jsonrpc: '2.0', id: 2, method: 'tools/list' }, { sessionId }); + return (res.result as { tools: Array<{ name: string; description?: string }> }).tools; +} + +const CATALOG = { + k8s: [{ name: 'get_pods' }, { name: 'get_secret' }, { name: 'scale_deployment' }], + vault: [{ name: 'read_secret' }, { name: 'list_secrets' }], +}; + +describe('favourite-index plugin', () => { + it('presents favourite/ first (curated) then all/ (full catalog)', async () => { + const router = setup({ favourites: ['k8s/get_pods', 'vault/read_secret'] }); + router.addUpstream(mockUpstream('k8s', CATALOG.k8s)); + router.addUpstream(mockUpstream('vault', CATALOG.vault)); + + const names = (await initAndList(router)).map((t) => t.name); + + // Favourites, in configured order, appear before any all/ entry. + expect(names.indexOf('favourite/get_pods')).toBe(0); + expect(names.indexOf('favourite/read_secret')).toBe(1); + expect(names.indexOf('favourite/get_pods')).toBeLessThan(names.indexOf('all/k8s/get_pods')); + + // Full catalog present under all/. + for (const t of [...CATALOG.k8s.map((x) => `all/k8s/${x.name}`), ...CATALOG.vault.map((x) => `all/vault/${x.name}`)]) { + expect(names).toContain(t); + } + // Gate virtual tools pass through untouched (not re-namespaced). + expect(names).toContain('read_prompts'); + expect(names).not.toContain('favourite/read_prompts'); + // Non-favourite tools are NOT duplicated into favourite/. + expect(names).not.toContain('favourite/get_secret'); + }); + + it('routes a favourite/ call back to the canonical upstream tool', async () => { + const router = setup({ favourites: ['vault/read_secret'] }); + router.addUpstream(mockUpstream('vault', CATALOG.vault)); + await initAndList(router); + + const res = await router.route( + { jsonrpc: '2.0', id: 3, method: 'tools/call', params: { name: 'favourite/read_secret', arguments: {} } }, + { sessionId: 's1' }, + ); + const text = (res.result as { content: Array<{ text: string }> }).content[0]!.text; + // Upstream sees the stripped canonical tool name. + expect(text).toBe('Called read_secret'); + }); + + it('routes an all// call back to the canonical upstream tool', async () => { + const router = setup({ favourites: [] }); + router.addUpstream(mockUpstream('k8s', CATALOG.k8s)); + await initAndList(router); + + const res = await router.route( + { jsonrpc: '2.0', id: 3, method: 'tools/call', params: { name: 'all/k8s/scale_deployment', arguments: {} } }, + { sessionId: 's1' }, + ); + const text = (res.result as { content: Array<{ text: string }> }).content[0]!.text; + expect(text).toBe('Called scale_deployment'); + }); + + it('no-ops while the session is gated (only begin_session visible)', async () => { + const router = setup({ gated: true, favourites: ['k8s/get_pods'] }); + router.addUpstream(mockUpstream('k8s', CATALOG.k8s)); + const names = (await initAndList(router)).map((t) => t.name); + expect(names).toEqual(['begin_session']); + }); + + it('skips stale favourites not in the catalog but keeps them in all/', async () => { + const router = setup({ favourites: ['k8s/get_pods', 'gone/missing_tool'] }); + router.addUpstream(mockUpstream('k8s', CATALOG.k8s)); + const names = (await initAndList(router)).map((t) => t.name); + expect(names).toContain('favourite/get_pods'); + expect(names).not.toContain('favourite/missing_tool'); + }); + + it('caps favourites at maxFavourites', async () => { + const router = setup({ favourites: ['k8s/get_pods', 'k8s/get_secret', 'vault/read_secret'], maxFavourites: 2 }); + router.addUpstream(mockUpstream('k8s', CATALOG.k8s)); + router.addUpstream(mockUpstream('vault', CATALOG.vault)); + const names = (await initAndList(router)).map((t) => t.name).filter((n) => n.startsWith('favourite/')); + expect(names).toHaveLength(2); + expect(names).toEqual(['favourite/get_pods', 'favourite/get_secret']); + }); + + it('disambiguates short-name collisions across servers', async () => { + // Both servers expose a `status` tool; both are favourites. + const router = setup({ favourites: ['k8s/status', 'vault/status'] }); + router.addUpstream(mockUpstream('k8s', [{ name: 'status' }])); + router.addUpstream(mockUpstream('vault', [{ name: 'status' }])); + const names = (await initAndList(router)).map((t) => t.name).filter((n) => n.startsWith('favourite/')); + expect(names).toContain('favourite/status'); + expect(names).toContain('favourite/vault-status'); + + // The disambiguated favourite still routes to the right server. + const res = await router.route( + { jsonrpc: '2.0', id: 3, method: 'tools/call', params: { name: 'favourite/vault-status', arguments: {} } }, + { sessionId: 's1' }, + ); + expect((res.result as { content: Array<{ text: string }> }).content[0]!.text).toBe('Called status'); + }); + + it('injects the load-bearing prefer-favourite instruction on initialize', async () => { + const router = setup({ favourites: ['k8s/get_pods'] }); + router.addUpstream(mockUpstream('k8s', CATALOG.k8s)); + const res = await router.route({ jsonrpc: '2.0', id: 1, method: 'initialize' }, { sessionId: 's1' }); + const instructions = (res.result as { instructions?: string }).instructions ?? ''; + expect(instructions).toContain(FAVOURITE_INDEX_INSTRUCTION); + }); +}); diff --git a/src/mcplocal/tests/smoke/favourite-index.test.ts b/src/mcplocal/tests/smoke/favourite-index.test.ts new file mode 100644 index 0000000..5b95061 --- /dev/null +++ b/src/mcplocal/tests/smoke/favourite-index.test.ts @@ -0,0 +1,125 @@ +/** + * Smoke test: favourite-index tool presentation end-to-end. + * + * Provisions an ungated project with favourite-index enabled + two pinned tools + * from the smoke-aws-docs server, then verifies through the LIVE mcplocal proxy: + * - initialize instructions carry the load-bearing "prefer favourite/" line, + * - tools/list presents favourite/ (curated) + all// (full), + * - a favourite/ call and an all/ call both ROUTE to the real upstream + * (they reach the server's arg validation, not a -32601 "unknown tool"). + * + * Run with: pnpm test:smoke + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { SmokeMcpSession, isMcplocalRunning, mcpctl } from './mcp-client.js'; +import { ChatReporter } from './reporter.js'; +import { resolve } from 'node:path'; + +const PROJECT_NAME = 'smoke-favindex'; +const SMOKE_DATA = 'smoke-data'; +const FIXTURE_PATH = resolve(import.meta.dirname, 'fixtures', 'smoke-data.yaml'); +const FAV_TOOL = 'smoke-aws-docs/read_documentation'; + +describe('Smoke: favourite-index presentation', () => { + let ready = false; + + beforeAll(async () => { + console.log('\n ━━━ Smoke Test: favourite-index ━━━'); + if (!(await isMcplocalRunning())) { + console.log(' ✗ mcplocal not running — skipping\n'); + return; + } + + // Ensure the shared smoke-aws-docs server exists (from the smoke-data fixture). + try { + await mcpctl(`describe project ${SMOKE_DATA}`); + } catch { + try { await mcpctl(`apply -f ${FIXTURE_PATH}`); } catch { /* best effort */ } + } + + // Dedicated ungated project with favourite-index on + two pins. + try { + await mcpctl( + `create project ${PROJECT_NAME} --force --no-gated --server smoke-aws-docs ` + + `--favourite-index --favourite ${FAV_TOOL} --favourite smoke-aws-docs/search_documentation`, + ); + } catch (err) { + console.log(` ⚠ project setup error: ${err instanceof Error ? err.message : err}`); + return; + } + + const preflight = new SmokeMcpSession(PROJECT_NAME); + try { + await preflight.initialize(); + ready = true; + console.log(' ✓ Server responding'); + } catch (err) { + console.log(` ✗ Server not responding: ${err instanceof Error ? err.message : err}`); + } finally { + await preflight.close(); + } + }, 60_000); + + afterAll(async () => { + try { await mcpctl(`delete project ${PROJECT_NAME}`); } catch { /* best effort cleanup */ } + console.log('\n ━━━ favourite-index smoke complete ━━━\n'); + }); + + it('presents favourite/ + all/ namespaces with the prefer-favourite instruction', async () => { + if (!ready) return; + const chat = new ChatReporter(new SmokeMcpSession(PROJECT_NAME)); + chat.section('favourite-index presentation'); + try { + const initResult = (await chat.initialize()) as { instructions?: string }; + const instructions = initResult?.instructions ?? ''; + chat.check('Instruction mentions favourite/', String(instructions.includes('favourite/')), (v) => v === 'true'); + expect(instructions).toContain('favourite/'); + + const tools = await chat.listTools(); + const names = tools.map((t) => t.name); + const favNames = names.filter((n) => n.startsWith('favourite/')); + const allNames = names.filter((n) => n.startsWith('all/')); + + chat.check('Has favourite/ tools', favNames.length, (v) => v >= 1); + chat.check('Has all/ catalog', allNames.length, (v) => v >= 1); + chat.check('favourite/read_documentation present', String(names.includes('favourite/read_documentation')), (v) => v === 'true'); + chat.check('all/smoke-aws-docs/read_documentation present', String(names.includes('all/smoke-aws-docs/read_documentation')), (v) => v === 'true'); + + // Favourites are listed before the all/ catalog. + const firstFav = names.findIndex((n) => n.startsWith('favourite/')); + const firstAll = names.findIndex((n) => n.startsWith('all/')); + chat.check('favourites precede all/', String(firstFav < firstAll), (v) => v === 'true'); + + expect(names).toContain('favourite/read_documentation'); + expect(names).toContain('all/smoke-aws-docs/read_documentation'); + expect(firstFav).toBeLessThan(firstAll); + } finally { + await chat.close(); + } + }, 30_000); + + it('routes favourite/ and all/ calls to the real upstream tool', async () => { + if (!ready) return; + const chat = new ChatReporter(new SmokeMcpSession(PROJECT_NAME)); + chat.section('favourite-index routing'); + try { + await chat.initialize(); + + // Calling with no args → the UPSTREAM tool's arg validation fires, proving + // the presented name routed to the real server (not a -32601 unknown tool). + const favRes = await chat.callTool('favourite/read_documentation', {}, 20_000).catch((e: Error) => ({ error: e.message })); + const allRes = await chat.callTool('all/smoke-aws-docs/read_documentation', {}, 20_000).catch((e: Error) => ({ error: e.message })); + + const favStr = JSON.stringify(favRes).toLowerCase(); + const allStr = JSON.stringify(allRes).toLowerCase(); + // Reached the upstream (arg validation / real response), not "unknown tool". + const routed = (s: string): boolean => !s.includes('-32601') && !s.includes('unknown') && !s.includes('method not found'); + chat.check('favourite/ routed to upstream', String(routed(favStr)), (v) => v === 'true'); + chat.check('all/ routed to upstream', String(routed(allStr)), (v) => v === 'true'); + expect(routed(favStr)).toBe(true); + expect(routed(allStr)).toBe(true); + } finally { + await chat.close(); + } + }, 30_000); +});