Compare commits
3 Commits
7fbb827aa5
...
test/wire-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b4ad95ca66 | ||
| bbd31883e8 | |||
|
|
c014fcdd82 |
@@ -26,12 +26,21 @@ import type { ToolDefinition } from '../types.js';
|
||||
/** Per-session state key holding the presented-name → canonical-name map. */
|
||||
const RESOLVER_KEY = 'favourite-index:resolver';
|
||||
|
||||
/** The load-bearing "prefer favourite/" instruction (see module doc). */
|
||||
/**
|
||||
* The load-bearing "prefer favourite" instruction (see module doc).
|
||||
*
|
||||
* Phrased in the WIRE form (underscore-joined): the HTTP boundary's
|
||||
* WireNameCodec rewrites the presented `favourite/<tool>` and
|
||||
* `all/<server>/<tool>` names to `favourite_<tool>` / `all_<server>_<tool>`
|
||||
* before any client sees them, so prose naming the slash form would tell the
|
||||
* model to call names that are not in its function list.
|
||||
*/
|
||||
export const FAVOURITE_INDEX_INSTRUCTION =
|
||||
'Tools are indexed in two namespaces. PREFER favourite/<tool> — a short ' +
|
||||
'curated list of the common tools that covers most tasks; reach for these ' +
|
||||
'first. Use the full catalog under all/<server>/<tool> only if nothing in ' +
|
||||
"favourites fits. (read_prompts gives this project's own guidance.)";
|
||||
'Tools are indexed in two namespaces. PREFER the favourite_-prefixed tools ' +
|
||||
'— a short curated list of the common tools that covers most tasks; reach ' +
|
||||
'for these first. Use the full catalog under the all_-prefixed names only ' +
|
||||
"if nothing in favourites fits. (read_prompts gives this project's own " +
|
||||
'guidance.)';
|
||||
|
||||
export interface FavouriteIndexConfig {
|
||||
/** Canonical `server/tool` names to surface as favourites, in display order. */
|
||||
|
||||
@@ -16,6 +16,7 @@ import type { TagMatchResult } from '../../gate/tag-matcher.js';
|
||||
import { LlmPromptSelector, pickCompletionText, type ServerInfer } from '../../gate/llm-selector.js';
|
||||
import type { ProviderRegistry } from '../../providers/registry.js';
|
||||
import { withTimeout, TimeoutError } from '../../util/with-timeout.js';
|
||||
import { sanitizeWireName } from '../../util/wire-names.js';
|
||||
|
||||
/** Cap on the gate's LLM prompt-selection. A slow/thinking LLM must never block
|
||||
* begin_session — on timeout we fall back to deterministic tag matching. */
|
||||
@@ -115,13 +116,15 @@ export function createGatePlugin(config: GatePluginConfig = {}): ProxyModelPlugi
|
||||
);
|
||||
parts.push(`\n${gateInstructions}`);
|
||||
|
||||
// Append tool inventory (names only)
|
||||
// Append tool inventory (names only). Sanitized to the wire charset:
|
||||
// the boundary WireNameCodec serves `server_tool`, so prose listing the
|
||||
// canonical `server/tool` would name functions the model cannot call.
|
||||
try {
|
||||
const tools = await ctx.discoverTools();
|
||||
if (tools.length > 0) {
|
||||
parts.push('\nAvailable MCP server tools (accessible after begin_session):');
|
||||
for (const t of tools) {
|
||||
parts.push(` ${t.name}`);
|
||||
parts.push(` ${sanitizeWireName(t.name)}`);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
@@ -424,13 +427,14 @@ async function handleBeginSession(
|
||||
);
|
||||
responseParts.push(encouragement);
|
||||
|
||||
// Append tool inventory (names only)
|
||||
// Append tool inventory (names only) — wire charset, see the initialize
|
||||
// inventory note.
|
||||
try {
|
||||
const tools = await ctx.discoverTools();
|
||||
if (tools.length > 0) {
|
||||
responseParts.push('\nAvailable MCP server tools:');
|
||||
for (const t of tools) {
|
||||
responseParts.push(` ${t.name}`);
|
||||
responseParts.push(` ${sanitizeWireName(t.name)}`);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
|
||||
@@ -525,8 +525,11 @@ describe('McpRouter gating', () => {
|
||||
);
|
||||
|
||||
const result = res.result as { instructions: string };
|
||||
expect(result.instructions).toContain('ha/get_entities');
|
||||
expect(result.instructions).toContain('node-red/get_flows');
|
||||
// Wire form: prose must name what the client can actually call — the
|
||||
// boundary WireNameCodec serves underscore-joined names.
|
||||
expect(result.instructions).toContain('ha_get_entities');
|
||||
expect(result.instructions).toContain('node-red_get_flows');
|
||||
expect(result.instructions).not.toContain('ha/get_entities');
|
||||
expect(result.instructions).toContain('after begin_session');
|
||||
// Descriptions should NOT be in init instructions (names only)
|
||||
expect(result.instructions).not.toContain('Get all entities');
|
||||
@@ -544,7 +547,9 @@ describe('McpRouter gating', () => {
|
||||
);
|
||||
|
||||
const text = (res.result as { content: Array<{ text: string }> }).content[0]!.text;
|
||||
expect(text).toContain('ha/get_entities');
|
||||
// Wire form in the inventory (see the initialize-instructions test).
|
||||
expect(text).toContain('ha_get_entities');
|
||||
expect(text).not.toContain('ha/get_entities');
|
||||
expect(text).not.toContain('Get all entities');
|
||||
});
|
||||
|
||||
|
||||
@@ -3,9 +3,9 @@
|
||||
*
|
||||
* Provisions an ungated project with favourite-index enabled + two pinned tools
|
||||
* from the smoke-aws-docs server, then verifies through the LIVE mcplocal proxy:
|
||||
* - initialize instructions carry the load-bearing "prefer favourite/" line,
|
||||
* - tools/list presents favourite/<tool> (curated) + all/<server>/<tool> (full),
|
||||
* - a favourite/ call and an all/ call both ROUTE to the real upstream
|
||||
* - initialize instructions carry the load-bearing "prefer favourite_" line,
|
||||
* - tools/list presents favourite_<tool> (curated) + all_<server>_<tool> (full, wire form),
|
||||
* - a favourite_ call and an all_ call both ROUTE to the real upstream
|
||||
* (they reach the server's arg validation, not a -32601 "unknown tool").
|
||||
*
|
||||
* Run with: pnpm test:smoke
|
||||
@@ -65,40 +65,40 @@ describe('Smoke: favourite-index presentation', () => {
|
||||
console.log('\n ━━━ favourite-index smoke complete ━━━\n');
|
||||
});
|
||||
|
||||
it('presents favourite/ + all/ namespaces with the prefer-favourite instruction', async () => {
|
||||
it('presents favourite_ + all_ namespaces with the prefer-favourite instruction', async () => {
|
||||
if (!ready) return;
|
||||
const chat = new ChatReporter(new SmokeMcpSession(PROJECT_NAME));
|
||||
chat.section('favourite-index presentation');
|
||||
try {
|
||||
const initResult = (await chat.initialize()) as { instructions?: string };
|
||||
const instructions = initResult?.instructions ?? '';
|
||||
chat.check('Instruction mentions favourite/', String(instructions.includes('favourite/')), (v) => v === 'true');
|
||||
expect(instructions).toContain('favourite/');
|
||||
chat.check('Instruction mentions favourite_', String(instructions.includes('favourite_')), (v) => v === 'true');
|
||||
expect(instructions).toContain('favourite_');
|
||||
|
||||
const tools = await chat.listTools();
|
||||
const names = tools.map((t) => t.name);
|
||||
const favNames = names.filter((n) => n.startsWith('favourite/'));
|
||||
const allNames = names.filter((n) => n.startsWith('all/'));
|
||||
const favNames = names.filter((n) => n.startsWith('favourite_'));
|
||||
const allNames = names.filter((n) => n.startsWith('all_'));
|
||||
|
||||
chat.check('Has favourite/ tools', favNames.length, (v) => v >= 1);
|
||||
chat.check('Has all/ catalog', allNames.length, (v) => v >= 1);
|
||||
chat.check('favourite/read_documentation present', String(names.includes('favourite/read_documentation')), (v) => v === 'true');
|
||||
chat.check('all/smoke-aws-docs/read_documentation present', String(names.includes('all/smoke-aws-docs/read_documentation')), (v) => v === 'true');
|
||||
chat.check('Has favourite_ tools', favNames.length, (v) => v >= 1);
|
||||
chat.check('Has all_ catalog', allNames.length, (v) => v >= 1);
|
||||
chat.check('favourite_read_documentation present', String(names.includes('favourite_read_documentation')), (v) => v === 'true');
|
||||
chat.check('all_smoke-aws-docs_read_documentation present', String(names.includes('all_smoke-aws-docs_read_documentation')), (v) => v === 'true');
|
||||
|
||||
// Favourites are listed before the all/ catalog.
|
||||
const firstFav = names.findIndex((n) => n.startsWith('favourite/'));
|
||||
const firstAll = names.findIndex((n) => n.startsWith('all/'));
|
||||
chat.check('favourites precede all/', String(firstFav < firstAll), (v) => v === 'true');
|
||||
// Favourites are listed before the all_ catalog.
|
||||
const firstFav = names.findIndex((n) => n.startsWith('favourite_'));
|
||||
const firstAll = names.findIndex((n) => n.startsWith('all_'));
|
||||
chat.check('favourites precede all_', String(firstFav < firstAll), (v) => v === 'true');
|
||||
|
||||
expect(names).toContain('favourite/read_documentation');
|
||||
expect(names).toContain('all/smoke-aws-docs/read_documentation');
|
||||
expect(names).toContain('favourite_read_documentation');
|
||||
expect(names).toContain('all_smoke-aws-docs_read_documentation');
|
||||
expect(firstFav).toBeLessThan(firstAll);
|
||||
} finally {
|
||||
await chat.close();
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
it('routes favourite/ and all/ calls to the real upstream tool', async () => {
|
||||
it('routes favourite_ and all_ calls to the real upstream tool', async () => {
|
||||
if (!ready) return;
|
||||
const chat = new ChatReporter(new SmokeMcpSession(PROJECT_NAME));
|
||||
chat.section('favourite-index routing');
|
||||
@@ -108,15 +108,15 @@ describe('Smoke: favourite-index presentation', () => {
|
||||
|
||||
// Calling with no args → the UPSTREAM tool's arg validation fires, proving
|
||||
// the presented name routed to the real server (not a -32601 unknown tool).
|
||||
const favRes = await chat.callTool('favourite/read_documentation', {}, 20_000).catch((e: Error) => ({ error: e.message }));
|
||||
const allRes = await chat.callTool('all/smoke-aws-docs/read_documentation', {}, 20_000).catch((e: Error) => ({ error: e.message }));
|
||||
const favRes = await chat.callTool('favourite_read_documentation', {}, 20_000).catch((e: Error) => ({ error: e.message }));
|
||||
const allRes = await chat.callTool('all_smoke-aws-docs_read_documentation', {}, 20_000).catch((e: Error) => ({ error: e.message }));
|
||||
|
||||
const favStr = JSON.stringify(favRes).toLowerCase();
|
||||
const allStr = JSON.stringify(allRes).toLowerCase();
|
||||
// Reached the upstream (arg validation / real response), not "unknown tool".
|
||||
const routed = (s: string): boolean => !s.includes('-32601') && !s.includes('unknown') && !s.includes('method not found');
|
||||
chat.check('favourite/ routed to upstream', String(routed(favStr)), (v) => v === 'true');
|
||||
chat.check('all/ routed to upstream', String(routed(allStr)), (v) => v === 'true');
|
||||
chat.check('favourite_ routed to upstream', String(routed(favStr)), (v) => v === 'true');
|
||||
chat.check('all_ routed to upstream', String(routed(allStr)), (v) => v === 'true');
|
||||
expect(routed(favStr)).toBe(true);
|
||||
expect(routed(allStr)).toBe(true);
|
||||
} finally {
|
||||
|
||||
@@ -158,8 +158,8 @@ describe('Smoke: ProxyModel pipeline', () => {
|
||||
const ungatedTools = await chat.listTools();
|
||||
chat.check('Ungated tools > 1', ungatedTools.length, (v) => v > 1);
|
||||
|
||||
const awsTools = ungatedTools.filter((t) => t.name.startsWith('smoke-aws-docs/'));
|
||||
chat.check('Has smoke-aws-docs/* tools', awsTools.length, (v) => v > 0);
|
||||
const awsTools = ungatedTools.filter((t) => t.name.startsWith('smoke-aws-docs_'));
|
||||
chat.check('Has smoke-aws-docs_* tools', awsTools.length, (v) => v > 0);
|
||||
|
||||
expect(ungatedTools.length).toBeGreaterThan(1);
|
||||
} finally {
|
||||
@@ -228,7 +228,7 @@ describe('Smoke: ProxyModel pipeline', () => {
|
||||
it('has AWS documentation tools after ungating', async () => {
|
||||
if (!serverResponding) return;
|
||||
|
||||
const awsTools = ungatedTools.filter((t) => t.name.startsWith('smoke-aws-docs/'));
|
||||
const awsTools = ungatedTools.filter((t) => t.name.startsWith('smoke-aws-docs_'));
|
||||
chat.check('AWS docs tools available', awsTools.length, (v) => v > 0);
|
||||
|
||||
if (awsTools.length > 0) {
|
||||
@@ -241,9 +241,9 @@ describe('Smoke: ProxyModel pipeline', () => {
|
||||
it('can call an AWS documentation tool', async () => {
|
||||
if (!serverResponding) return;
|
||||
|
||||
const searchTool = ungatedTools.find((t) => t.name === 'smoke-aws-docs/search_documentation');
|
||||
const recommendTool = ungatedTools.find((t) => t.name === 'smoke-aws-docs/recommend');
|
||||
const readTool = ungatedTools.find((t) => t.name === 'smoke-aws-docs/read_documentation');
|
||||
const searchTool = ungatedTools.find((t) => t.name === 'smoke-aws-docs_search_documentation');
|
||||
const recommendTool = ungatedTools.find((t) => t.name === 'smoke-aws-docs_recommend');
|
||||
const readTool = ungatedTools.find((t) => t.name === 'smoke-aws-docs_read_documentation');
|
||||
|
||||
// Prefer search_documentation — most reliable (no URL format requirements)
|
||||
const toolToTest = searchTool ?? recommendTool ?? readTool;
|
||||
@@ -271,13 +271,13 @@ describe('Smoke: ProxyModel pipeline', () => {
|
||||
it('large tool result gets paginated with _resultId', async () => {
|
||||
if (!serverResponding) return;
|
||||
|
||||
const readTool = ungatedTools.find((t) => t.name === 'smoke-aws-docs/read_documentation');
|
||||
const readTool = ungatedTools.find((t) => t.name === 'smoke-aws-docs_read_documentation');
|
||||
if (!readTool) {
|
||||
chat.skip('read_documentation not available');
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await chat.callTool('smoke-aws-docs/read_documentation', {
|
||||
const result = await chat.callTool('smoke-aws-docs_read_documentation', {
|
||||
url: 'https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html',
|
||||
});
|
||||
const text = result.content[0]?.text ?? '';
|
||||
@@ -296,13 +296,13 @@ describe('Smoke: ProxyModel pipeline', () => {
|
||||
it('section drill-down via _resultId and _section', async () => {
|
||||
if (!serverResponding) return;
|
||||
|
||||
const readTool = ungatedTools.find((t) => t.name === 'smoke-aws-docs/read_documentation');
|
||||
const readTool = ungatedTools.find((t) => t.name === 'smoke-aws-docs_read_documentation');
|
||||
if (!readTool) {
|
||||
chat.skip('read_documentation not available');
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await chat.callTool('smoke-aws-docs/read_documentation', {
|
||||
const result = await chat.callTool('smoke-aws-docs_read_documentation', {
|
||||
url: 'https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html',
|
||||
});
|
||||
const text = result.content[0]?.text ?? '';
|
||||
@@ -315,7 +315,7 @@ describe('Smoke: ProxyModel pipeline', () => {
|
||||
|
||||
const resultId = match[1]!.replace(/[^a-zA-Z0-9-]/g, '');
|
||||
|
||||
const sectionResult = await chat.callTool('smoke-aws-docs/read_documentation', {
|
||||
const sectionResult = await chat.callTool('smoke-aws-docs_read_documentation', {
|
||||
url: 'https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html',
|
||||
_resultId: resultId,
|
||||
_section: 'page-1',
|
||||
@@ -386,13 +386,13 @@ describe('Smoke: ProxyModel pipeline', () => {
|
||||
it('subindex model produces structural sections (not flat pages)', async () => {
|
||||
if (!serverResponding) return;
|
||||
|
||||
const readTool = (await chat.listTools()).find((t) => t.name === 'smoke-aws-docs/read_documentation');
|
||||
const readTool = (await chat.listTools()).find((t) => t.name === 'smoke-aws-docs_read_documentation');
|
||||
if (!readTool) {
|
||||
chat.skip('read_documentation not available');
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await chat.callTool('smoke-aws-docs/read_documentation', {
|
||||
const result = await chat.callTool('smoke-aws-docs_read_documentation', {
|
||||
url: 'https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html',
|
||||
});
|
||||
const text = result.content[0]?.text ?? '';
|
||||
@@ -409,13 +409,13 @@ describe('Smoke: ProxyModel pipeline', () => {
|
||||
it('subindex drill-down returns section content', async () => {
|
||||
if (!serverResponding) return;
|
||||
|
||||
const readTool = (await chat.listTools()).find((t) => t.name === 'smoke-aws-docs/read_documentation');
|
||||
const readTool = (await chat.listTools()).find((t) => t.name === 'smoke-aws-docs_read_documentation');
|
||||
if (!readTool) {
|
||||
chat.skip('read_documentation not available');
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await chat.callTool('smoke-aws-docs/read_documentation', {
|
||||
const result = await chat.callTool('smoke-aws-docs_read_documentation', {
|
||||
url: 'https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html',
|
||||
});
|
||||
const text = result.content[0]?.text ?? '';
|
||||
@@ -428,7 +428,7 @@ describe('Smoke: ProxyModel pipeline', () => {
|
||||
|
||||
const resultId = match[1]!.replace(/[^a-zA-Z0-9-]/g, '');
|
||||
|
||||
const sectionResult = await chat.callTool('smoke-aws-docs/read_documentation', {
|
||||
const sectionResult = await chat.callTool('smoke-aws-docs_read_documentation', {
|
||||
url: 'https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html',
|
||||
_resultId: resultId,
|
||||
_section: 'section-0',
|
||||
@@ -482,14 +482,14 @@ describe('Smoke: ProxyModel pipeline', () => {
|
||||
if (!serverResponding) return;
|
||||
|
||||
const tools = await chat.listTools();
|
||||
const readTool = tools.find((t) => t.name === 'smoke-aws-docs/read_documentation');
|
||||
const readTool = tools.find((t) => t.name === 'smoke-aws-docs_read_documentation');
|
||||
if (!readTool) {
|
||||
chat.skip('read_documentation not available');
|
||||
return;
|
||||
}
|
||||
|
||||
chat.info('Call 1: using default model (passthrough + paginate)');
|
||||
const result1 = await chat.callTool('smoke-aws-docs/read_documentation', {
|
||||
const result1 = await chat.callTool('smoke-aws-docs_read_documentation', {
|
||||
url: 'https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html',
|
||||
});
|
||||
const text1 = result1.content[0]?.text ?? '';
|
||||
@@ -516,7 +516,7 @@ describe('Smoke: ProxyModel pipeline', () => {
|
||||
);
|
||||
|
||||
chat.info('Call 2: using new model (should produce different output)');
|
||||
const result2 = await chat.callTool('smoke-aws-docs/read_documentation', {
|
||||
const result2 = await chat.callTool('smoke-aws-docs_read_documentation', {
|
||||
url: 'https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html',
|
||||
});
|
||||
const text2 = result2.content[0]?.text ?? '';
|
||||
|
||||
Reference in New Issue
Block a user