Compare commits
2 Commits
bbd31883e8
...
a87c4faf21
| Author | SHA1 | Date | |
|---|---|---|---|
| a87c4faf21 | |||
|
|
b4ad95ca66 |
@@ -3,9 +3,9 @@
|
||||
*
|
||||
* Provisions an ungated project with favourite-index enabled + two pinned tools
|
||||
* from the smoke-aws-docs server, then verifies through the LIVE mcplocal proxy:
|
||||
* - initialize instructions carry the load-bearing "prefer favourite/" line,
|
||||
* - tools/list presents favourite/<tool> (curated) + all/<server>/<tool> (full),
|
||||
* - a favourite/ call and an all/ call both ROUTE to the real upstream
|
||||
* - initialize instructions carry the load-bearing "prefer favourite_" line,
|
||||
* - tools/list presents favourite_<tool> (curated) + all_<server>_<tool> (full, wire form),
|
||||
* - a favourite_ call and an all_ call both ROUTE to the real upstream
|
||||
* (they reach the server's arg validation, not a -32601 "unknown tool").
|
||||
*
|
||||
* Run with: pnpm test:smoke
|
||||
@@ -65,40 +65,40 @@ describe('Smoke: favourite-index presentation', () => {
|
||||
console.log('\n ━━━ favourite-index smoke complete ━━━\n');
|
||||
});
|
||||
|
||||
it('presents favourite/ + all/ namespaces with the prefer-favourite instruction', async () => {
|
||||
it('presents favourite_ + all_ namespaces with the prefer-favourite instruction', async () => {
|
||||
if (!ready) return;
|
||||
const chat = new ChatReporter(new SmokeMcpSession(PROJECT_NAME));
|
||||
chat.section('favourite-index presentation');
|
||||
try {
|
||||
const initResult = (await chat.initialize()) as { instructions?: string };
|
||||
const instructions = initResult?.instructions ?? '';
|
||||
chat.check('Instruction mentions favourite/', String(instructions.includes('favourite/')), (v) => v === 'true');
|
||||
expect(instructions).toContain('favourite/');
|
||||
chat.check('Instruction mentions favourite_', String(instructions.includes('favourite_')), (v) => v === 'true');
|
||||
expect(instructions).toContain('favourite_');
|
||||
|
||||
const tools = await chat.listTools();
|
||||
const names = tools.map((t) => t.name);
|
||||
const favNames = names.filter((n) => n.startsWith('favourite/'));
|
||||
const allNames = names.filter((n) => n.startsWith('all/'));
|
||||
const favNames = names.filter((n) => n.startsWith('favourite_'));
|
||||
const allNames = names.filter((n) => n.startsWith('all_'));
|
||||
|
||||
chat.check('Has favourite/ tools', favNames.length, (v) => v >= 1);
|
||||
chat.check('Has all/ catalog', allNames.length, (v) => v >= 1);
|
||||
chat.check('favourite/read_documentation present', String(names.includes('favourite/read_documentation')), (v) => v === 'true');
|
||||
chat.check('all/smoke-aws-docs/read_documentation present', String(names.includes('all/smoke-aws-docs/read_documentation')), (v) => v === 'true');
|
||||
chat.check('Has favourite_ tools', favNames.length, (v) => v >= 1);
|
||||
chat.check('Has all_ catalog', allNames.length, (v) => v >= 1);
|
||||
chat.check('favourite_read_documentation present', String(names.includes('favourite_read_documentation')), (v) => v === 'true');
|
||||
chat.check('all_smoke-aws-docs_read_documentation present', String(names.includes('all_smoke-aws-docs_read_documentation')), (v) => v === 'true');
|
||||
|
||||
// Favourites are listed before the all/ catalog.
|
||||
const firstFav = names.findIndex((n) => n.startsWith('favourite/'));
|
||||
const firstAll = names.findIndex((n) => n.startsWith('all/'));
|
||||
chat.check('favourites precede all/', String(firstFav < firstAll), (v) => v === 'true');
|
||||
// Favourites are listed before the all_ catalog.
|
||||
const firstFav = names.findIndex((n) => n.startsWith('favourite_'));
|
||||
const firstAll = names.findIndex((n) => n.startsWith('all_'));
|
||||
chat.check('favourites precede all_', String(firstFav < firstAll), (v) => v === 'true');
|
||||
|
||||
expect(names).toContain('favourite/read_documentation');
|
||||
expect(names).toContain('all/smoke-aws-docs/read_documentation');
|
||||
expect(names).toContain('favourite_read_documentation');
|
||||
expect(names).toContain('all_smoke-aws-docs_read_documentation');
|
||||
expect(firstFav).toBeLessThan(firstAll);
|
||||
} finally {
|
||||
await chat.close();
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
it('routes favourite/ and all/ calls to the real upstream tool', async () => {
|
||||
it('routes favourite_ and all_ calls to the real upstream tool', async () => {
|
||||
if (!ready) return;
|
||||
const chat = new ChatReporter(new SmokeMcpSession(PROJECT_NAME));
|
||||
chat.section('favourite-index routing');
|
||||
@@ -108,15 +108,15 @@ describe('Smoke: favourite-index presentation', () => {
|
||||
|
||||
// Calling with no args → the UPSTREAM tool's arg validation fires, proving
|
||||
// the presented name routed to the real server (not a -32601 unknown tool).
|
||||
const favRes = await chat.callTool('favourite/read_documentation', {}, 20_000).catch((e: Error) => ({ error: e.message }));
|
||||
const allRes = await chat.callTool('all/smoke-aws-docs/read_documentation', {}, 20_000).catch((e: Error) => ({ error: e.message }));
|
||||
const favRes = await chat.callTool('favourite_read_documentation', {}, 20_000).catch((e: Error) => ({ error: e.message }));
|
||||
const allRes = await chat.callTool('all_smoke-aws-docs_read_documentation', {}, 20_000).catch((e: Error) => ({ error: e.message }));
|
||||
|
||||
const favStr = JSON.stringify(favRes).toLowerCase();
|
||||
const allStr = JSON.stringify(allRes).toLowerCase();
|
||||
// Reached the upstream (arg validation / real response), not "unknown tool".
|
||||
const routed = (s: string): boolean => !s.includes('-32601') && !s.includes('unknown') && !s.includes('method not found');
|
||||
chat.check('favourite/ routed to upstream', String(routed(favStr)), (v) => v === 'true');
|
||||
chat.check('all/ routed to upstream', String(routed(allStr)), (v) => v === 'true');
|
||||
chat.check('favourite_ routed to upstream', String(routed(favStr)), (v) => v === 'true');
|
||||
chat.check('all_ routed to upstream', String(routed(allStr)), (v) => v === 'true');
|
||||
expect(routed(favStr)).toBe(true);
|
||||
expect(routed(allStr)).toBe(true);
|
||||
} finally {
|
||||
|
||||
@@ -158,8 +158,8 @@ describe('Smoke: ProxyModel pipeline', () => {
|
||||
const ungatedTools = await chat.listTools();
|
||||
chat.check('Ungated tools > 1', ungatedTools.length, (v) => v > 1);
|
||||
|
||||
const awsTools = ungatedTools.filter((t) => t.name.startsWith('smoke-aws-docs/'));
|
||||
chat.check('Has smoke-aws-docs/* tools', awsTools.length, (v) => v > 0);
|
||||
const awsTools = ungatedTools.filter((t) => t.name.startsWith('smoke-aws-docs_'));
|
||||
chat.check('Has smoke-aws-docs_* tools', awsTools.length, (v) => v > 0);
|
||||
|
||||
expect(ungatedTools.length).toBeGreaterThan(1);
|
||||
} finally {
|
||||
@@ -228,7 +228,7 @@ describe('Smoke: ProxyModel pipeline', () => {
|
||||
it('has AWS documentation tools after ungating', async () => {
|
||||
if (!serverResponding) return;
|
||||
|
||||
const awsTools = ungatedTools.filter((t) => t.name.startsWith('smoke-aws-docs/'));
|
||||
const awsTools = ungatedTools.filter((t) => t.name.startsWith('smoke-aws-docs_'));
|
||||
chat.check('AWS docs tools available', awsTools.length, (v) => v > 0);
|
||||
|
||||
if (awsTools.length > 0) {
|
||||
@@ -241,9 +241,9 @@ describe('Smoke: ProxyModel pipeline', () => {
|
||||
it('can call an AWS documentation tool', async () => {
|
||||
if (!serverResponding) return;
|
||||
|
||||
const searchTool = ungatedTools.find((t) => t.name === 'smoke-aws-docs/search_documentation');
|
||||
const recommendTool = ungatedTools.find((t) => t.name === 'smoke-aws-docs/recommend');
|
||||
const readTool = ungatedTools.find((t) => t.name === 'smoke-aws-docs/read_documentation');
|
||||
const searchTool = ungatedTools.find((t) => t.name === 'smoke-aws-docs_search_documentation');
|
||||
const recommendTool = ungatedTools.find((t) => t.name === 'smoke-aws-docs_recommend');
|
||||
const readTool = ungatedTools.find((t) => t.name === 'smoke-aws-docs_read_documentation');
|
||||
|
||||
// Prefer search_documentation — most reliable (no URL format requirements)
|
||||
const toolToTest = searchTool ?? recommendTool ?? readTool;
|
||||
@@ -271,13 +271,13 @@ describe('Smoke: ProxyModel pipeline', () => {
|
||||
it('large tool result gets paginated with _resultId', async () => {
|
||||
if (!serverResponding) return;
|
||||
|
||||
const readTool = ungatedTools.find((t) => t.name === 'smoke-aws-docs/read_documentation');
|
||||
const readTool = ungatedTools.find((t) => t.name === 'smoke-aws-docs_read_documentation');
|
||||
if (!readTool) {
|
||||
chat.skip('read_documentation not available');
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await chat.callTool('smoke-aws-docs/read_documentation', {
|
||||
const result = await chat.callTool('smoke-aws-docs_read_documentation', {
|
||||
url: 'https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html',
|
||||
});
|
||||
const text = result.content[0]?.text ?? '';
|
||||
@@ -296,13 +296,13 @@ describe('Smoke: ProxyModel pipeline', () => {
|
||||
it('section drill-down via _resultId and _section', async () => {
|
||||
if (!serverResponding) return;
|
||||
|
||||
const readTool = ungatedTools.find((t) => t.name === 'smoke-aws-docs/read_documentation');
|
||||
const readTool = ungatedTools.find((t) => t.name === 'smoke-aws-docs_read_documentation');
|
||||
if (!readTool) {
|
||||
chat.skip('read_documentation not available');
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await chat.callTool('smoke-aws-docs/read_documentation', {
|
||||
const result = await chat.callTool('smoke-aws-docs_read_documentation', {
|
||||
url: 'https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html',
|
||||
});
|
||||
const text = result.content[0]?.text ?? '';
|
||||
@@ -315,7 +315,7 @@ describe('Smoke: ProxyModel pipeline', () => {
|
||||
|
||||
const resultId = match[1]!.replace(/[^a-zA-Z0-9-]/g, '');
|
||||
|
||||
const sectionResult = await chat.callTool('smoke-aws-docs/read_documentation', {
|
||||
const sectionResult = await chat.callTool('smoke-aws-docs_read_documentation', {
|
||||
url: 'https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html',
|
||||
_resultId: resultId,
|
||||
_section: 'page-1',
|
||||
@@ -386,13 +386,13 @@ describe('Smoke: ProxyModel pipeline', () => {
|
||||
it('subindex model produces structural sections (not flat pages)', async () => {
|
||||
if (!serverResponding) return;
|
||||
|
||||
const readTool = (await chat.listTools()).find((t) => t.name === 'smoke-aws-docs/read_documentation');
|
||||
const readTool = (await chat.listTools()).find((t) => t.name === 'smoke-aws-docs_read_documentation');
|
||||
if (!readTool) {
|
||||
chat.skip('read_documentation not available');
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await chat.callTool('smoke-aws-docs/read_documentation', {
|
||||
const result = await chat.callTool('smoke-aws-docs_read_documentation', {
|
||||
url: 'https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html',
|
||||
});
|
||||
const text = result.content[0]?.text ?? '';
|
||||
@@ -409,13 +409,13 @@ describe('Smoke: ProxyModel pipeline', () => {
|
||||
it('subindex drill-down returns section content', async () => {
|
||||
if (!serverResponding) return;
|
||||
|
||||
const readTool = (await chat.listTools()).find((t) => t.name === 'smoke-aws-docs/read_documentation');
|
||||
const readTool = (await chat.listTools()).find((t) => t.name === 'smoke-aws-docs_read_documentation');
|
||||
if (!readTool) {
|
||||
chat.skip('read_documentation not available');
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await chat.callTool('smoke-aws-docs/read_documentation', {
|
||||
const result = await chat.callTool('smoke-aws-docs_read_documentation', {
|
||||
url: 'https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html',
|
||||
});
|
||||
const text = result.content[0]?.text ?? '';
|
||||
@@ -428,7 +428,7 @@ describe('Smoke: ProxyModel pipeline', () => {
|
||||
|
||||
const resultId = match[1]!.replace(/[^a-zA-Z0-9-]/g, '');
|
||||
|
||||
const sectionResult = await chat.callTool('smoke-aws-docs/read_documentation', {
|
||||
const sectionResult = await chat.callTool('smoke-aws-docs_read_documentation', {
|
||||
url: 'https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html',
|
||||
_resultId: resultId,
|
||||
_section: 'section-0',
|
||||
@@ -482,14 +482,14 @@ describe('Smoke: ProxyModel pipeline', () => {
|
||||
if (!serverResponding) return;
|
||||
|
||||
const tools = await chat.listTools();
|
||||
const readTool = tools.find((t) => t.name === 'smoke-aws-docs/read_documentation');
|
||||
const readTool = tools.find((t) => t.name === 'smoke-aws-docs_read_documentation');
|
||||
if (!readTool) {
|
||||
chat.skip('read_documentation not available');
|
||||
return;
|
||||
}
|
||||
|
||||
chat.info('Call 1: using default model (passthrough + paginate)');
|
||||
const result1 = await chat.callTool('smoke-aws-docs/read_documentation', {
|
||||
const result1 = await chat.callTool('smoke-aws-docs_read_documentation', {
|
||||
url: 'https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html',
|
||||
});
|
||||
const text1 = result1.content[0]?.text ?? '';
|
||||
@@ -516,7 +516,7 @@ describe('Smoke: ProxyModel pipeline', () => {
|
||||
);
|
||||
|
||||
chat.info('Call 2: using new model (should produce different output)');
|
||||
const result2 = await chat.callTool('smoke-aws-docs/read_documentation', {
|
||||
const result2 = await chat.callTool('smoke-aws-docs_read_documentation', {
|
||||
url: 'https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html',
|
||||
});
|
||||
const text2 = result2.content[0]?.text ?? '';
|
||||
|
||||
Reference in New Issue
Block a user