fix(smoke): match the _resultId format the pipeline actually emits #130

Merged
michal merged 1 commits from fix/smoke-resultid-regex into main 2026-09-05 17:04:10 +00:00
3 changed files with 25 additions and 17 deletions

View File

@@ -261,6 +261,21 @@ export async function isMcplocalRunning(): Promise<boolean> {
} }
} }
/**
* Pull the drill-down id out of a paginated result or TOC.
*
* The instruction is written `_resultId="pm-abc"`, while the legacy paginator
* emits it as JSON (`"_resultId": "pm-abc"`). A regex matching only the JSON
* form silently found nothing in the current format, so drill-down checks
* either reported a false failure or skipped themselves as "not large enough
* for section-split" — both invisible in a green run. Accept either spelling
* and return the bare id.
*/
export function extractResultId(text: string): string | null {
const match = /_resultId"?\s*[:=]\s*"?(pm-[a-zA-Z0-9]+)/.exec(text);
return match?.[1] ?? null;
}
/** /**
* Run an mcpctl CLI command and return stdout. * Run an mcpctl CLI command and return stdout.
*/ */

View File

@@ -7,7 +7,7 @@
* Requires: mcplocal running on localhost:3200, mcpd at https://mcpctl.ad.itaz.eu * Requires: mcplocal running on localhost:3200, mcpd at https://mcpctl.ad.itaz.eu
*/ */
import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { describe, it, expect, beforeAll, afterAll } from 'vitest';
import { SmokeMcpSession, isMcplocalRunning } from './mcp-client.js'; import { SmokeMcpSession, isMcplocalRunning, extractResultId } from './mcp-client.js';
const PROJECT_NAME = 'smoke-data'; const PROJECT_NAME = 'smoke-data';
@@ -90,9 +90,8 @@ describe('Smoke: Prompt section drill-down', () => {
console.log(` TOC preview: ${text.slice(0, 200)}...`); console.log(` TOC preview: ${text.slice(0, 200)}...`);
// Extract _resultId // Extract _resultId
const match = /_resultId:\s*(pm-[a-z0-9]+)/.exec(text); const resultId = extractResultId(text);
if (match) { if (resultId !== null) {
const resultId = match[1];
// Extract first section id from TOC // Extract first section id from TOC
const sectionMatch = /\[([^\]]+)\]/.exec(text); const sectionMatch = /\[([^\]]+)\]/.exec(text);
if (sectionMatch) { if (sectionMatch) {

View File

@@ -17,7 +17,7 @@
import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { describe, it, expect, beforeAll, afterAll } from 'vitest';
import { writeFile, mkdir, rm } from 'node:fs/promises'; import { writeFile, mkdir, rm } from 'node:fs/promises';
import { join, resolve } from 'node:path'; import { join, resolve } from 'node:path';
import { SmokeMcpSession, isMcplocalRunning, mcpctl } from './mcp-client.js'; import { SmokeMcpSession, isMcplocalRunning, mcpctl, extractResultId } from './mcp-client.js';
import { ChatReporter } from './reporter.js'; import { ChatReporter } from './reporter.js';
const PROJECT_NAME = 'smoke-data'; const PROJECT_NAME = 'smoke-data';
@@ -286,8 +286,7 @@ describe('Smoke: ProxyModel pipeline', () => {
chat.check('Response is manageable size', text.length, (v) => v < 20_000); chat.check('Response is manageable size', text.length, (v) => v < 20_000);
if (text.includes('_resultId')) { if (text.includes('_resultId')) {
const match = text.match(/_resultId:\s*(\S+)/); chat.check('_resultId is present', extractResultId(text) !== null, (v) => v === true);
chat.check('_resultId is present', !!match, (v) => v === true);
} else { } else {
chat.info('Content small enough — no pagination needed'); chat.info('Content small enough — no pagination needed');
} }
@@ -307,14 +306,12 @@ describe('Smoke: ProxyModel pipeline', () => {
}); });
const text = result.content[0]?.text ?? ''; const text = result.content[0]?.text ?? '';
const match = text.match(/_resultId:\s*(\S+)/); const resultId = extractResultId(text);
if (!match) { if (resultId === null) {
chat.info('Content not large enough for pagination — skip drill-down'); chat.info('Content not large enough for pagination — skip drill-down');
return; return;
} }
const resultId = match[1]!.replace(/[^a-zA-Z0-9-]/g, '');
const sectionResult = await chat.callTool('smoke-aws-docs_read_documentation', { const sectionResult = await chat.callTool('smoke-aws-docs_read_documentation', {
url: 'https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html', url: 'https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html',
_resultId: resultId, _resultId: resultId,
@@ -401,8 +398,7 @@ describe('Smoke: ProxyModel pipeline', () => {
chat.check('Response is manageable size', text.length, (v) => v < 20_000); chat.check('Response is manageable size', text.length, (v) => v < 20_000);
if (text.includes('_resultId')) { if (text.includes('_resultId')) {
const match = text.match(/_resultId:\s*(\S+)/); chat.check('Has _resultId for drill-down', extractResultId(text) !== null, (v) => v === true);
chat.check('Has _resultId for drill-down', !!match, (v) => v === true);
} }
}, 60_000); }, 60_000);
@@ -420,14 +416,12 @@ describe('Smoke: ProxyModel pipeline', () => {
}); });
const text = result.content[0]?.text ?? ''; const text = result.content[0]?.text ?? '';
const match = text.match(/_resultId:\s*(\S+)/); const resultId = extractResultId(text);
if (!match) { if (resultId === null) {
chat.info('Content not large enough for section-split'); chat.info('Content not large enough for section-split');
return; return;
} }
const resultId = match[1]!.replace(/[^a-zA-Z0-9-]/g, '');
const sectionResult = await chat.callTool('smoke-aws-docs_read_documentation', { const sectionResult = await chat.callTool('smoke-aws-docs_read_documentation', {
url: 'https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html', url: 'https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html',
_resultId: resultId, _resultId: resultId,