diff --git a/.taskmaster/tasks/tasks.json b/.taskmaster/tasks/tasks.json index 043856f..ef8024a 100644 --- a/.taskmaster/tasks/tasks.json +++ b/.taskmaster/tasks/tasks.json @@ -729,48 +729,53 @@ "updatedAt": "2026-02-21T03:23:02.583Z" }, { - "id": 25, + "id": "25", "title": "Complete MCP Registry Client with Proxy, Metrics Exposure, and HTTP/CA Support", "description": "Finalize the registry client implementation by adding HTTP proxy support, custom CA certificates for enterprise environments, and exposing SRE metrics via a dedicated metrics interface. The core client with strategy pattern, caching, deduplication, and ranking is already implemented.", "details": "The registry client foundation already exists in src/cli/src/registry/ with:\n- RegistryClient class with search(), caching, metrics tracking\n- OfficialRegistrySource, GlamaRegistrySource, SmitheryRegistrySource\n- Deduplication by npm package/repo URL\n- Ranking by relevance, popularity, verified status\n- Zod validation of API responses\n- sanitizeString() for XSS prevention\n\nRemaining implementation:\n\n1. **HTTP Proxy & Custom CA Support** (src/cli/src/registry/http-agent.ts):\n```typescript\nimport { Agent } from 'undici';\nimport { ProxyAgent } from 'undici';\nimport fs from 'node:fs';\n\nexport function createHttpAgent(config: {\n httpProxy?: string;\n httpsProxy?: string;\n caPath?: string;\n}): Agent | ProxyAgent | undefined {\n const proxy = config.httpsProxy ?? config.httpProxy;\n if (proxy) {\n const ca = config.caPath ? fs.readFileSync(config.caPath) : undefined;\n return new ProxyAgent({ uri: proxy, connect: { ca } });\n }\n if (config.caPath) {\n const ca = fs.readFileSync(config.caPath);\n return new Agent({ connect: { ca } });\n }\n return undefined;\n}\n```\n\n2. **Update fetch calls** in each source to accept dispatcher option:\n```typescript\n// In retry.ts or each source\nconst agent = createHttpAgent(config);\nconst response = await fetch(url, { dispatcher: agent });\n```\n\n3. **Metrics Exposure Interface** (src/cli/src/registry/metrics.ts):\n```typescript\nexport interface RegistryMetrics {\n queryLatencyMs: { source: string; latencies: number[] }[];\n cacheHitRatio: number;\n cacheHits: number;\n cacheMisses: number;\n errorCounts: { source: string; count: number }[];\n}\n\nexport function collectMetrics(client: RegistryClient): RegistryMetrics {\n const cacheMetrics = client.getCacheMetrics();\n return {\n queryLatencyMs: Array.from(client.getQueryLatencies().entries())\n .map(([source, latencies]) => ({ source, latencies })),\n cacheHitRatio: cacheMetrics.ratio,\n cacheHits: cacheMetrics.hits,\n cacheMisses: cacheMetrics.misses,\n errorCounts: Array.from(client.getErrorCounts().entries())\n .map(([source, count]) => ({ source, count })),\n };\n}\n```\n\n4. **Update RegistryClientConfig** to include caPath:\n```typescript\nexport interface RegistryClientConfig {\n registries?: RegistryName[];\n cacheTTLMs?: number;\n smitheryApiKey?: string;\n httpProxy?: string;\n httpsProxy?: string;\n caPath?: string; // ADD THIS\n}\n```\n\n5. **Add data platform category filter** - update SearchOptions:\n```typescript\ncategory?: 'devops' | 'data-platform' | 'analytics' | 'communication' | 'development' | string;\n```\n\nTDD approach:\n- Write tests for createHttpAgent() with proxy, CA, and combined configs\n- Write tests for metrics collection interface\n- Write tests for category filtering in search results\n- All tests should be written BEFORE implementation", "testStrategy": "1. Unit tests for http-agent.ts: verify ProxyAgent created with correct proxy URI, verify custom CA loaded from file path, verify combined proxy+CA configuration\n2. Unit tests for metrics.ts: verify collectMetrics() returns correct structure, verify latency arrays are captured per-source\n3. Integration test: mock HTTP server with self-signed cert, verify client connects with custom CA\n4. Test category filtering returns only servers matching category\n5. Run existing test suite to ensure no regressions: pnpm --filter @mcpctl/cli test", "priority": "high", "dependencies": [], - "status": "pending", - "subtasks": [] + "status": "done", + "subtasks": [], + "updatedAt": "2026-02-21T03:52:54.909Z" }, { - "id": 26, + "id": "26", "title": "Implement mcpctl discover Command with Interactive Mode", "description": "Create the `mcpctl discover` CLI command that lets users search for MCP servers across all configured registries with rich filtering, table/JSON/YAML output, and an interactive browsing mode using inquirer.", "details": "Create src/cli/src/commands/discover.ts with Commander.js:\n\n```typescript\nimport { Command } from 'commander';\nimport { RegistryClient, type SearchOptions, type RegistryServer } from '../registry/index.js';\nimport { getConfig } from '../config/index.js';\nimport inquirer from 'inquirer';\nimport chalk from 'chalk';\n\nexport function createDiscoverCommand(): Command {\n return new Command('discover')\n .description('Search for MCP servers across registries')\n .argument('', 'Search query (e.g., \"slack\", \"database\", \"terraform\")')\n .option('-c, --category ', 'Filter by category (devops, data-platform, analytics)')\n .option('-v, --verified', 'Only show verified servers')\n .option('-t, --transport ', 'Filter by transport (stdio, sse)', undefined)\n .option('-r, --registry ', 'Query specific registry (official, glama, smithery, all)', 'all')\n .option('-l, --limit ', 'Maximum results', '20')\n .option('-o, --output ', 'Output format (table, json, yaml)', 'table')\n .option('-i, --interactive', 'Interactive browsing mode')\n .action(async (query, options) => {\n const config = await getConfig();\n const client = new RegistryClient({\n smitheryApiKey: config.smitheryApiKey,\n httpProxy: config.httpProxy,\n httpsProxy: config.httpsProxy,\n caPath: config.caPath,\n });\n\n const searchOpts: SearchOptions = {\n query,\n limit: parseInt(options.limit, 10),\n verified: options.verified,\n transport: options.transport,\n category: options.category,\n registries: options.registry === 'all' \n ? undefined \n : [options.registry],\n };\n\n const results = await client.search(searchOpts);\n\n if (results.length === 0) {\n console.log('No servers found matching your query.');\n process.exitCode = 2;\n return;\n }\n\n if (options.interactive) {\n await runInteractiveMode(results);\n } else {\n outputResults(results, options.output);\n }\n });\n}\n\nfunction outputResults(results: RegistryServer[], format: string): void {\n switch (format) {\n case 'json':\n console.log(JSON.stringify(results, null, 2));\n break;\n case 'yaml':\n // Use yaml library\n import('yaml').then(yaml => console.log(yaml.stringify(results)));\n break;\n default:\n printTable(results);\n }\n}\n\nfunction printTable(results: RegistryServer[]): void {\n console.log('NAME'.padEnd(30) + 'DESCRIPTION'.padEnd(50) + 'PACKAGE'.padEnd(35) + 'TRANSPORT VERIFIED POPULARITY');\n console.log('-'.repeat(140));\n for (const s of results) {\n const pkg = s.packages.npm ?? s.packages.pypi ?? s.packages.docker ?? '-';\n const verified = s.verified ? chalk.green('āœ“') : '-';\n console.log(\n s.name.slice(0, 28).padEnd(30) +\n s.description.slice(0, 48).padEnd(50) +\n pkg.slice(0, 33).padEnd(35) +\n s.transport.padEnd(11) +\n verified.padEnd(10) +\n String(s.popularityScore)\n );\n }\n console.log(`\\nRun 'mcpctl install ' to set up a server.`);\n}\n\nasync function runInteractiveMode(results: RegistryServer[]): Promise {\n const { selected } = await inquirer.prompt([{\n type: 'list',\n name: 'selected',\n message: 'Select an MCP server to install:',\n choices: results.map(s => ({\n name: `${s.name} - ${s.description.slice(0, 60)}`,\n value: s,\n })),\n }]);\n\n const { action } = await inquirer.prompt([{\n type: 'list',\n name: 'action',\n message: `What would you like to do with ${selected.name}?`,\n choices: [\n { name: 'Install and configure', value: 'install' },\n { name: 'View details', value: 'details' },\n { name: 'Cancel', value: 'cancel' },\n ],\n }]);\n\n if (action === 'install') {\n // Invoke install command programmatically\n const { execSync } = await import('node:child_process');\n execSync(`mcpctl install ${selected.name}`, { stdio: 'inherit' });\n } else if (action === 'details') {\n console.log(JSON.stringify(selected, null, 2));\n }\n}\n```\n\nRegister command in src/cli/src/commands/index.ts.\n\nExit codes for scripting:\n- 0: Results found\n- 1: Error occurred\n- 2: No results found\n\nTDD: Write all tests BEFORE implementation:\n- Test command parsing with all options\n- Test table output formatting\n- Test JSON/YAML output\n- Test exit codes\n- Mock inquirer for interactive mode tests", "testStrategy": "1. Unit tests (src/cli/tests/commands/discover.test.ts):\n - Test argument parsing: verify query is required, options have defaults\n - Test table output: mock RegistryClient, verify correct columns printed\n - Test JSON output: verify valid JSON with all fields\n - Test YAML output: verify valid YAML structure\n - Test --verified filter is passed to client\n - Test --registry parses correctly\n2. Integration tests:\n - Mock registry sources, run full discover command, verify output\n - Test exit code 2 when no results\n - Test exit code 1 on network error\n3. Interactive mode tests:\n - Mock inquirer responses, verify correct server selected\n - Verify install command invoked with correct name\n4. Run: pnpm --filter @mcpctl/cli test", "priority": "high", "dependencies": [ - 25 + "25" ], "status": "pending", "subtasks": [] }, { - "id": 27, + "id": "27", "title": "Implement mcpctl install with LLM-Assisted Auto-Configuration", "description": "Create the `mcpctl install ` command that uses a local LLM (Claude Code session, Ollama, or configured provider) to automatically analyze MCP server READMEs, generate envTemplate and setup guides, walk users through configuration, and register the server in mcpd.", "details": "Create src/cli/src/commands/install.ts:\n\n```typescript\nimport { Command } from 'commander';\nimport { RegistryClient, type RegistryServer, type EnvVar } from '../registry/index.js';\nimport { getConfig } from '../config/index.js';\nimport { z } from 'zod';\nimport inquirer from 'inquirer';\n\n// Zod schema for validating LLM-generated envTemplate\nconst LLMEnvVarSchema = z.object({\n name: z.string().min(1),\n description: z.string(),\n isSecret: z.boolean(),\n setupUrl: z.string().url().optional(),\n defaultValue: z.string().optional(),\n});\n\nconst LLMConfigResponseSchema = z.object({\n envTemplate: z.array(LLMEnvVarSchema),\n setupGuide: z.array(z.string()),\n defaultProfiles: z.array(z.object({\n name: z.string(),\n permissions: z.array(z.string()),\n })).optional().default([]),\n});\n\nexport type LLMConfigResponse = z.infer;\n\nexport function createInstallCommand(): Command {\n return new Command('install')\n .description('Install and configure an MCP server')\n .argument('', 'Server name(s) from discover results')\n .option('--non-interactive', 'Use env vars for credentials (no prompts)')\n .option('--profile-name ', 'Name for the created profile')\n .option('--project ', 'Add to existing project after install')\n .option('--dry-run', 'Show configuration without applying')\n .option('--skip-llm', 'Skip LLM analysis, use registry metadata only')\n .action(async (servers, options) => {\n for (const serverName of servers) {\n await installServer(serverName, options);\n }\n });\n}\n\nasync function installServer(serverName: string, options: {\n nonInteractive?: boolean;\n profileName?: string;\n project?: string;\n dryRun?: boolean;\n skipLlm?: boolean;\n}): Promise {\n const config = await getConfig();\n const client = new RegistryClient(config);\n\n // Step 1: Fetch server metadata from registry\n console.log(`Searching for ${serverName}...`);\n const results = await client.search({ query: serverName, limit: 10 });\n const server = results.find(s => \n s.name.toLowerCase() === serverName.toLowerCase() ||\n s.packages.npm?.includes(serverName)\n );\n\n if (!server) {\n console.error(`Server \"${serverName}\" not found. Run 'mcpctl discover ${serverName}' to search.`);\n process.exitCode = 1;\n return;\n }\n\n console.log(`Found: ${server.name} (${server.packages.npm ?? server.packages.docker ?? 'N/A'})`);\n\n // Step 2: Determine envTemplate\n let envTemplate: EnvVar[] = server.envTemplate;\n let setupGuide: string[] = [];\n\n // Step 3: If envTemplate incomplete and LLM not skipped, use LLM\n if (envTemplate.length === 0 && !options.skipLlm && server.repositoryUrl) {\n console.log('Registry metadata incomplete. Analyzing README with LLM...');\n const llmConfig = await analyzeWithLLM(server.repositoryUrl, config);\n if (llmConfig) {\n envTemplate = llmConfig.envTemplate;\n setupGuide = llmConfig.setupGuide;\n }\n }\n\n // Step 4: Show setup guide if available\n if (setupGuide.length > 0) {\n console.log('\\nšŸ“‹ Setup Guide:');\n setupGuide.forEach((step, i) => console.log(` ${i + 1}. ${step}`));\n console.log('');\n }\n\n if (options.dryRun) {\n console.log('Dry run - would configure:');\n console.log(JSON.stringify({ server, envTemplate }, null, 2));\n return;\n }\n\n // Step 5: Collect credentials\n const credentials: Record = {};\n if (!options.nonInteractive) {\n for (const env of envTemplate) {\n const { value } = await inquirer.prompt([{\n type: env.isSecret ? 'password' : 'input',\n name: 'value',\n message: `${env.name}${env.description ? ` (${env.description})` : ''}:`,\n default: env.defaultValue,\n }]);\n credentials[env.name] = value;\n }\n } else {\n // Use environment variables\n for (const env of envTemplate) {\n credentials[env.name] = process.env[env.name] ?? env.defaultValue ?? '';\n }\n }\n\n // Step 6: Register with mcpd (mock for now until mcpd integration)\n console.log(`\\nRegistering ${server.name} with mcpd...`);\n // TODO: POST to mcpd /api/mcp-servers when mcpd is implemented\n // For now, write to local config\n await saveServerConfig(server, credentials, options.profileName ?? server.name);\n\n // Step 7: Add to project if specified\n if (options.project) {\n console.log(`Adding to project: ${options.project}`);\n // TODO: Call mcpd project API\n }\n\n console.log(`\\nāœ… ${server.name} installed successfully!`);\n console.log(`Run 'mcpctl get servers' to see installed servers.`);\n}\n\nasync function analyzeWithLLM(repoUrl: string, config: any): Promise {\n try {\n // Fetch README from GitHub\n const readmeUrl = convertToRawReadmeUrl(repoUrl);\n const response = await fetch(readmeUrl);\n if (!response.ok) {\n console.warn('Could not fetch README.');\n return null;\n }\n const readme = await response.text();\n\n // Sanitize README - prevent prompt injection\n const sanitizedReadme = sanitizeReadme(readme);\n\n // Use configured LLM provider (Ollama, OpenAI, etc. from Task 12)\n // For Claude Code integration, output prompt for user to paste\n const prompt = buildLLMPrompt(sanitizedReadme);\n \n // TODO: Integrate with actual LLM provider from Task 12\n // For now, attempt Ollama if configured\n const llmResponse = await callLLM(prompt, config);\n \n // Parse and validate response\n const parsed = JSON.parse(llmResponse);\n return LLMConfigResponseSchema.parse(parsed);\n } catch (error) {\n console.warn('LLM analysis failed, using registry metadata only.');\n return null;\n }\n}\n\nfunction buildLLMPrompt(readme: string): string {\n return `Analyze this MCP server README and extract configuration requirements.\n\nRETURN ONLY VALID JSON matching this schema:\n{\n \"envTemplate\": [{ \"name\": string, \"description\": string, \"isSecret\": boolean, \"setupUrl\"?: string }],\n \"setupGuide\": [\"Step 1...\", \"Step 2...\"],\n \"defaultProfiles\": [{ \"name\": string, \"permissions\": string[] }]\n}\n\nREADME content (trusted, from official repository):\n${readme.slice(0, 8000)}\n\nJSON output:`;\n}\n\nfunction sanitizeReadme(readme: string): string {\n // Remove potential prompt injection patterns\n return readme\n .replace(/ignore.*instructions/gi, '')\n .replace(/disregard.*above/gi, '')\n .replace(/system.*prompt/gi, '');\n}\n\nfunction convertToRawReadmeUrl(repoUrl: string): string {\n // Convert GitHub repo URL to raw README URL\n const match = repoUrl.match(/github\\.com\\/([^/]+)\\/([^/]+)/);\n if (match) {\n return `https://raw.githubusercontent.com/${match[1]}/${match[2]}/main/README.md`;\n }\n return repoUrl;\n}\n\nasync function callLLM(prompt: string, config: any): Promise {\n // Try Ollama first if available\n if (config.ollamaUrl) {\n const response = await fetch(`${config.ollamaUrl}/api/generate`, {\n method: 'POST',\n headers: { 'Content-Type': 'application/json' },\n body: JSON.stringify({\n model: config.ollamaModel ?? 'llama3',\n prompt,\n stream: false,\n }),\n });\n const data = await response.json();\n return data.response;\n }\n throw new Error('No LLM provider configured. Set OLLAMA_URL or use --skip-llm.');\n}\n\nasync function saveServerConfig(server: RegistryServer, credentials: Record, profileName: string): Promise {\n // Save to ~/.mcpctl/servers/.json\n const fs = await import('node:fs/promises');\n const path = await import('node:path');\n const os = await import('node:os');\n \n const configDir = path.join(os.homedir(), '.mcpctl', 'servers');\n await fs.mkdir(configDir, { recursive: true });\n \n await fs.writeFile(\n path.join(configDir, `${profileName}.json`),\n JSON.stringify({ server, credentials, createdAt: new Date().toISOString() }, null, 2)\n );\n}\n```\n\nSecurity considerations:\n- sanitizeReadme() removes prompt injection patterns\n- LLM responses validated against Zod schema before use\n- Never auto-execute commands suggested by LLM\n- Credentials stored in separate secure config (encrypted via Task 7.2)\n\nTDD: Write comprehensive tests BEFORE implementation.", "testStrategy": "1. Unit tests (src/cli/tests/commands/install.test.ts):\n - Test server lookup from registry results\n - Test LLMConfigResponseSchema validates correct JSON\n - Test LLMConfigResponseSchema rejects invalid JSON\n - Test sanitizeReadme() removes injection patterns\n - Test buildLLMPrompt() generates valid prompt structure\n - Test convertToRawReadmeUrl() for various GitHub URL formats\n - Test --dry-run outputs config without saving\n - Test --non-interactive uses env vars\n - Test batch install: multiple servers processed sequentially\n2. Security tests:\n - Test sanitizeReadme blocks 'ignore all instructions'\n - Test LLM response with extra fields is safely parsed\n - Test credentials are not logged\n3. Integration tests:\n - Mock registry client and LLM endpoint\n - Full install flow with mocked dependencies\n - Verify server config file created with correct structure\n4. Run: pnpm --filter @mcpctl/cli test", "priority": "high", "dependencies": [ - 25, - 26 + "25", + "26" ], "status": "pending", "subtasks": [] } ], "metadata": { - "created": "2026-02-21T03:25:05.784Z", - "updated": "2026-02-21T03:25:05.784Z", - "description": "Tasks for master context" + "version": "1.0.0", + "lastModified": "2026-02-21T03:52:54.909Z", + "taskCount": 27, + "completedCount": 2, + "tags": [ + "master" + ] } } } \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 253e672..ce678b4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -59,6 +59,9 @@ importers: js-yaml: specifier: ^4.1.0 version: 4.1.1 + undici: + specifier: ^7.22.0 + version: 7.22.0 zod: specifier: ^3.24.0 version: 3.25.76 @@ -1807,6 +1810,10 @@ packages: undici-types@7.18.2: resolution: {integrity: sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==} + undici@7.22.0: + resolution: {integrity: sha512-RqslV2Us5BrllB+JeiZnK4peryVTndy9Dnqq62S3yYRRTj0tFQCwEniUy2167skdGOy3vqRzEvl1Dm4sV2ReDg==} + engines: {node: '>=20.18.1'} + unpipe@1.0.0: resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} engines: {node: '>= 0.8'} @@ -3580,6 +3587,8 @@ snapshots: undici-types@7.18.2: {} + undici@7.22.0: {} + unpipe@1.0.0: {} uri-js@4.4.1: diff --git a/pr.sh b/pr.sh new file mode 100755 index 0000000..eee874d --- /dev/null +++ b/pr.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# pr.sh - Create PRs on Gitea from current branch +# Usage: ./pr.sh [base_branch] [title] +set -euo pipefail + +GITEA_API="http://10.0.0.194:3012/api/v1" +GITEA_PUBLIC="https://mysources.co.uk" +GITEA_TOKEN="$(grep '^GITEA_TOKEN=' /home/michal/developer/michalzxc/claude/homeassistant-alchemy/stack/.env | cut -d= -f2-)" +REPO="michal/mcpctl" + +if [[ -z "$GITEA_TOKEN" ]]; then + echo "Error: GITEA_TOKEN not found" >&2 + exit 1 +fi + +BRANCH=$(git branch --show-current) +BASE="${1:-main}" +TITLE="${2:-}" + +if [[ "$BRANCH" == "$BASE" ]]; then + echo "Error: already on $BASE, switch to a feature branch first" >&2 + exit 1 +fi + +# Check for existing open PR for this branch +EXISTING=$(curl -s "$GITEA_API/repos/$REPO/pulls?state=open&head=$BRANCH" \ + -H "Authorization: token $GITEA_TOKEN" | jq -r '.[0].number // empty' 2>/dev/null) + +if [[ -n "$EXISTING" ]]; then + echo "PR #$EXISTING already exists for $BRANCH" + echo "$GITEA_PUBLIC/$REPO/pulls/$EXISTING" + exit 0 +fi + +# Auto-generate title from branch name if not provided +if [[ -z "$TITLE" ]]; then + TITLE=$(echo "$BRANCH" | sed 's|^feat/||;s|^fix/||;s|^chore/||' | tr '-' ' ' | sed 's/.*/\u&/') +fi + +# Build body from commit messages on this branch +COMMITS=$(git log "$BASE..$BRANCH" --pretty=format:"- %s" 2>/dev/null) +BODY="## Summary +${COMMITS} + +--- +Generated with [Claude Code](https://claude.com/claude-code)" + +# Push if needed +if ! git rev-parse --verify "origin/$BRANCH" &>/dev/null; then + echo "Pushing $BRANCH to origin..." + git push -u origin "$BRANCH" +fi + +# Create PR +RESPONSE=$(curl -s -X POST "$GITEA_API/repos/$REPO/pulls" \ + -H "Content-Type: application/json" \ + -H "Authorization: token $GITEA_TOKEN" \ + -d "$(jq -n --arg title "$TITLE" --arg body "$BODY" --arg head "$BRANCH" --arg base "$BASE" \ + '{title: $title, body: $body, head: $head, base: $base}')") + +PR_NUM=$(echo "$RESPONSE" | jq -r '.number // empty') +if [[ -z "$PR_NUM" ]]; then + echo "Error creating PR: $(echo "$RESPONSE" | jq -r '.message // "unknown error"')" >&2 + exit 1 +fi + +echo "Created PR #$PR_NUM: $TITLE" +echo "$GITEA_PUBLIC/$REPO/pulls/$PR_NUM" diff --git a/src/cli/package.json b/src/cli/package.json index 0726d48..51cfa5c 100644 --- a/src/cli/package.json +++ b/src/cli/package.json @@ -22,6 +22,7 @@ "commander": "^13.0.0", "inquirer": "^12.0.0", "js-yaml": "^4.1.0", + "undici": "^7.22.0", "zod": "^3.24.0" } } diff --git a/src/cli/src/registry/base.ts b/src/cli/src/registry/base.ts index aff9e8d..00bc7e6 100644 --- a/src/cli/src/registry/base.ts +++ b/src/cli/src/registry/base.ts @@ -2,8 +2,21 @@ import type { RegistryServer } from './types.js'; export abstract class RegistrySource { abstract readonly name: string; + protected dispatcher: unknown | undefined; + + setDispatcher(dispatcher: unknown | undefined): void { + this.dispatcher = dispatcher; + } abstract search(query: string, limit: number): Promise; protected abstract normalizeResult(raw: unknown): RegistryServer; + + protected fetchWithDispatcher(url: string): Promise { + if (this.dispatcher) { + // Node.js built-in fetch accepts undici dispatcher option + return fetch(url, { dispatcher: this.dispatcher } as RequestInit); + } + return fetch(url); + } } diff --git a/src/cli/src/registry/client.ts b/src/cli/src/registry/client.ts index 32feb06..f55c69c 100644 --- a/src/cli/src/registry/client.ts +++ b/src/cli/src/registry/client.ts @@ -6,6 +6,7 @@ import { SmitheryRegistrySource } from './sources/smithery.js'; import { RegistryCache } from './cache.js'; import { deduplicateResults } from './dedup.js'; import { rankResults } from './ranking.js'; +import { createHttpAgent } from './http-agent.js'; export class RegistryClient { private sources: Map; @@ -20,11 +21,27 @@ export class RegistryClient { this.enabledRegistries = config.registries ?? ['official', 'glama', 'smithery']; this.cache = new RegistryCache(config.cacheTTLMs); - this.sources = new Map([ + // Create HTTP agent for proxy/CA support + const dispatcher = createHttpAgent({ + httpProxy: config.httpProxy, + httpsProxy: config.httpsProxy, + caPath: config.caPath, + }); + + const sources: [RegistryName, RegistrySource][] = [ ['official', new OfficialRegistrySource()], ['glama', new GlamaRegistrySource()], ['smithery', new SmitheryRegistrySource()], - ]); + ]; + + // Set dispatcher on all sources + if (dispatcher) { + for (const [, source] of sources) { + source.setDispatcher(dispatcher); + } + } + + this.sources = new Map(sources); } async search(options: SearchOptions): Promise { @@ -64,6 +81,12 @@ export class RegistryClient { if (options.transport !== undefined) { combined = combined.filter((s) => s.transport === options.transport); } + if (options.category !== undefined) { + const cat = options.category.toLowerCase(); + combined = combined.filter((s) => + s.category !== undefined && s.category.toLowerCase() === cat + ); + } // Deduplicate, rank, and limit const deduped = deduplicateResults(combined); diff --git a/src/cli/src/registry/http-agent.ts b/src/cli/src/registry/http-agent.ts new file mode 100644 index 0000000..1675e94 --- /dev/null +++ b/src/cli/src/registry/http-agent.ts @@ -0,0 +1,26 @@ +import fs from 'node:fs'; +import { Agent, ProxyAgent } from 'undici'; + +export interface HttpAgentConfig { + httpProxy?: string; + httpsProxy?: string; + caPath?: string; +} + +export function createHttpAgent(config: HttpAgentConfig): Agent | ProxyAgent | undefined { + const proxy = (config.httpsProxy ?? config.httpProxy) || undefined; + const caPath = config.caPath || undefined; + + if (!proxy && !caPath) return undefined; + + const ca = caPath ? fs.readFileSync(caPath) : undefined; + + if (proxy) { + return new ProxyAgent({ + uri: proxy, + connect: ca ? { ca } : undefined, + }); + } + + return new Agent({ connect: { ca } }); +} diff --git a/src/cli/src/registry/index.ts b/src/cli/src/registry/index.ts index ffaf929..0d2724f 100644 --- a/src/cli/src/registry/index.ts +++ b/src/cli/src/registry/index.ts @@ -4,6 +4,8 @@ export { RegistrySource } from './base.js'; export { deduplicateResults } from './dedup.js'; export { rankResults } from './ranking.js'; export { withRetry } from './retry.js'; +export { createHttpAgent, type HttpAgentConfig } from './http-agent.js'; +export { collectMetrics, type RegistryMetrics } from './metrics.js'; export { OfficialRegistrySource } from './sources/official.js'; export { GlamaRegistrySource } from './sources/glama.js'; export { SmitheryRegistrySource } from './sources/smithery.js'; diff --git a/src/cli/src/registry/metrics.ts b/src/cli/src/registry/metrics.ts new file mode 100644 index 0000000..d05832b --- /dev/null +++ b/src/cli/src/registry/metrics.ts @@ -0,0 +1,22 @@ +import type { RegistryClient } from './client.js'; + +export interface RegistryMetrics { + queryLatencyMs: { source: string; latencies: number[] }[]; + cacheHitRatio: number; + cacheHits: number; + cacheMisses: number; + errorCounts: { source: string; count: number }[]; +} + +export function collectMetrics(client: RegistryClient): RegistryMetrics { + const cacheMetrics = client.getCacheMetrics(); + return { + queryLatencyMs: Array.from(client.getQueryLatencies().entries()) + .map(([source, latencies]) => ({ source, latencies })), + cacheHitRatio: cacheMetrics.ratio, + cacheHits: cacheMetrics.hits, + cacheMisses: cacheMetrics.misses, + errorCounts: Array.from(client.getErrorCounts().entries()) + .map(([source, count]) => ({ source, count })), + }; +} diff --git a/src/cli/src/registry/sources/glama.ts b/src/cli/src/registry/sources/glama.ts index f2d1796..d93e6cc 100644 --- a/src/cli/src/registry/sources/glama.ts +++ b/src/cli/src/registry/sources/glama.ts @@ -23,7 +23,7 @@ export class GlamaRegistrySource extends RegistrySource { url.searchParams.set('after', cursor); } - const response = await withRetry(() => fetch(url.toString())); + const response = await withRetry(() => this.fetchWithDispatcher(url.toString())); if (!response.ok) { throw new Error(`Glama registry returned ${String(response.status)}`); } @@ -74,6 +74,10 @@ export class GlamaRegistrySource extends RegistrySource { packages.npm = entry.slug; } + // Extract category from attributes (e.g. "category:devops" -> "devops") + const categoryAttr = attrs.find((a: string) => a.startsWith('category:')); + const category = categoryAttr ? categoryAttr.split(':')[1] : undefined; + const result: RegistryServer = { name: sanitizeString(entry.name), description: sanitizeString(entry.description), @@ -84,6 +88,9 @@ export class GlamaRegistrySource extends RegistrySource { verified: attrs.includes('author:official'), sourceRegistry: 'glama', }; + if (category !== undefined) { + result.category = category; + } if (entry.repository?.url !== undefined) { result.repositoryUrl = entry.repository.url; } diff --git a/src/cli/src/registry/sources/official.ts b/src/cli/src/registry/sources/official.ts index 3771ea1..29c0c05 100644 --- a/src/cli/src/registry/sources/official.ts +++ b/src/cli/src/registry/sources/official.ts @@ -24,7 +24,7 @@ export class OfficialRegistrySource extends RegistrySource { url.searchParams.set('cursor', cursor); } - const response = await withRetry(() => fetch(url.toString())); + const response = await withRetry(() => this.fetchWithDispatcher(url.toString())); if (!response.ok) { throw new Error(`Official registry returned ${String(response.status)}`); } diff --git a/src/cli/src/registry/sources/smithery.ts b/src/cli/src/registry/sources/smithery.ts index b5cf563..70e85c1 100644 --- a/src/cli/src/registry/sources/smithery.ts +++ b/src/cli/src/registry/sources/smithery.ts @@ -22,7 +22,7 @@ export class SmitheryRegistrySource extends RegistrySource { url.searchParams.set('pageSize', String(Math.min(limit - results.length, 50))); url.searchParams.set('page', String(page)); - const response = await withRetry(() => fetch(url.toString())); + const response = await withRetry(() => this.fetchWithDispatcher(url.toString())); if (!response.ok) { throw new Error(`Smithery registry returned ${String(response.status)}`); } diff --git a/src/cli/src/registry/types.ts b/src/cli/src/registry/types.ts index 3155e12..7a98238 100644 --- a/src/cli/src/registry/types.ts +++ b/src/cli/src/registry/types.ts @@ -23,6 +23,7 @@ export interface RegistryServer { repositoryUrl?: string; popularityScore: number; verified: boolean; + category?: string; sourceRegistry: 'official' | 'glama' | 'smithery'; lastUpdated?: Date; } @@ -44,6 +45,7 @@ export interface RegistryClientConfig { smitheryApiKey?: string; httpProxy?: string; httpsProxy?: string; + caPath?: string; } // ā”€ā”€ Zod schemas for API response validation ā”€ā”€ diff --git a/src/cli/tests/registry/http-agent.test.ts b/src/cli/tests/registry/http-agent.test.ts new file mode 100644 index 0000000..e198818 --- /dev/null +++ b/src/cli/tests/registry/http-agent.test.ts @@ -0,0 +1,89 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { createHttpAgent } from '../../src/registry/http-agent.js'; + +// Mock undici with proper constructable classes +vi.mock('undici', () => { + class MockAgent { + __type = 'Agent'; + __opts: unknown; + constructor(opts: unknown) { + this.__opts = opts; + } + } + class MockProxyAgent { + __type = 'ProxyAgent'; + __opts: unknown; + constructor(opts: unknown) { + this.__opts = opts; + } + } + return { Agent: MockAgent, ProxyAgent: MockProxyAgent }; +}); + +// Mock fs +vi.mock('node:fs', () => ({ + default: { + readFileSync: vi.fn().mockReturnValue(Buffer.from('mock-ca-cert')), + }, + readFileSync: vi.fn().mockReturnValue(Buffer.from('mock-ca-cert')), +})); + +describe('createHttpAgent', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('returns undefined when no proxy and no CA configured', () => { + const result = createHttpAgent({}); + expect(result).toBeUndefined(); + }); + + it('returns undefined when config has empty strings', () => { + const result = createHttpAgent({ httpProxy: '', httpsProxy: '' }); + expect(result).toBeUndefined(); + }); + + it('returns a ProxyAgent when httpProxy is configured', () => { + const result = createHttpAgent({ httpProxy: 'http://proxy:8080' }) as { __type: string }; + expect(result).toBeDefined(); + expect(result.__type).toBe('ProxyAgent'); + }); + + it('returns a ProxyAgent when httpsProxy is configured', () => { + const result = createHttpAgent({ httpsProxy: 'http://proxy:8443' }) as { __type: string }; + expect(result).toBeDefined(); + expect(result.__type).toBe('ProxyAgent'); + }); + + it('prefers httpsProxy over httpProxy', () => { + const result = createHttpAgent({ + httpProxy: 'http://proxy:8080', + httpsProxy: 'http://proxy:8443', + }) as { __type: string; __opts: { uri: string } }; + expect(result.__type).toBe('ProxyAgent'); + expect(result.__opts.uri).toBe('http://proxy:8443'); + }); + + it('returns an Agent with CA when only caPath is configured', () => { + const result = createHttpAgent({ caPath: '/path/to/ca.pem' }) as { __type: string }; + expect(result).toBeDefined(); + expect(result.__type).toBe('Agent'); + }); + + it('returns a ProxyAgent with CA when both proxy and caPath are configured', () => { + const result = createHttpAgent({ + httpsProxy: 'http://proxy:8443', + caPath: '/path/to/ca.pem', + }) as { __type: string; __opts: { uri: string; connect: { ca: Buffer } } }; + expect(result.__type).toBe('ProxyAgent'); + expect(result.__opts.uri).toBe('http://proxy:8443'); + expect(result.__opts.connect).toBeDefined(); + expect(result.__opts.connect.ca).toBeDefined(); + }); + + it('reads CA file from filesystem', async () => { + const fs = await import('node:fs'); + createHttpAgent({ caPath: '/etc/ssl/custom-ca.pem' }); + expect(fs.default.readFileSync).toHaveBeenCalledWith('/etc/ssl/custom-ca.pem'); + }); +}); diff --git a/src/cli/tests/registry/metrics.test.ts b/src/cli/tests/registry/metrics.test.ts new file mode 100644 index 0000000..cdbee2d --- /dev/null +++ b/src/cli/tests/registry/metrics.test.ts @@ -0,0 +1,164 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { collectMetrics, type RegistryMetrics } from '../../src/registry/metrics.js'; +import { RegistryClient } from '../../src/registry/client.js'; +import type { RegistryServer } from '../../src/registry/types.js'; + +const mockFetch = vi.fn(); + +function makeServer(name: string, source: 'official' | 'glama' | 'smithery'): RegistryServer { + return { + name, + description: `${name} description`, + packages: { npm: `@test/${name}` }, + envTemplate: [], + transport: 'stdio', + popularityScore: 50, + verified: false, + sourceRegistry: source, + }; +} + +function mockAllRegistries(servers: RegistryServer[]): void { + mockFetch.mockImplementation((url: string) => { + if (url.includes('registry.modelcontextprotocol.io')) { + return Promise.resolve({ + ok: true, + json: () => Promise.resolve({ + servers: servers + .filter((s) => s.sourceRegistry === 'official') + .map((s) => ({ + server: { + name: s.name, + description: s.description, + packages: [{ registryType: 'npm', identifier: s.packages.npm, transport: { type: 'stdio' }, environmentVariables: [] }], + remotes: [], + }, + })), + metadata: { nextCursor: null }, + }), + }); + } + if (url.includes('glama.ai')) { + return Promise.resolve({ + ok: true, + json: () => Promise.resolve({ + servers: servers + .filter((s) => s.sourceRegistry === 'glama') + .map((s) => ({ id: s.name, name: s.name, description: s.description, attributes: [], slug: '' })), + pageInfo: { hasNextPage: false, hasPreviousPage: false }, + }), + }); + } + if (url.includes('registry.smithery.ai')) { + return Promise.resolve({ + ok: true, + json: () => Promise.resolve({ + servers: servers + .filter((s) => s.sourceRegistry === 'smithery') + .map((s) => ({ qualifiedName: s.name, displayName: s.name, description: s.description, verified: false, useCount: 0, remote: false })), + pagination: { currentPage: 1, pageSize: 20, totalPages: 1, totalCount: 1 }, + }), + }); + } + return Promise.reject(new Error(`Unexpected URL: ${url}`)); + }); +} + +describe('collectMetrics', () => { + beforeEach(() => { + vi.stubGlobal('fetch', mockFetch); + mockFetch.mockReset(); + }); + + it('returns correct structure with all required fields', async () => { + mockAllRegistries([makeServer('test', 'official')]); + const client = new RegistryClient(); + await client.search({ query: 'test' }); + + const metrics = collectMetrics(client); + + expect(metrics).toHaveProperty('queryLatencyMs'); + expect(metrics).toHaveProperty('cacheHitRatio'); + expect(metrics).toHaveProperty('cacheHits'); + expect(metrics).toHaveProperty('cacheMisses'); + expect(metrics).toHaveProperty('errorCounts'); + expect(Array.isArray(metrics.queryLatencyMs)).toBe(true); + expect(Array.isArray(metrics.errorCounts)).toBe(true); + expect(typeof metrics.cacheHitRatio).toBe('number'); + }); + + it('captures latencies per source', async () => { + mockAllRegistries([ + makeServer('test', 'official'), + makeServer('test', 'glama'), + makeServer('test', 'smithery'), + ]); + const client = new RegistryClient(); + await client.search({ query: 'test' }); + + const metrics = collectMetrics(client); + + expect(metrics.queryLatencyMs.length).toBeGreaterThan(0); + for (const entry of metrics.queryLatencyMs) { + expect(entry).toHaveProperty('source'); + expect(entry).toHaveProperty('latencies'); + expect(Array.isArray(entry.latencies)).toBe(true); + expect(entry.latencies.length).toBeGreaterThan(0); + } + }); + + it('captures cache hit ratio', async () => { + mockAllRegistries([makeServer('test', 'official')]); + const client = new RegistryClient(); + + // First call: miss + await client.search({ query: 'test' }); + // Second call: hit + await client.search({ query: 'test' }); + + const metrics = collectMetrics(client); + expect(metrics.cacheHits).toBe(1); + expect(metrics.cacheMisses).toBe(1); + expect(metrics.cacheHitRatio).toBe(0.5); + }); + + it('captures error counts per source', async () => { + mockFetch.mockImplementation((url: string) => { + if (url.includes('glama.ai')) { + return Promise.reject(new Error('fail')); + } + if (url.includes('registry.modelcontextprotocol.io')) { + return Promise.resolve({ + ok: true, + json: () => Promise.resolve({ servers: [], metadata: { nextCursor: null } }), + }); + } + return Promise.resolve({ + ok: true, + json: () => Promise.resolve({ + servers: [], + pagination: { currentPage: 1, pageSize: 20, totalPages: 1, totalCount: 0 }, + }), + }); + }); + + const client = new RegistryClient(); + await client.search({ query: 'test' }); + + const metrics = collectMetrics(client); + const glamaError = metrics.errorCounts.find((e) => e.source === 'glama'); + expect(glamaError).toBeDefined(); + expect(glamaError!.count).toBe(1); + }); + + it('works with empty metrics (no queries made)', () => { + const client = new RegistryClient(); + const metrics = collectMetrics(client); + + expect(metrics.queryLatencyMs).toEqual([]); + expect(metrics.errorCounts).toEqual([]); + expect(metrics.cacheHits).toBe(0); + expect(metrics.cacheMisses).toBe(0); + expect(metrics.cacheHitRatio).toBe(0); + }); +});