diff --git a/src/local-proxy/src/index.ts b/src/local-proxy/src/index.ts index 800ef88..ae3d83f 100644 --- a/src/local-proxy/src/index.ts +++ b/src/local-proxy/src/index.ts @@ -5,6 +5,9 @@ export { StdioUpstream, HttpUpstream } from './upstream/index.js'; export { HealthMonitor } from './health.js'; export type { HealthState, HealthStatus, HealthMonitorOptions } from './health.js'; export { main } from './main.js'; +export { ProviderRegistry } from './providers/index.js'; +export type { LlmProvider, CompletionOptions, CompletionResult, ChatMessage } from './providers/index.js'; +export { OpenAiProvider, AnthropicProvider, OllamaProvider } from './providers/index.js'; export type { JsonRpcRequest, JsonRpcResponse, diff --git a/src/local-proxy/src/providers/anthropic.ts b/src/local-proxy/src/providers/anthropic.ts new file mode 100644 index 0000000..ba4d7e2 --- /dev/null +++ b/src/local-proxy/src/providers/anthropic.ts @@ -0,0 +1,179 @@ +import https from 'node:https'; +import type { LlmProvider, CompletionOptions, CompletionResult, ChatMessage, ToolCall } from './types.js'; + +export interface AnthropicConfig { + apiKey: string; + defaultModel?: string; +} + +/** + * Anthropic Claude provider using the Messages API. + */ +export class AnthropicProvider implements LlmProvider { + readonly name = 'anthropic'; + private apiKey: string; + private defaultModel: string; + + constructor(config: AnthropicConfig) { + this.apiKey = config.apiKey; + this.defaultModel = config.defaultModel ?? 'claude-sonnet-4-20250514'; + } + + async complete(options: CompletionOptions): Promise { + const model = options.model ?? this.defaultModel; + + // Separate system message from conversation + const systemMessages = options.messages.filter((m) => m.role === 'system'); + const conversationMessages = options.messages.filter((m) => m.role !== 'system'); + + const body: Record = { + model, + max_tokens: options.maxTokens ?? 4096, + messages: conversationMessages.map(toAnthropicMessage), + }; + + if (systemMessages.length > 0) { + body.system = systemMessages.map((m) => m.content).join('\n'); + } + if (options.temperature !== undefined) body.temperature = options.temperature; + + if (options.tools && options.tools.length > 0) { + body.tools = options.tools.map((t) => ({ + name: t.name, + description: t.description, + input_schema: t.inputSchema, + })); + } + + const response = await this.request(body); + return parseAnthropicResponse(response); + } + + async listModels(): Promise { + // Anthropic doesn't have a models listing endpoint; return known models + return [ + 'claude-opus-4-20250514', + 'claude-sonnet-4-20250514', + 'claude-haiku-3-5-20241022', + ]; + } + + async isAvailable(): Promise { + try { + // Send a minimal request to check API key + await this.complete({ + messages: [{ role: 'user', content: 'hi' }], + maxTokens: 1, + }); + return true; + } catch { + return false; + } + } + + private request(body: unknown): Promise { + return new Promise((resolve, reject) => { + const payload = JSON.stringify(body); + const opts = { + hostname: 'api.anthropic.com', + port: 443, + path: '/v1/messages', + method: 'POST', + timeout: 120000, + headers: { + 'x-api-key': this.apiKey, + 'anthropic-version': '2023-06-01', + 'Content-Type': 'application/json', + 'Content-Length': Buffer.byteLength(payload), + }, + }; + + const req = https.request(opts, (res) => { + const chunks: Buffer[] = []; + res.on('data', (chunk: Buffer) => chunks.push(chunk)); + res.on('end', () => { + const raw = Buffer.concat(chunks).toString('utf-8'); + try { + resolve(JSON.parse(raw)); + } catch { + reject(new Error(`Invalid JSON response: ${raw.slice(0, 200)}`)); + } + }); + }); + req.on('error', reject); + req.on('timeout', () => { + req.destroy(); + reject(new Error('Request timed out')); + }); + req.write(payload); + req.end(); + }); + } +} + +function toAnthropicMessage(msg: ChatMessage): Record { + if (msg.role === 'tool') { + return { + role: 'user', + content: [{ + type: 'tool_result', + tool_use_id: msg.toolCallId, + content: msg.content, + }], + }; + } + return { + role: msg.role === 'user' ? 'user' : 'assistant', + content: msg.content, + }; +} + +function parseAnthropicResponse(raw: unknown): CompletionResult { + const data = raw as { + content?: Array<{ + type: string; + text?: string; + id?: string; + name?: string; + input?: Record; + }>; + stop_reason?: string; + usage?: { + input_tokens?: number; + output_tokens?: number; + }; + }; + + let content = ''; + const toolCalls: ToolCall[] = []; + + for (const block of data.content ?? []) { + if (block.type === 'text' && block.text) { + content += block.text; + } else if (block.type === 'tool_use' && block.id && block.name) { + toolCalls.push({ + id: block.id, + name: block.name, + arguments: block.input ?? {}, + }); + } + } + + const inputTokens = data.usage?.input_tokens ?? 0; + const outputTokens = data.usage?.output_tokens ?? 0; + + const finishReason = data.stop_reason === 'tool_use' ? 'tool_calls' as const + : data.stop_reason === 'max_tokens' ? 'length' as const + : 'stop' as const; + + return { + content, + toolCalls, + usage: { + promptTokens: inputTokens, + completionTokens: outputTokens, + totalTokens: inputTokens + outputTokens, + }, + finishReason, + }; +} diff --git a/src/local-proxy/src/providers/index.ts b/src/local-proxy/src/providers/index.ts new file mode 100644 index 0000000..0963ef8 --- /dev/null +++ b/src/local-proxy/src/providers/index.ts @@ -0,0 +1,8 @@ +export type { LlmProvider, CompletionOptions, CompletionResult, ChatMessage, ToolDefinition, ToolCall } from './types.js'; +export { OpenAiProvider } from './openai.js'; +export type { OpenAiConfig } from './openai.js'; +export { AnthropicProvider } from './anthropic.js'; +export type { AnthropicConfig } from './anthropic.js'; +export { OllamaProvider } from './ollama.js'; +export type { OllamaConfig } from './ollama.js'; +export { ProviderRegistry } from './registry.js'; diff --git a/src/local-proxy/src/providers/ollama.ts b/src/local-proxy/src/providers/ollama.ts new file mode 100644 index 0000000..9a059f1 --- /dev/null +++ b/src/local-proxy/src/providers/ollama.ts @@ -0,0 +1,138 @@ +import http from 'node:http'; +import type { LlmProvider, CompletionOptions, CompletionResult, ToolCall } from './types.js'; + +export interface OllamaConfig { + baseUrl?: string; + defaultModel?: string; +} + +/** + * Ollama provider for local model inference. + * Uses the Ollama HTTP API at /api/chat. + */ +export class OllamaProvider implements LlmProvider { + readonly name = 'ollama'; + private baseUrl: string; + private defaultModel: string; + + constructor(config?: OllamaConfig) { + this.baseUrl = (config?.baseUrl ?? 'http://localhost:11434').replace(/\/$/, ''); + this.defaultModel = config?.defaultModel ?? 'llama3.2'; + } + + async complete(options: CompletionOptions): Promise { + const model = options.model ?? this.defaultModel; + const body: Record = { + model, + messages: options.messages.map((m) => ({ + role: m.role === 'tool' ? 'assistant' : m.role, + content: m.content, + })), + stream: false, + }; + if (options.temperature !== undefined) { + body.options = { temperature: options.temperature }; + } + + if (options.tools && options.tools.length > 0) { + body.tools = options.tools.map((t) => ({ + type: 'function', + function: { + name: t.name, + description: t.description, + parameters: t.inputSchema, + }, + })); + } + + const response = await this.request('/api/chat', body); + return parseOllamaResponse(response); + } + + async listModels(): Promise { + const response = await this.request('/api/tags', undefined, 'GET') as { + models?: Array<{ name: string }>; + }; + return (response.models ?? []).map((m) => m.name); + } + + async isAvailable(): Promise { + try { + await this.listModels(); + return true; + } catch { + return false; + } + } + + private request(path: string, body?: unknown, method = 'POST'): Promise { + return new Promise((resolve, reject) => { + const url = new URL(path, this.baseUrl); + const payload = body !== undefined ? JSON.stringify(body) : undefined; + const opts: http.RequestOptions = { + hostname: url.hostname, + port: url.port || 11434, + path: url.pathname, + method, + timeout: 300000, // Ollama can be slow on first inference + headers: { + 'Content-Type': 'application/json', + ...(payload ? { 'Content-Length': Buffer.byteLength(payload) } : {}), + }, + }; + + const req = http.request(opts, (res) => { + const chunks: Buffer[] = []; + res.on('data', (chunk: Buffer) => chunks.push(chunk)); + res.on('end', () => { + const raw = Buffer.concat(chunks).toString('utf-8'); + try { + resolve(JSON.parse(raw)); + } catch { + reject(new Error(`Invalid JSON from Ollama: ${raw.slice(0, 200)}`)); + } + }); + }); + req.on('error', reject); + req.on('timeout', () => { + req.destroy(); + reject(new Error('Ollama request timed out')); + }); + if (payload) req.write(payload); + req.end(); + }); + } +} + +function parseOllamaResponse(raw: unknown): CompletionResult { + const data = raw as { + message?: { + content?: string; + tool_calls?: Array<{ + function: { name: string; arguments: Record }; + }>; + }; + prompt_eval_count?: number; + eval_count?: number; + }; + + const toolCalls: ToolCall[] = (data.message?.tool_calls ?? []).map((tc, i) => ({ + id: `ollama-${i}`, + name: tc.function.name, + arguments: tc.function.arguments, + })); + + const promptTokens = data.prompt_eval_count ?? 0; + const completionTokens = data.eval_count ?? 0; + + return { + content: data.message?.content ?? '', + toolCalls, + usage: { + promptTokens, + completionTokens, + totalTokens: promptTokens + completionTokens, + }, + finishReason: toolCalls.length > 0 ? 'tool_calls' : 'stop', + }; +} diff --git a/src/local-proxy/src/providers/openai.ts b/src/local-proxy/src/providers/openai.ts new file mode 100644 index 0000000..1adf0b6 --- /dev/null +++ b/src/local-proxy/src/providers/openai.ts @@ -0,0 +1,179 @@ +import https from 'node:https'; +import http from 'node:http'; +import type { LlmProvider, CompletionOptions, CompletionResult, ChatMessage, ToolCall } from './types.js'; + +export interface OpenAiConfig { + apiKey: string; + baseUrl?: string; + defaultModel?: string; +} + +interface OpenAiMessage { + role: string; + content: string | null; + tool_calls?: Array<{ + id: string; + type: 'function'; + function: { name: string; arguments: string }; + }>; + tool_call_id?: string; + name?: string; +} + +/** + * OpenAI-compatible provider. Works with OpenAI API, Azure OpenAI, + * and any service with an OpenAI-compatible chat completions endpoint. + */ +export class OpenAiProvider implements LlmProvider { + readonly name = 'openai'; + private apiKey: string; + private baseUrl: string; + private defaultModel: string; + + constructor(config: OpenAiConfig) { + this.apiKey = config.apiKey; + this.baseUrl = (config.baseUrl ?? 'https://api.openai.com').replace(/\/$/, ''); + this.defaultModel = config.defaultModel ?? 'gpt-4o'; + } + + async complete(options: CompletionOptions): Promise { + const model = options.model ?? this.defaultModel; + const body: Record = { + model, + messages: options.messages.map(toOpenAiMessage), + }; + if (options.temperature !== undefined) body.temperature = options.temperature; + if (options.maxTokens !== undefined) body.max_tokens = options.maxTokens; + + if (options.tools && options.tools.length > 0) { + body.tools = options.tools.map((t) => ({ + type: 'function', + function: { + name: t.name, + description: t.description, + parameters: t.inputSchema, + }, + })); + } + + const response = await this.request('/v1/chat/completions', body); + return parseResponse(response); + } + + async listModels(): Promise { + const response = await this.request('/v1/models', undefined, 'GET'); + const data = response as { data?: Array<{ id: string }> }; + return (data.data ?? []).map((m) => m.id); + } + + async isAvailable(): Promise { + try { + await this.listModels(); + return true; + } catch { + return false; + } + } + + private request(path: string, body: unknown, method = 'POST'): Promise { + return new Promise((resolve, reject) => { + const url = new URL(path, this.baseUrl); + const isHttps = url.protocol === 'https:'; + const transport = isHttps ? https : http; + + const payload = body !== undefined ? JSON.stringify(body) : undefined; + const opts = { + hostname: url.hostname, + port: url.port || (isHttps ? 443 : 80), + path: url.pathname, + method, + timeout: 120000, + headers: { + 'Authorization': `Bearer ${this.apiKey}`, + 'Content-Type': 'application/json', + ...(payload ? { 'Content-Length': Buffer.byteLength(payload) } : {}), + }, + }; + + const req = transport.request(opts, (res) => { + const chunks: Buffer[] = []; + res.on('data', (chunk: Buffer) => chunks.push(chunk)); + res.on('end', () => { + const raw = Buffer.concat(chunks).toString('utf-8'); + try { + resolve(JSON.parse(raw)); + } catch { + reject(new Error(`Invalid JSON response from ${path}: ${raw.slice(0, 200)}`)); + } + }); + }); + req.on('error', reject); + req.on('timeout', () => { + req.destroy(); + reject(new Error('Request timed out')); + }); + if (payload) req.write(payload); + req.end(); + }); + } +} + +function toOpenAiMessage(msg: ChatMessage): OpenAiMessage { + const result: OpenAiMessage = { + role: msg.role, + content: msg.content, + }; + if (msg.toolCallId !== undefined) result.tool_call_id = msg.toolCallId; + if (msg.name !== undefined) result.name = msg.name; + return result; +} + +function parseResponse(raw: unknown): CompletionResult { + const data = raw as { + choices?: Array<{ + message?: { + content?: string | null; + tool_calls?: Array<{ + id: string; + function: { name: string; arguments: string }; + }>; + }; + finish_reason?: string; + }>; + usage?: { + prompt_tokens?: number; + completion_tokens?: number; + total_tokens?: number; + }; + }; + + const choice = data.choices?.[0]; + const toolCalls: ToolCall[] = (choice?.message?.tool_calls ?? []).map((tc) => ({ + id: tc.id, + name: tc.function.name, + arguments: safeParse(tc.function.arguments), + })); + + const finishReason = choice?.finish_reason === 'tool_calls' ? 'tool_calls' as const + : choice?.finish_reason === 'length' ? 'length' as const + : 'stop' as const; + + return { + content: choice?.message?.content ?? '', + toolCalls, + usage: { + promptTokens: data.usage?.prompt_tokens ?? 0, + completionTokens: data.usage?.completion_tokens ?? 0, + totalTokens: data.usage?.total_tokens ?? 0, + }, + finishReason, + }; +} + +function safeParse(json: string): Record { + try { + return JSON.parse(json) as Record; + } catch { + return {}; + } +} diff --git a/src/local-proxy/src/providers/registry.ts b/src/local-proxy/src/providers/registry.ts new file mode 100644 index 0000000..f164307 --- /dev/null +++ b/src/local-proxy/src/providers/registry.ts @@ -0,0 +1,48 @@ +import type { LlmProvider } from './types.js'; + +/** + * Registry for LLM providers. Supports switching the active provider at runtime. + */ +export class ProviderRegistry { + private providers = new Map(); + private activeProvider: string | null = null; + + register(provider: LlmProvider): void { + this.providers.set(provider.name, provider); + if (this.activeProvider === null) { + this.activeProvider = provider.name; + } + } + + unregister(name: string): void { + this.providers.delete(name); + if (this.activeProvider === name) { + const first = this.providers.keys().next(); + this.activeProvider = first.done ? null : first.value; + } + } + + setActive(name: string): void { + if (!this.providers.has(name)) { + throw new Error(`Provider '${name}' is not registered`); + } + this.activeProvider = name; + } + + getActive(): LlmProvider | null { + if (this.activeProvider === null) return null; + return this.providers.get(this.activeProvider) ?? null; + } + + get(name: string): LlmProvider | undefined { + return this.providers.get(name); + } + + list(): string[] { + return [...this.providers.keys()]; + } + + getActiveName(): string | null { + return this.activeProvider; + } +} diff --git a/src/local-proxy/src/providers/types.ts b/src/local-proxy/src/providers/types.ts new file mode 100644 index 0000000..8aab79a --- /dev/null +++ b/src/local-proxy/src/providers/types.ts @@ -0,0 +1,56 @@ +/** + * LLM Provider abstraction for the local proxy. + * + * When the proxy intercepts tool calls, it can optionally route them + * through an LLM provider for autonomous tool use, summarization, + * or other processing before forwarding to the upstream MCP server. + */ + +export interface ChatMessage { + role: 'system' | 'user' | 'assistant' | 'tool'; + content: string; + toolCallId?: string; + name?: string; +} + +export interface ToolDefinition { + name: string; + description: string; + inputSchema: Record; +} + +export interface ToolCall { + id: string; + name: string; + arguments: Record; +} + +export interface CompletionResult { + content: string; + toolCalls: ToolCall[]; + usage: { + promptTokens: number; + completionTokens: number; + totalTokens: number; + }; + finishReason: 'stop' | 'tool_calls' | 'length' | 'error'; +} + +export interface CompletionOptions { + messages: ChatMessage[]; + tools?: ToolDefinition[]; + temperature?: number; + maxTokens?: number; + model?: string; +} + +export interface LlmProvider { + /** Provider identifier (e.g., 'openai', 'anthropic', 'ollama') */ + readonly name: string; + /** Create a chat completion */ + complete(options: CompletionOptions): Promise; + /** List available models */ + listModels(): Promise; + /** Check if the provider is configured and reachable */ + isAvailable(): Promise; +} diff --git a/src/local-proxy/tests/providers.test.ts b/src/local-proxy/tests/providers.test.ts new file mode 100644 index 0000000..66bf520 --- /dev/null +++ b/src/local-proxy/tests/providers.test.ts @@ -0,0 +1,118 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { ProviderRegistry } from '../src/providers/registry.js'; +import type { LlmProvider, CompletionOptions, CompletionResult } from '../src/providers/types.js'; + +function mockProvider(name: string): LlmProvider { + return { + name, + complete: vi.fn(async (): Promise => ({ + content: `Response from ${name}`, + toolCalls: [], + usage: { promptTokens: 10, completionTokens: 20, totalTokens: 30 }, + finishReason: 'stop', + })), + listModels: vi.fn(async () => [`${name}-model-1`, `${name}-model-2`]), + isAvailable: vi.fn(async () => true), + }; +} + +describe('ProviderRegistry', () => { + let registry: ProviderRegistry; + + beforeEach(() => { + registry = new ProviderRegistry(); + }); + + it('starts with no providers', () => { + expect(registry.list()).toEqual([]); + expect(registry.getActive()).toBeNull(); + expect(registry.getActiveName()).toBeNull(); + }); + + it('registers a provider and sets it as active', () => { + const openai = mockProvider('openai'); + registry.register(openai); + + expect(registry.list()).toEqual(['openai']); + expect(registry.getActive()).toBe(openai); + expect(registry.getActiveName()).toBe('openai'); + }); + + it('first registered provider becomes active', () => { + registry.register(mockProvider('openai')); + registry.register(mockProvider('anthropic')); + + expect(registry.getActiveName()).toBe('openai'); + expect(registry.list()).toEqual(['openai', 'anthropic']); + }); + + it('switches active provider', () => { + registry.register(mockProvider('openai')); + registry.register(mockProvider('anthropic')); + + registry.setActive('anthropic'); + expect(registry.getActiveName()).toBe('anthropic'); + }); + + it('throws when setting unknown provider as active', () => { + expect(() => registry.setActive('unknown')).toThrow("Provider 'unknown' is not registered"); + }); + + it('gets provider by name', () => { + const openai = mockProvider('openai'); + registry.register(openai); + + expect(registry.get('openai')).toBe(openai); + expect(registry.get('unknown')).toBeUndefined(); + }); + + it('unregisters a provider', () => { + registry.register(mockProvider('openai')); + registry.register(mockProvider('anthropic')); + + registry.unregister('openai'); + expect(registry.list()).toEqual(['anthropic']); + // Active should switch to remaining provider + expect(registry.getActiveName()).toBe('anthropic'); + }); + + it('unregistering active provider switches to next available', () => { + registry.register(mockProvider('openai')); + registry.register(mockProvider('anthropic')); + registry.setActive('openai'); + + registry.unregister('openai'); + expect(registry.getActiveName()).toBe('anthropic'); + }); + + it('unregistering last provider clears active', () => { + registry.register(mockProvider('openai')); + registry.unregister('openai'); + + expect(registry.getActive()).toBeNull(); + expect(registry.getActiveName()).toBeNull(); + }); + + it('active provider can complete', async () => { + const provider = mockProvider('openai'); + registry.register(provider); + + const active = registry.getActive()!; + const result = await active.complete({ + messages: [{ role: 'user', content: 'Hello' }], + }); + + expect(result.content).toBe('Response from openai'); + expect(result.finishReason).toBe('stop'); + expect(provider.complete).toHaveBeenCalled(); + }); + + it('active provider can list models', async () => { + registry.register(mockProvider('anthropic')); + + const active = registry.getActive()!; + const models = await active.listModels(); + + expect(models).toEqual(['anthropic-model-1', 'anthropic-model-2']); + }); +});