mcpctl/src/mcplocal/tests/anthropic-model-resolution.test.ts

import { describe, it, expect, vi, afterEach } from 'vitest';
import { AnthropicProvider } from '../src/providers/anthropic.js';

/** The real payload shape, abbreviated — newest first, as the API returns it. */
const MODELS = [
  { id: 'claude-opus-5', created_at: '2026-07-24T00:00:00Z' },
  { id: 'claude-sonnet-5', created_at: '2026-06-29T00:00:00Z' },
  { id: 'claude-fable-5', created_at: '2026-06-07T00:00:00Z' },
  { id: 'claude-opus-4-8', created_at: '2026-05-28T00:00:00Z' },
  { id: 'claude-opus-4-5-20251101', created_at: '2025-11-24T00:00:00Z' },
  { id: 'claude-haiku-4-5-20251001', created_at: '2025-10-15T00:00:00Z' },
];

function providerWith(models: typeof MODELS | Error): AnthropicProvider {
  const p = new AnthropicProvider({ apiKey: 'sk-ant-api-test' });
  // Stub the private transport rather than the network.
  (p as unknown as { get: (path: string) => Promise<unknown> }).get = async () => {
    if (models instanceof Error) throw models;
    return { data: models, has_more: false };
  };
  return p;
}

afterEach(() => {
  // The cache is static — clear it so cases don't leak into each other.
  (AnthropicProvider as unknown as { modelCache: Map<string, unknown> }).modelCache.clear();
  vi.restoreAllMocks();
});

describe('Anthropic model resolution', () => {
  it('resolves a family selector to the newest member', async () => {
    await expect(providerWith(MODELS).resolveModel('claude-opus-latest')).resolves.toBe('claude-opus-5');
  });

  it('accepts the bare family name too', async () => {
    await expect(providerWith(MODELS).resolveModel('opus')).resolves.toBe('claude-opus-5');
    await expect(providerWith(MODELS).resolveModel('haiku')).resolves.toBe('claude-haiku-4-5-20251001');
  });

  it('picks by created_at, not by parsing the version', async () => {
    // The trap: claude-opus-4-5 sorts ABOVE claude-opus-5 as a string, and
    // "4-5" parses as a bigger minor than "5". Only the date is reliable.
    const out = await providerWith(MODELS).resolveModel('claude-opus-latest');
    expect(out).toBe('claude-opus-5');
    expect(out).not.toBe('claude-opus-4-5-20251101');
  });

  it('leaves an exact model id alone, so pinning still works', async () => {
    const p = providerWith(MODELS);
    await expect(p.resolveModel('claude-opus-4-8')).resolves.toBe('claude-opus-4-8');
    await expect(p.resolveModel('claude-haiku-4-5-20251001')).resolves.toBe('claude-haiku-4-5-20251001');
  });

  it('does not treat a dated id as a family selector', async () => {
    await expect(providerWith(MODELS).resolveModel('claude-opus-4-20250514'))
      .resolves.toBe('claude-opus-4-20250514');
  });

  it('falls back deterministically when the models endpoint is unreachable', async () => {
    const err = vi.spyOn(process.stderr, 'write').mockImplementation(() => true);
    await expect(providerWith(new Error('network down')).resolveModel('claude-opus-latest'))
      .resolves.toBe('claude-opus-5');
    // Loud, not silent.
    expect(err).toHaveBeenCalledWith(expect.stringContaining('falling back'));
  });

  it('caches, so resolution is not a per-call network hop', async () => {
    const p = providerWith(MODELS);
    let calls = 0;
    (p as unknown as { get: () => Promise<unknown> }).get = async () => {
      calls++;
      return { data: MODELS, has_more: false };
    };
    await p.resolveModel('claude-opus-latest');
    await p.resolveModel('claude-opus-latest');
    await p.resolveModel('claude-opus-latest');
    expect(calls).toBe(1);
  });

  it('lists real models instead of a hardcoded table', async () => {
    await expect(providerWith(MODELS).listModels()).resolves.toContain('claude-opus-5');
  });
});

describe('sampling-parameter rejection', () => {
  afterEach(() => {
    (AnthropicProvider as unknown as { rejectsSampling: Set<string> }).rejectsSampling.clear();
  });

  /** Rejects `temperature` exactly as the current Anthropic models do. */
  function samplingStrictProvider(): { provider: AnthropicProvider; bodies: Array<Record<string, unknown>> } {
    const provider = new AnthropicProvider({ apiKey: 'sk-ant-api-test' });
    const bodies: Array<Record<string, unknown>> = [];
    (provider as unknown as { request: (b: unknown) => Promise<unknown> }).request = async (b) => {
      const body = b as Record<string, unknown>;
      bodies.push({ ...body });
      if (body.temperature !== undefined) {
        throw new Error(
          'Anthropic HTTP 400: {"type":"error","error":{"type":"invalid_request_error",'
          + '"message":"`temperature` is deprecated for this model."}}',
        );
      }
      return { content: [{ type: 'text', text: 'ok' }], stop_reason: 'end_turn' };
    };
    return { provider, bodies };
  }

  it('retries without temperature when the model rejects it', async () => {
    vi.spyOn(process.stderr, 'write').mockImplementation(() => true);
    const { provider, bodies } = samplingStrictProvider();

    const result = await provider.complete({
      model: 'claude-opus-5',
      messages: [{ role: 'user', content: 'hi' }],
      temperature: 0,
    });

    expect(result.content).toBe('ok');
    expect(bodies).toHaveLength(2);
    expect(bodies[0]!.temperature).toBe(0);
    expect(bodies[1]!.temperature).toBeUndefined();
  });

  it('remembers, so the wasted call happens once and not forever', async () => {
    vi.spyOn(process.stderr, 'write').mockImplementation(() => true);
    const { provider, bodies } = samplingStrictProvider();
    const req = { model: 'claude-opus-5', messages: [{ role: 'user' as const, content: 'hi' }], temperature: 0 };

    await provider.complete(req);
    await provider.complete(req);
    await provider.complete(req);

    // 2 for the first call (reject + retry), then 1 each — not 2 each.
    expect(bodies).toHaveLength(4);
  });

  it('does not swallow unrelated 400s', async () => {
    const provider = new AnthropicProvider({ apiKey: 'sk-ant-api-test' });
    (provider as unknown as { request: () => Promise<unknown> }).request = async () => {
      throw new Error('Anthropic HTTP 400: {"error":{"message":"max_tokens is required"}}');
    };
    await expect(provider.complete({
      model: 'claude-opus-5',
      messages: [{ role: 'user', content: 'hi' }],
      temperature: 0,
    })).rejects.toThrow(/max_tokens/);
  });
});