feat: add generic AI provider support (#64)

.env.example

@@ -61,6 +61,16 @@ GOOGLE_GENERATIVE_AI_API_KEY=AIza...
 # AXIOM_TOKEN=
 # AXIOM_DATASET=
 
+# =============================================================================
+# Optional: Custom Providers
+# =============================================================================
+
+# Set these when using a custom provider registered via configure().
+# Reference them in your createProvider() factory function.
+# Example for a corporate OpenAI-compatible LLM proxy:
+# LLM_PROXY_API_KEY=your-proxy-key
+# LLM_PROXY_API_BASE_URL=https://llm-proxy.internal.company.com
+
 # =============================================================================
 # Optional: Logging
 # =============================================================================

README.md

@@ -293,6 +293,73 @@ configure({
 });
 ```
 
+### Custom Providers
+
+If you use a self-hosted LLM proxy, corporate gateway, or any OpenAI-compatible endpoint, you can register custom providers via `configure()`. This lets you use any [Vercel AI SDK](https://ai-sdk.dev/providers)-compatible provider package.
+
+Install the SDK package you need (e.g. `@ai-sdk/openai-compatible` for generic OpenAI-compatible APIs):
+
+```bash
+npm install @ai-sdk/openai-compatible
+```
+
+Then register the provider and use it in model IDs:
+
+```typescript
+import { configure } from "passmark";
+import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
+
+configure({
+  ai: {
+    providers: {
+      "llm-proxy": {
+        createProvider: () => createOpenAICompatible({
+          name: "llm-proxy",
+          apiKey: process.env.LLM_PROXY_API_KEY!,
+          baseURL: `${process.env.LLM_PROXY_API_BASE_URL}/api/v1/proxy/openai/auto`,
+        }),
+      },
+    },
+    models: {
+      stepExecution: "llm-proxy/google/gemini-3.5-flash",
+      assertionPrimary: "llm-proxy/anthropic/claude-haiku-4.5",
+      assertionSecondary: "llm-proxy/google/gemini-3-flash",
+      utility: "llm-proxy/google/gemini-2.5-flash",
+    },
+  },
+});
+```
+
+You can also route **all** models through a custom provider by setting `gateway` to the provider name:
+
+```typescript
+configure({
+  ai: {
+    gateway: "llm-proxy",  // all models route through this provider
+    providers: {
+      "llm-proxy": {
+        createProvider: () => createOpenAICompatible({
+          name: "llm-proxy",
+          apiKey: process.env.LLM_PROXY_API_KEY!,
+          baseURL: process.env.LLM_PROXY_API_BASE_URL!,
+        }),
+        // Optional: remap passmark's default model IDs to your proxy's model names
+        models: {
+          "google/gemini-3-flash": "gemini-3-flash",
+          "anthropic/claude-haiku-4.5": "claude-haiku-4-5",
+        },
+      },
+    },
+  },
+});
+```
+
+Notes:
+- Provider instances are created lazily and cached — `createProvider()` is called only once per provider name.
+- Custom providers work with per-step and per-call `ai` overrides, the same as built-in providers.
+- Any `@ai-sdk/*` package can be used (e.g. `@ai-sdk/openai`, `@ai-sdk/anthropic`, `@ai-sdk/google`), not just `@ai-sdk/openai-compatible`.
+- Video assertions still require `GOOGLE_GENERATIVE_AI_API_KEY` regardless of custom providers (Gemini Files API is used directly).
+
 ## Environment Variables
 
 | Variable | Required | Default | Description |

src/__tests__/config.test.ts

@@ -1,5 +1,13 @@
 import { describe, it, expect, beforeEach } from "vitest";
-import { configure, getConfig, getModelId, resetConfig, DEFAULT_MODELS } from "../config";
+import {
+  configure,
+  getConfig,
+  getModelId,
+  resetConfig,
+  DEFAULT_MODELS,
+  resolveAI,
+  type CustomProviderConfig,
+} from "../config";
 
 describe("config", () => {
   beforeEach(() => {
@@ -96,4 +104,71 @@ describe("config", () => {
     resetConfig();
     expect(getConfig()).toEqual({});
   });
+
+  describe("custom providers", () => {
+    const mockProvider: CustomProviderConfig = {
+      createProvider: () => ({
+        languageModel: (id: string) => id as never,
+      }) as never,
+    };
+
+    it("configure stores custom providers", () => {
+      configure({
+        ai: {
+          providers: { "my-proxy": mockProvider },
+        },
+      });
+      expect(getConfig().ai?.providers).toBeDefined();
+      expect(getConfig().ai?.providers?.["my-proxy"]).toBe(mockProvider);
+    });
+
+    it("configure stores custom providers with model aliases", () => {
+      const providerWithAliases: CustomProviderConfig = {
+        ...mockProvider,
+        models: { "gemini-flash": "google/gemini-3.5-flash" },
+      };
+      configure({
+        ai: {
+          providers: { "corp-proxy": providerWithAliases },
+        },
+      });
+      expect(getConfig().ai?.providers?.["corp-proxy"]?.models).toEqual({
+        "gemini-flash": "google/gemini-3.5-flash",
+      });
+    });
+
+    it("resolveAI merges providers from global config and overrides", () => {
+      const providerA: CustomProviderConfig = {
+        createProvider: () => ({ languageModel: () => "a" }) as never,
+      };
+      const providerB: CustomProviderConfig = {
+        createProvider: () => ({ languageModel: () => "b" }) as never,
+      };
+      configure({
+        ai: { providers: { alpha: providerA } },
+      });
+      const resolved = resolveAI({ providers: { beta: providerB } });
+      expect(resolved.providers?.["alpha"]).toBe(providerA);
+      expect(resolved.providers?.["beta"]).toBe(providerB);
+    });
+
+    it("resolveAI later overrides win on same provider key", () => {
+      const providerV1: CustomProviderConfig = {
+        createProvider: () => ({ languageModel: () => "v1" }) as never,
+      };
+      const providerV2: CustomProviderConfig = {
+        createProvider: () => ({ languageModel: () => "v2" }) as never,
+      };
+      configure({
+        ai: { providers: { "my-proxy": providerV1 } },
+      });
+      const resolved = resolveAI({ providers: { "my-proxy": providerV2 } });
+      expect(resolved.providers?.["my-proxy"]).toBe(providerV2);
+    });
+
+    it("resolveAI returns undefined providers when none configured", () => {
+      const resolved = resolveAI();
+      expect(resolved.providers).toBeUndefined();
+    });
+  });
 });

src/__tests__/models.test.ts

@@ -0,0 +1,150 @@
+import { describe, it, expect, beforeEach, vi } from "vitest";
+import { resetConfig, configure, type CustomProviderConfig } from "../config";
+
+// Mock dependencies that resolveModel uses internally
+vi.mock("@ai-sdk/anthropic", () => ({
+  createAnthropic: vi.fn(),
+}));
+vi.mock("@ai-sdk/google", () => ({
+  createGoogleGenerativeAI: vi.fn(),
+}));
+vi.mock("@ai-sdk/openai", () => ({
+  createOpenAI: vi.fn(),
+}));
+vi.mock("@openrouter/ai-sdk-provider", () => ({
+  createOpenRouter: vi.fn(),
+}));
+vi.mock("ai", async (importOriginal) => {
+  const actual = await importOriginal<typeof import("ai")>();
+  return {
+    ...actual,
+    gateway: vi.fn(),
+  };
+});
+vi.mock("axiom/ai", () => ({
+  wrapAISDKModel: vi.fn((model) => model),
+}));
+vi.mock("../instrumentation", () => ({
+  isAxiomEnabled: vi.fn(() => false),
+}));
+
+describe("resolveModel — custom providers", () => {
+  // We need to import resolveModel after mocks are set up
+  let resolveModel: typeof import("../models").resolveModel;
+
+  beforeEach(async () => {
+    resetConfig();
+    // Re-import to get fresh module state
+    const models = await import("../models");
+    resolveModel = models.resolveModel;
+  });
+
+  function createMockProvider(returnValue?: string): CustomProviderConfig {
+    const languageModel = vi.fn((id: string) => returnValue ?? `resolved:${id}`);
+    const createProvider = vi.fn(() => ({
+      languageModel,
+      textEmbeddingModel: vi.fn(),
+    }));
+    return { createProvider } as unknown as CustomProviderConfig;
+  }
+
+  function createMockProviderWithAliases(
+    aliases: Record<string, string>,
+  ): CustomProviderConfig {
+    const languageModel = vi.fn((id: string) => `resolved:${id}`);
+    const createProvider = vi.fn(() => ({
+      languageModel,
+      textEmbeddingModel: vi.fn(),
+    }));
+    return { createProvider, models: aliases } as unknown as CustomProviderConfig;
+  }
+
+  it("resolves model via custom provider prefix", () => {
+    const cp = createMockProvider();
+    const result = resolveModel("my-proxy/gpt-4", "none", { "my-proxy": cp });
+    expect(result).toBe("resolved:gpt-4");
+    expect(cp.createProvider).toHaveBeenCalledOnce();
+  });
+
+  it("resolves model via custom provider in gateway mode", () => {
+    const cp = createMockProvider();
+    const result = resolveModel(
+      "google/gemini-3-flash",
+      "llm-proxy",
+      { "llm-proxy": cp },
+    );
+    expect(result).toBe("resolved:google/gemini-3-flash");
+    expect(cp.createProvider).toHaveBeenCalledOnce();
+  });
+
+  it("applies model aliases in provider-prefix mode", () => {
+    const cp = createMockProviderWithAliases({
+      "gemini-flash": "google/gemini-3.5-flash-internal",
+    });
+    const result = resolveModel("my-proxy/gemini-flash", "none", {
+      "my-proxy": cp,
+    });
+    expect(result).toBe("resolved:google/gemini-3.5-flash-internal");
+  });
+
+  it("applies model aliases in gateway mode", () => {
+    const cp = createMockProviderWithAliases({
+      "google/gemini-3-flash": "gemini-3-flash-corp",
+    });
+    const result = resolveModel("google/gemini-3-flash", "my-proxy", {
+      "my-proxy": cp,
+    });
+    expect(result).toBe("resolved:gemini-3-flash-corp");
+  });
+
+  it("passes through model name when no alias matches", () => {
+    const cp = createMockProviderWithAliases({
+      "some-model": "other-model",
+    });
+    const result = resolveModel("my-proxy/gpt-4", "none", { "my-proxy": cp });
+    // No alias for "gpt-4", so it passes through as-is
+    expect(result).toBe("resolved:gpt-4");
+  });
+
+  it("caches provider instances (createProvider called once)", () => {
+    const cp = createMockProvider();
+    resolveModel("my-proxy/model-a", "none", { "my-proxy": cp });
+    resolveModel("my-proxy/model-b", "none", { "my-proxy": cp });
+    expect(cp.createProvider).toHaveBeenCalledOnce();
+  });
+
+  it("throws for unknown provider when no custom provider matches", () => {
+    expect(() =>
+      resolveModel("unknown-provider/model", "none"),
+    ).toThrow("Unknown AI provider: unknown-provider");
+  });
+
+  it("falls back to built-in providers when custom provider doesn't match prefix", () => {
+    const cp = createMockProvider();
+    // "google/..." should NOT match custom provider "my-proxy"
+    expect(() =>
+      resolveModel("google/gemini-3-flash", "none", { "my-proxy": cp }),
+    ).toThrow(); // Throws because GOOGLE_GENERATIVE_AI_API_KEY is not set in test
+  });
+
+  it("reads custom providers from global config when not passed explicitly", () => {
+    const cp = createMockProvider();
+    configure({
+      ai: {
+        providers: { "global-proxy": cp },
+      },
+    });
+    const result = resolveModel("global-proxy/some-model");
+    expect(result).toBe("resolved:some-model");
+    expect(cp.createProvider).toHaveBeenCalledOnce();
+  });
+
+  it("handles nested model names with slashes in provider-prefix mode", () => {
+    const cp = createMockProvider();
+    const result = resolveModel("my-proxy/org/model-name", "none", {
+      "my-proxy": cp,
+    });
+    // Model name should be "org/model-name"
+    expect(result).toBe("resolved:org/model-name");
+  });
+});