pi-web-search/src/runtime.test.ts

import test from "node:test";
import assert from "node:assert/strict";
import { createWebSearchRuntime } from "./runtime.ts";

function createProvider(name: string, type: string, handlers: Partial<any>) {
  return {
    name,
    type,
    async search(request: any) {
      return handlers.search?.(request);
    },
    async fetch(request: any) {
      return handlers.fetch?.(request);
    },
  };
}

test("search follows configured fallback chains and records every attempt", async () => {
  const runtime = createWebSearchRuntime({
    loadConfig: async () => ({
      path: "test.json",
      defaultProviderName: "firecrawl-main",
      defaultProvider: {
        name: "firecrawl-main",
        type: "firecrawl",
        apiKey: "fc",
        fallbackProviders: ["tavily-backup"],
      },
      providers: [
        {
          name: "firecrawl-main",
          type: "firecrawl",
          apiKey: "fc",
          fallbackProviders: ["tavily-backup"],
        },
        {
          name: "tavily-backup",
          type: "tavily",
          apiKey: "tvly",
          fallbackProviders: ["exa-fallback"],
        },
        { name: "exa-fallback", type: "exa", apiKey: "exa" },
      ],
      providersByName: new Map([
        [
          "firecrawl-main",
          { name: "firecrawl-main", type: "firecrawl", apiKey: "fc", fallbackProviders: ["tavily-backup"] },
        ],
        [
          "tavily-backup",
          { name: "tavily-backup", type: "tavily", apiKey: "tvly", fallbackProviders: ["exa-fallback"] },
        ],
        ["exa-fallback", { name: "exa-fallback", type: "exa", apiKey: "exa" }],
      ]),
    }),
    createProvider(providerConfig) {
      if (providerConfig.name === "exa-fallback") {
        return createProvider(providerConfig.name, providerConfig.type, {
          search: async () => ({
            providerName: providerConfig.name,
            results: [{ title: "Exa hit", url: "https://exa.ai" }],
          }),
        });
      }

      return createProvider(providerConfig.name, providerConfig.type, {
        search: async () => {
          throw new Error(`boom:${providerConfig.name}`);
        },
      });
    },
  });

  const result = await runtime.search({ query: "pi docs" });

  assert.equal(result.execution.actualProviderName, "exa-fallback");
  assert.equal(result.execution.failoverFromProviderName, "firecrawl-main");
  assert.deepEqual(result.execution.attempts, [
    {
      providerName: "firecrawl-main",
      status: "failed",
      reason: "boom:firecrawl-main",
    },
    {
      providerName: "tavily-backup",
      status: "failed",
      reason: "boom:tavily-backup",
    },
    {
      providerName: "exa-fallback",
      status: "succeeded",
    },
  ]);
});

test("search rejects a mismatched provider-specific options block before provider execution", async () => {
  let callCount = 0;

  const runtime = createWebSearchRuntime({
    loadConfig: async () => ({
      path: "test.json",
      defaultProviderName: "firecrawl-main",
      defaultProvider: { name: "firecrawl-main", type: "firecrawl", apiKey: "fc" },
      providers: [{ name: "firecrawl-main", type: "firecrawl", apiKey: "fc" }],
      providersByName: new Map([["firecrawl-main", { name: "firecrawl-main", type: "firecrawl", apiKey: "fc" }]]),
    }),
    createProvider(providerConfig) {
      return createProvider(providerConfig.name, providerConfig.type, {
        search: async () => {
          callCount += 1;
          return {
            providerName: providerConfig.name,
            results: [],
          };
        },
      });
    },
  });

  await assert.rejects(
    () => runtime.search({ query: "pi docs", tavily: { topic: "news" } }),
    /does not accept the "tavily" options block/,
  );
  assert.equal(callCount, 0);
});

test("search rejects Firecrawl requests with multiple includeDomains before provider execution", async () => {
  const calls: string[] = [];

  const runtime = createWebSearchRuntime({
    loadConfig: async () => ({
      path: "test.json",
      defaultProviderName: "firecrawl-main",
      defaultProvider: {
        name: "firecrawl-main",
        type: "firecrawl",
        apiKey: "fc",
        fallbackProviders: ["exa-fallback"],
      },
      providers: [
        {
          name: "firecrawl-main",
          type: "firecrawl",
          apiKey: "fc",
          fallbackProviders: ["exa-fallback"],
        },
        { name: "exa-fallback", type: "exa", apiKey: "exa" },
      ],
      providersByName: new Map([
        [
          "firecrawl-main",
          { name: "firecrawl-main", type: "firecrawl", apiKey: "fc", fallbackProviders: ["exa-fallback"] },
        ],
        ["exa-fallback", { name: "exa-fallback", type: "exa", apiKey: "exa" }],
      ]),
    }),
    createProvider(providerConfig) {
      return createProvider(providerConfig.name, providerConfig.type, {
        search: async () => {
          calls.push(providerConfig.name);
          throw new Error(`boom:${providerConfig.name}`);
        },
      });
    },
  });

  await assert.rejects(
    () =>
      runtime.search({
        query: "pi docs",
        provider: "firecrawl-main",
        includeDomains: ["pi.dev", "exa.ai"],
      }),
    /Provider "firecrawl-main" accepts at most one includeDomains entry/,
  );
  assert.deepEqual(calls, []);
});

test("search rejects Firecrawl category conflicts before provider execution", async () => {
  const calls: string[] = [];

  const runtime = createWebSearchRuntime({
    loadConfig: async () => ({
      path: "test.json",
      defaultProviderName: "firecrawl-main",
      defaultProvider: {
        name: "firecrawl-main",
        type: "firecrawl",
        apiKey: "fc",
        fallbackProviders: ["exa-fallback"],
      },
      providers: [
        {
          name: "firecrawl-main",
          type: "firecrawl",
          apiKey: "fc",
          fallbackProviders: ["exa-fallback"],
        },
        { name: "exa-fallback", type: "exa", apiKey: "exa" },
      ],
      providersByName: new Map([
        [
          "firecrawl-main",
          { name: "firecrawl-main", type: "firecrawl", apiKey: "fc", fallbackProviders: ["exa-fallback"] },
        ],
        ["exa-fallback", { name: "exa-fallback", type: "exa", apiKey: "exa" }],
      ]),
    }),
    createProvider(providerConfig) {
      return createProvider(providerConfig.name, providerConfig.type, {
        search: async () => {
          calls.push(providerConfig.name);
          throw new Error(`boom:${providerConfig.name}`);
        },
      });
    },
  });

  await assert.rejects(
    () =>
      runtime.search({
        query: "pi docs",
        provider: "firecrawl-main",
        category: "research",
        firecrawl: { categories: ["github"] },
      }),
    /Provider "firecrawl-main" does not accept both top-level category and firecrawl.categories/,
  );
  assert.deepEqual(calls, []);
});

test("fetch rejects Firecrawl highlights before provider execution", async () => {
  const calls: string[] = [];

  const runtime = createWebSearchRuntime({
    loadConfig: async () => ({
      path: "test.json",
      defaultProviderName: "firecrawl-main",
      defaultProvider: {
        name: "firecrawl-main",
        type: "firecrawl",
        apiKey: "fc",
        fallbackProviders: ["exa-fallback"],
      },
      providers: [
        {
          name: "firecrawl-main",
          type: "firecrawl",
          apiKey: "fc",
          fallbackProviders: ["exa-fallback"],
        },
        { name: "exa-fallback", type: "exa", apiKey: "exa" },
      ],
      providersByName: new Map([
        [
          "firecrawl-main",
          { name: "firecrawl-main", type: "firecrawl", apiKey: "fc", fallbackProviders: ["exa-fallback"] },
        ],
        ["exa-fallback", { name: "exa-fallback", type: "exa", apiKey: "exa" }],
      ]),
    }),
    createProvider(providerConfig) {
      return createProvider(providerConfig.name, providerConfig.type, {
        fetch: async () => {
          calls.push(providerConfig.name);
          return {
            providerName: providerConfig.name,
            results: [],
          };
        },
      });
    },
  });

  await assert.rejects(
    () => runtime.fetch({ urls: ["https://pi.dev"], provider: "firecrawl-main", highlights: true }),
    /does not support generic fetch option "highlights"/,
  );
  assert.deepEqual(calls, []);
});

test("fetch rejects Firecrawl format mismatches before provider execution", async () => {
  const calls: string[] = [];

  const runtime = createWebSearchRuntime({
    loadConfig: async () => ({
      path: "test.json",
      defaultProviderName: "firecrawl-main",
      defaultProvider: {
        name: "firecrawl-main",
        type: "firecrawl",
        apiKey: "fc",
        fallbackProviders: ["exa-fallback"],
      },
      providers: [
        {
          name: "firecrawl-main",
          type: "firecrawl",
          apiKey: "fc",
          fallbackProviders: ["exa-fallback"],
        },
        { name: "exa-fallback", type: "exa", apiKey: "exa" },
      ],
      providersByName: new Map([
        [
          "firecrawl-main",
          { name: "firecrawl-main", type: "firecrawl", apiKey: "fc", fallbackProviders: ["exa-fallback"] },
        ],
        ["exa-fallback", { name: "exa-fallback", type: "exa", apiKey: "exa" }],
      ]),
    }),
    createProvider(providerConfig) {
      return createProvider(providerConfig.name, providerConfig.type, {
        fetch: async () => {
          calls.push(providerConfig.name);
          return {
            providerName: providerConfig.name,
            results: [],
          };
        },
      });
    },
  });

  await assert.rejects(
    () =>
      runtime.fetch({
        urls: ["https://pi.dev"],
        provider: "firecrawl-main",
        summary: true,
        firecrawl: { formats: ["markdown"] },
      }),
    /Provider "firecrawl-main" requires firecrawl.formats to include "summary" when summary is true/,
  );
  assert.deepEqual(calls, []);
});

test("search throws a clear error for unknown provider types", async () => {
  const runtime = createWebSearchRuntime({
    loadConfig: async () => ({
      path: "test.json",
      defaultProviderName: "mystery-main",
      defaultProvider: { name: "mystery-main", type: "mystery", apiKey: "??" } as any,
      providers: [{ name: "mystery-main", type: "mystery", apiKey: "??" } as any],
      providersByName: new Map([["mystery-main", { name: "mystery-main", type: "mystery", apiKey: "??" } as any]]),
    }),
  });

  await assert.rejects(() => runtime.search({ query: "pi docs" }), /Unknown provider type: mystery/);
});

test("search starts with the explicitly requested provider and still follows its fallback chain", async () => {
  const calls: string[] = [];

  const runtime = createWebSearchRuntime({
    loadConfig: async () => ({
      path: "test.json",
      defaultProviderName: "tavily-main",
      defaultProvider: { name: "tavily-main", type: "tavily", apiKey: "tvly" },
      providers: [
        { name: "tavily-main", type: "tavily", apiKey: "tvly" },
        {
          name: "firecrawl-main",
          type: "firecrawl",
          apiKey: "fc",
          fallbackProviders: ["exa-fallback"],
        },
        { name: "exa-fallback", type: "exa", apiKey: "exa" },
      ],
      providersByName: new Map([
        ["tavily-main", { name: "tavily-main", type: "tavily", apiKey: "tvly" }],
        [
          "firecrawl-main",
          { name: "firecrawl-main", type: "firecrawl", apiKey: "fc", fallbackProviders: ["exa-fallback"] },
        ],
        ["exa-fallback", { name: "exa-fallback", type: "exa", apiKey: "exa" }],
      ]),
    }),
    createProvider(providerConfig) {
      return createProvider(providerConfig.name, providerConfig.type, {
        search: async () => {
          calls.push(providerConfig.name);
          if (providerConfig.name === "exa-fallback") {
            return {
              providerName: providerConfig.name,
              results: [{ title: "Exa hit", url: "https://exa.ai" }],
            };
          }
          throw new Error(`boom:${providerConfig.name}`);
        },
      });
    },
  });

  const result = await runtime.search({ query: "pi docs", provider: "firecrawl-main" });

  assert.deepEqual(calls, ["firecrawl-main", "exa-fallback"]);
  assert.equal(result.execution.requestedProviderName, "firecrawl-main");
  assert.equal(result.execution.actualProviderName, "exa-fallback");
});