From 63f7cba1f53515135abe1dfbd9a0d3ccbd4baf4a Mon Sep 17 00:00:00 2001 From: Vaibhav Lajurkar Date: Tue, 20 Oct 2026 00:40:18 +0530 Subject: [PATCH 1/2] Publish Resend's hosted MCP endpoint; stop endpoint-less curated MCP entries hiding hosted ones curated/resend.json described only the local stdio server (npx -y resend-mcp) with no endpoint. Curated records claim their (domain, kind) slot before any other source, so the hosted https://mcp.resend.com/mcp already present in domains/resend.com was discarded and the catalog's Resend MCP surface had no connect URL. - curated/resend.json: add the hosted MCP interface (Streamable HTTP, OAuth in supporting clients, API-key bearer header otherwise; auth "mixed" like Stripe and GitHub) and keep the local stdio server as a second interface. - normalize: when a curated file lists several MCP interfaces, a hosted, publishable endpoint beats an endpoint-less one listed before it. - normalize: an endpoint-less curated MCP record no longer holds its slot. A feed or discovered hosted endpoint that passes the usual isPublic and endpoint-verdict rules replaces it, and the build prints a curated-mcp-no-endpoint warning so the curated file gets fixed. - The curated/feed/discovered merge moves out of main() into assembleRecords() so tests exercise the real wiring with local fixtures. Co-Authored-By: Claude Opus 5.5 --- curated/resend.json | 25 ++++- scripts/normalize.test.ts | 224 +++++++++++++++++++++++++++++++++++++- scripts/normalize.ts | 209 +++++++++++++++++++++++++---------- 3 files changed, 392 insertions(+), 66 deletions(-) diff --git a/curated/resend.json b/curated/resend.json index 36eae47c..46d7aa6c 100644 --- a/curated/resend.json +++ b/curated/resend.json @@ -1,7 +1,7 @@ { "slug": "resend", "name": "Resend", - "tagline": "Send email from any agent with one API key.", + "tagline": "Send email from any agent over hosted MCP, OAuth or one API key.", "description": "Resend is an email API built for developers. The official MCP server exposes sending, contacts, audiences, broadcasts, and domain management; the REST API is small enough for an agent to learn in one read.", "domain": "resend.com", "icon": "https://www.google.com/s2/favicons?domain=resend.com&sz=64", @@ -12,13 +12,18 @@ ], "auth": { "methods": [ + { + "type": "oauth2", + "label": "OAuth (hosted MCP)", + "note": "Browser login to Resend, approved per client. Recommended for MCP clients that support OAuth." + }, { "type": "api_key", "label": "API key", - "note": "One key, sent as a bearer header. Scoped keys can restrict to sending-only." + "note": "One key, sent as a bearer header. Works for headless MCP clients, the local MCP server and the REST API. Scoped keys can restrict to sending-only." } ], - "guide": "Create an API key at [resend.com/api-keys](https://resend.com/api-keys) (`re_\u2026`). Keys can be **full access** or restricted to **sending access** only \u2014 prefer the latter for agents that just send mail.\n\n```bash\ncurl https://api.resend.com/emails \\\n -H \"Authorization: Bearer $RESEND_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\"from\": \"you@yourdomain.com\", \"to\": [\"them@example.com\"], \"subject\": \"Hi\", \"text\": \"Hello from an agent.\"}'\n```\n\n**MCP (stdio):** the official server runs locally with the same key:\n\n```bash\nclaude mcp add resend -e RESEND_API_KEY=re_xxxxxxxxx -- npx -y resend-mcp\n```\n\nYou'll need a verified sending domain before production sends \u2014 set one up under [Domains](https://resend.com/domains).", + "guide": "**Hosted MCP with OAuth (fastest):** Resend hosts a Streamable HTTP server at `https://mcp.resend.com/mcp`. Connecting opens a browser window to log in to Resend and approve access.\n\n```bash\nclaude mcp add --transport http resend https://mcp.resend.com/mcp\n```\n\n**API key (headless agents):** create a key at [resend.com/api-keys](https://resend.com/api-keys) (`re_\u2026`). Keys can be **full access** or restricted to **sending access** only \u2014 prefer the latter for agents that just send mail. Pass it to the hosted server as a bearer header:\n\n```bash\nclaude mcp add --transport http resend https://mcp.resend.com/mcp --header \"Authorization: Bearer $RESEND_API_KEY\"\n```\n\n**Local MCP (stdio):** the open-source server runs on your machine with the same key:\n\n```bash\nclaude mcp add resend -e RESEND_API_KEY=$RESEND_API_KEY -- npx -y resend-mcp\n```\n\nThe same header works against the REST API (`curl https://api.resend.com/emails -H \"Authorization: Bearer $RESEND_API_KEY\"`). You'll need a verified sending domain before production sends \u2014 set one up under [Domains](https://resend.com/domains).", "sources": [ { "title": "Resend Docs \u2014 API keys & authentication", @@ -29,17 +34,27 @@ "url": "https://resend.com/docs/mcp-server" } ], - "generatedAt": "2026-06-11", + "generatedAt": "2026-10-20", "verified": true }, "interfaces": [ { "format": "mcp", "name": "Resend MCP", + "endpoint": "https://mcp.resend.com/mcp", + "auth": "mixed", + "authHeader": "Authorization: Bearer {re_key}", + "docs": "https://resend.com/docs/mcp-server", + "note": "OAuth in supporting clients (browser login); API-key bearer header for headless agents. Local stdio alternative: npx -y resend-mcp", + "origin": "vendor" + }, + { + "format": "mcp", + "name": "Resend MCP (local)", "auth": "api_key", "install": "claude mcp add resend -e RESEND_API_KEY=re_xxx -- npx -y resend-mcp", "docs": "https://resend.com/docs/mcp-server", - "note": "Runs locally over stdio with your API key.", + "note": "Open-source server; runs locally over stdio with your API key (RESEND_API_KEY).", "origin": "vendor" }, { diff --git a/scripts/normalize.test.ts b/scripts/normalize.test.ts index 0d29e445..69ad6523 100644 --- a/scripts/normalize.test.ts +++ b/scripts/normalize.test.ts @@ -1,6 +1,16 @@ import { describe, expect, test } from "bun:test"; -import { applyProviderIds, buildDiscoveredEntries, buildSearchIndex } from "./normalize.ts"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { + applyProviderIds, + assembleRecords, + buildCuratedEntries, + buildDiscoveredEntries, + buildIndex, + buildSearchIndex, +} from "./normalize.ts"; import { denylistEntries } from "../src/lib/catalog-denylist.ts"; +import type { Integration } from "../src/lib/types.ts"; describe("normalize discovered zero-surface domains", () => { test("threads empty and all-filtered discovered domains into the search index without fake kinds", () => { @@ -141,3 +151,215 @@ describe("provider IDs", () => { expect(out.map((r) => r.providerId)).toEqual(["a.com/mcp", undefined]); }); }); + +// ── Curated MCP entries vs hosted endpoints ───────────────────────────────── +// Local fixtures only: nothing here signs in anywhere or opens an MCP session. + +const noFeeds = { mcp: [], openapi: [], graphql: [], cli: [] }; +const DOMAIN = "synthetic-mail.com"; +const HOSTED = "https://mcp.synthetic-mail.com/mcp"; + +/** The shape curated/resend.json had: an MCP entry with only a local install. */ +const localOnlyCurated = { + slug: "synthetic-mail", + name: "Synthetic Mail", + description: "Email API.", + domain: DOMAIN, + interfaces: [ + { format: "mcp", name: "Synthetic MCP", auth: "api_key", install: "npx -y synthetic-mcp" }, + { format: "openapi", name: "Synthetic REST", spec: `https://${DOMAIN}/openapi.yaml`, auth: "api_key" }, + ], +}; + +const discoveredDomain = (mcpUrl: string | undefined) => ({ + domain: DOMAIN, + summary: "Email API.", + description: "Email API.", + surfaces: [ + { slug: "rest", name: "REST", type: "http" as const, url: `https://api.${DOMAIN}`, spec: `https://${DOMAIN}/spec.yaml`, authStatus: "required" as const }, + { slug: "mcp", name: "MCP", type: "mcp" as const, ...(mcpUrl ? { url: mcpUrl } : {}), authStatus: "required" as const }, + { slug: "cli", name: "CLI", type: "cli" as const, command: "synthetic", authStatus: "required" as const }, + ], +}); + +const unrelatedDomain = { + domain: "synthetic-other.com", + summary: "Another API.", + surfaces: [ + { slug: "mcp", name: "MCP", type: "mcp" as const, url: "https://mcp.synthetic-other.com/mcp", authStatus: "required" as const }, + { slug: "rest", name: "REST", type: "http" as const, url: "https://api.synthetic-other.com", authStatus: "required" as const }, + ], +}; + +const recordsFor = (records: Integration[], domain: string, kind: string) => + records.filter((r) => r.kind === kind && buildIndex([r])[0]?.domain === domain); + +describe("curated MCP interface selection", () => { + test("a hosted MCP interface listed after a local one is the one published", () => { + const records = buildCuratedEntries([ + { + ...localOnlyCurated, + interfaces: [ + ...localOnlyCurated.interfaces, + { format: "mcp", name: "Synthetic MCP (hosted)", endpoint: HOSTED, auth: "oauth" }, + ], + }, + ]); + + const mcp = records.filter((r) => r.kind === "mcp"); + expect(mcp).toHaveLength(1); + expect(mcp[0]!.mcp?.remoteUrl).toBe(HOSTED); + expect(mcp[0]!.mcp?.authTypes).toEqual(["oauth"]); + // The kind keeps its listing position, so slugs do not shift. + expect(records.map((r) => r.slug)).toEqual(["synthetic-mail-com", "synthetic-mail-com-openapi"]); + }); + + test("an unusable hosted endpoint does not displace the local entry", () => { + const records = buildCuratedEntries([ + { + ...localOnlyCurated, + interfaces: [ + ...localOnlyCurated.interfaces, + { format: "mcp", name: "Placeholder", endpoint: "https://{tenant}.synthetic-mail.com/mcp", auth: "oauth" }, + ], + }, + ]); + const mcp = records.filter((r) => r.kind === "mcp"); + expect(mcp).toHaveLength(1); + expect(mcp[0]!.mcp?.remoteUrl).toBeUndefined(); + expect(mcp[0]!.mcp?.authTypes).toEqual(["api_key"]); + }); + + test("other kinds keep first-listed-wins", () => { + const records = buildCuratedEntries([ + { + domain: DOMAIN, + interfaces: [ + { format: "openapi", name: "First", docs: `https://${DOMAIN}/docs` }, + { format: "openapi", name: "Second", spec: `https://${DOMAIN}/openapi.yaml` }, + ], + }, + ]); + expect(records).toHaveLength(1); + expect(records[0]!.openapi?.specUrl).toBeUndefined(); + expect(records[0]!.openapi?.docsUrl).toBe(`https://${DOMAIN}/docs`); + }); +}); + +describe("Resend hosted MCP", () => { + const resend = JSON.parse(readFileSync(join(import.meta.dir, "..", "curated", "resend.json"), "utf8")); + // Mirrors domains/resend.com/integrations.json. + const discoveredResend = { + domain: "resend.com", + summary: "Resend email API.", + surfaces: [ + { slug: "resend-rest-api-2", name: "Resend REST API", type: "http" as const, url: "https://api.resend.com", spec: "https://resend.com/openapi.yaml", authStatus: "required" as const }, + { slug: "resend-mcp", name: "Resend MCP", type: "mcp" as const, url: "https://mcp.resend.com/mcp", authStatus: "required" as const }, + { slug: "resend-cli", name: "Resend CLI", type: "cli" as const, command: "resend", authStatus: "required" as const }, + ], + }; + + test("the curated record publishes the hosted endpoint with OAuth + API-key auth, once", () => { + const { records, warnings } = assembleRecords(buildCuratedEntries([resend]), noFeeds, { domains: [discoveredResend] }); + + expect(warnings).toEqual([]); + const mcp = recordsFor(records, "resend.com", "mcp"); + expect(mcp).toHaveLength(1); + expect(mcp[0]!.feeds).toEqual(["curated"]); + expect(mcp[0]!.mcp?.remoteUrl).toBe("https://mcp.resend.com/mcp"); + expect(recordsFor(records, "resend.com", "openapi")).toHaveLength(1); + + const rows = buildSearchIndex(buildIndex(records)).filter((row) => row.domain === "resend.com"); + expect(rows).toHaveLength(1); + const surface = rows[0]!.surfaces?.find((s) => s.kind === "mcp"); + expect(surface?.url).toBe("https://mcp.resend.com/mcp"); + expect(surface?.auth?.kind).toBe("mixed"); + expect(surface?.auth?.header).toBe("Authorization: Bearer {re_key}"); + expect(surface?.auth?.note).toContain("OAuth"); + // The local stdio option stays discoverable alongside the hosted one. + expect(surface?.auth?.note).toContain("npx -y resend-mcp"); + }); + + test("the curated file keeps the local stdio interface", () => { + const local = resend.interfaces.find((i: { format: string; install?: string }) => i.format === "mcp" && i.install); + expect(local?.install).toContain("npx -y resend-mcp"); + expect(local?.endpoint).toBeUndefined(); + }); +}); + +describe("endpoint-less curated MCP entries", () => { + test("cannot silently suppress a discovered hosted endpoint", () => { + const { records, warnings } = assembleRecords(buildCuratedEntries([localOnlyCurated]), noFeeds, { + domains: [discoveredDomain(HOSTED)], + }); + + const mcp = recordsFor(records, DOMAIN, "mcp"); + expect(mcp).toHaveLength(1); + expect(mcp[0]!.feeds).toEqual(["discovered"]); + expect(mcp[0]!.mcp?.remoteUrl).toBe(HOSTED); + expect(warnings).toHaveLength(1); + expect(warnings[0]).toContain("curated/synthetic-mail-com-mcp"); + expect(warnings[0]).toContain(HOSTED); + // Only the open MCP slot is filled: the curated API still wins, and the + // domain's other discovered surfaces stay out as before. + const openapi = recordsFor(records, DOMAIN, "openapi"); + expect(openapi.map((r) => r.feeds)).toEqual([["curated"]]); + expect(recordsFor(records, DOMAIN, "cli")).toHaveLength(0); + }); + + test("a feed's hosted endpoint fills the slot ahead of discovery", () => { + const feed: Integration = { + id: "mcp/synthetic", + kind: "mcp", + slug: "synthetic", + name: "Synthetic", + description: "", + categories: [], + feeds: ["claude"], + mcp: { remoteUrl: "https://feed.synthetic-mail.com/mcp" }, + raw: {}, + }; + const { records, warnings } = assembleRecords( + buildCuratedEntries([localOnlyCurated]), + { ...noFeeds, mcp: [feed] }, + { domains: [discoveredDomain(HOSTED)] }, + ); + + const mcp = recordsFor(records, DOMAIN, "mcp"); + expect(mcp.map((r) => r.id)).toEqual(["mcp/synthetic"]); + expect(warnings).toHaveLength(1); + }); + + test("stays when no other source has a usable endpoint", () => { + for (const url of [undefined, "https://{tenant}.synthetic-mail.com/mcp", "http://localhost:3000/mcp"]) { + const { records, warnings } = assembleRecords(buildCuratedEntries([localOnlyCurated]), noFeeds, { + domains: [discoveredDomain(url)], + }); + const mcp = recordsFor(records, DOMAIN, "mcp"); + expect(mcp).toHaveLength(1); + expect(mcp[0]!.feeds).toEqual(["curated"]); + expect(warnings).toEqual([]); + } + }); + + test("a curated entry with an endpoint still owns its slot, and unrelated domains are untouched", () => { + const curated = buildCuratedEntries([ + { + ...localOnlyCurated, + interfaces: [{ format: "mcp", name: "Hosted", endpoint: "https://curated.synthetic-mail.com/mcp", auth: "oauth" }], + }, + ]); + const { records, warnings } = assembleRecords(curated, noFeeds, { + domains: [discoveredDomain(HOSTED), unrelatedDomain], + }); + + expect(warnings).toEqual([]); + const mcp = recordsFor(records, DOMAIN, "mcp"); + expect(mcp).toHaveLength(1); + expect(mcp[0]!.mcp?.remoteUrl).toBe("https://curated.synthetic-mail.com/mcp"); + expect(records.filter((r) => r.feeds.includes("discovered")).map((r) => r.id)).toEqual([ + "discovered/synthetic-other-com-mcp", + "discovered/synthetic-other-com-openapi", + ]); + }); +}); diff --git a/scripts/normalize.ts b/scripts/normalize.ts index 1ff392d9..e94638a3 100644 --- a/scripts/normalize.ts +++ b/scripts/normalize.ts @@ -455,6 +455,10 @@ export function buildDiscoveredEntries( knownRawDomains: Set, knownDomainKinds: Set, knownDomains: Set, + /** (domain, kind) slots a known record left without an endpoint. A + * discovered surface may fill one even though its domain is known — but + * only with a URL, since an endpoint is the one thing the slot is missing. */ + openDomainKinds: ReadonlySet = new Set(), ): { records: Integration[]; zeroSurfaceDomains: ZeroSurfaceDomain[] } { const recs: Integration[] = []; const zeroSurfaceDomains = new Map(); @@ -463,7 +467,10 @@ export function buildDiscoveredEntries( const domain = d.domain.trim().toLowerCase(); if (!domain) continue; const canonical = canonicalDomain(domain); - if (knownRawDomains.has(domain)) continue; + const knownRaw = knownRawDomains.has(domain); + const fillsOpenSlot = (s: DiscoveredSurface) => + Boolean(s.url) && openDomainKinds.has(domainKindKey(domain, discoveredKind(s.type))); + if (knownRaw && !(d.surfaces ?? []).some(fillsOpenSlot)) continue; if (DISCOVERED_ALIAS_DEDUPE.has(domain) && knownDomains.has(canonical)) continue; const surfaces = d.surfaces ?? []; const byKind = new Map(); @@ -471,6 +478,12 @@ export function buildDiscoveredEntries( // A client SDK/library mis-typed as `cli` is not a CLI surface — skip it. if (isSdkNotCli(surface)) continue; const kind = discoveredKind(surface.type); + if (fillsOpenSlot(surface)) { + if (!byKind.has(kind)) byKind.set(kind, surface); + continue; + } + // A known domain contributes only the slots left open for it. + if (knownRaw) continue; if (canonical === domain && knownDomainKinds.has(domainKindKey(domain, kind))) continue; if (!byKind.has(kind)) byKind.set(kind, surface); } @@ -592,27 +605,59 @@ const CURATED_KIND: Record = { cli: "cli", }; +/** Whether a curated MCP interface names a hosted endpoint a client can + * connect to, by the same rules the build publishes MCP endpoints with. */ +function hasHostedMcpEndpoint(iface: CuratedInterface): boolean { + return isHttpUrl(iface.endpoint) && isPublishableMcpUrl(iface.endpoint); +} + +/** The interface that represents each kind. The first one listed, except + * that a hosted MCP endpoint beats an endpoint-less (local stdio) entry + * listed before it — the picker's MCP surface is the one a client connects + * to, and the local program is the fallback, not the headline. */ +function primaryCuratedInterfaces(interfaces: CuratedInterface[]): Map { + const primary = new Map(); + for (const iface of interfaces) { + const kind = CURATED_KIND[(iface.format ?? "").toLowerCase()]; + if (!kind || iface.slug?.trim()) continue; + const current = primary.get(kind); + if (!current || (kind === "mcp" && !hasHostedMcpEndpoint(current) && hasHostedMcpEndpoint(iface))) { + primary.set(kind, iface); + } + } + return primary; +} + export function buildCurated(): Integration[] { if (!existsSync(CURATED)) return []; + const entries = readdirSync(CURATED) + .filter((file) => file.endsWith(".json")) + .map((file) => readJson(join(CURATED, file))); + return buildCuratedEntries(entries); +} + +export function buildCuratedEntries(entries: CuratedRecord[]): Integration[] { const recs: Integration[] = []; - for (const file of readdirSync(CURATED)) { - if (!file.endsWith(".json")) continue; - const entry = readJson(join(CURATED, file)); + for (const entry of entries) { const domain = (entry.domain ?? "").trim().toLowerCase(); if (!domain) continue; const description = (entry.description ?? entry.tagline ?? "").replace(/\s+/g, " ").trim(); const domainSlug = slugify(domain); + const primary = primaryCuratedInterfaces(entry.interfaces ?? []); const seen = new Set(); - for (const iface of entry.interfaces ?? []) { - const kind = CURATED_KIND[(iface.format ?? "").toLowerCase()]; + for (const listed of entry.interfaces ?? []) { + const kind = CURATED_KIND[(listed.format ?? "").toLowerCase()]; if (!kind) continue; - const productSlug = iface.slug?.trim().toLowerCase(); + const productSlug = listed.slug?.trim().toLowerCase(); // One record per kind: the picker offers a surface, not every endpoint. // Slugged interfaces are exempt — each is a product in its own right. if (!productSlug) { if (seen.has(kind)) continue; seen.add(kind); } + // The record keeps the position of the kind's first listing (which + // decides its slug) but takes its facts from the kind's primary interface. + const iface = productSlug ? listed : primary.get(kind)!; const productDescription = (iface.description ?? "").replace(/\s+/g, " ").trim(); const productDomain = iface.domain?.trim().toLowerCase() || domain; const rec: Integration = { @@ -667,14 +712,6 @@ export function buildCurated(): Integration[] { return dedupeSlugs(recs); } -function buildDiscovered( - knownRawDomains: Set, - knownDomainKinds: Set, - knownDomains: Set, -): { records: Integration[]; zeroSurfaceDomains: ZeroSurfaceDomain[] } { - return buildDiscoveredEntries(readDomainCatalogTree(DOMAINS), knownRawDomains, knownDomainKinds, knownDomains); -} - // ───────────────────────────────────────────────────────────────────────────── // Overrides: deep-merged onto records by id // ───────────────────────────────────────────────────────────────────────────── @@ -972,7 +1009,7 @@ function printDomainMismatchWarnings(all: Integration[]): void { if (warnings.length > 50) console.warn(`domain-mismatch warnings truncated: showing 50 of ${warnings.length}`); } -function buildIndex(all: Integration[]) { +export function buildIndex(all: Integration[]) { return all.map((r) => { const domain = recordDomain(r); const remapped = r.kind === "openapi" && DOMAIN_REMAP[(r.openapi?.provider ?? "").trim()]; @@ -1271,62 +1308,114 @@ export function applyProviderIds(recs: Integration[], ids: ReadonlyMap domainKindKey(recordDomain(r), r.kind))); +/** A curated MCP record with no endpoint describes a local program, not a + * surface a client can connect to. It must not hold its (domain, mcp) slot + * against a hosted endpoint from another source: that is how Resend's + * https://mcp.resend.com/mcp, sitting in domains/resend.com, never reached + * the catalog behind a curated entry that only had `npx -y resend-mcp`. */ +const leavesMcpSlotOpen = (r: Integration) => + r.kind === "mcp" && !r.standalone && !r.mcp?.remoteUrl; + +export interface FeedRecords { + mcp: Integration[]; + openapi: Integration[]; + graphql: Integration[]; + cli: Integration[]; +} + +/** Merge curated, feed and discovered records into the catalog's record list. + * Feeds arrive already filtered to public, publishable records; curated + * records come first so every other source dedupes against them. */ +export function assembleRecords( + curated: Integration[], + feeds: FeedRecords, + discoveredCatalog: Pick, +): { records: Integration[]; discoveredCount: number; zeroSurfaceDomains: ZeroSurfaceDomain[]; warnings: string[] } { + const slotKey = (r: Integration) => domainKindKey(recordDomain(r), r.kind); + const curatedKinds = new Set(curated.filter((r) => !leavesMcpSlotOpen(r)).map(slotKey)); // A crawled or third-party row for a (domain, kind) a human has already // verified is noise beside it, not extra coverage. - const notCurated = (r: Integration) => - !curatedKinds.has(domainKindKey(recordDomain(r), r.kind)); + const notCurated = (r: Integration) => !curatedKinds.has(slotKey(r)); - const baseMcp = [ + const base = [ ...curated.filter((r) => r.kind === "mcp"), - ...applyEndpointVerdicts(applyFavicons(applyToolsCache("mcp", applyOverrides("mcp", buildMcp())))) - .filter(isPublic) - .filter(notCurated), - ]; - const baseOpenapi = [ + ...feeds.mcp.filter(notCurated), ...curated.filter((r) => r.kind === "openapi"), - ...applyFavicons(applyToolsCache("openapi", applyOverrides("openapi", buildOpenapi()))) - .filter(isPublic) - .filter(notCurated), - ]; - const baseGraphql = [ + ...feeds.openapi.filter(notCurated), ...curated.filter((r) => r.kind === "graphql"), - ...applyFavicons(applyToolsCache("graphql", applyOverrides("graphql", buildGraphql()))) - .filter(isPublic) - .filter(notCurated), - ]; - const baseCli = [ + ...feeds.graphql.filter(notCurated), ...curated.filter((r) => r.kind === "cli"), - ...buildCli().filter(isPublic).filter(notCurated), + ...feeds.cli.filter(notCurated), ]; - const knownRawDomains = new Set( - [...baseMcp, ...baseOpenapi, ...baseGraphql, ...baseCli] - .map(rawRecordDomain) - .filter(Boolean), + const knownRawDomains = new Set(base.map(rawRecordDomain).filter(Boolean)); + const knownDomains = new Set(base.map(recordDomain).filter(Boolean)); + const knownDomainKinds = new Set( + base.flatMap((r) => { + const domain = recordDomain(r); + return domain ? [domainKindKey(domain, r.kind)] : []; + }), ); - const knownDomains = new Set( - [...baseMcp, ...baseOpenapi, ...baseGraphql, ...baseCli] - .map(recordDomain) - .filter(Boolean), + // Slots an endpoint-less curated record left open that no feed filled: a + // discovered hosted endpoint may still fill them. + const filled = new Set(base.filter((r) => r.kind === "mcp" && r.mcp?.remoteUrl).map(slotKey)); + const openDomainKinds = new Set( + curated.filter(leavesMcpSlotOpen).map(slotKey).filter((key) => !filled.has(key)), ); - const knownDomainKinds = new Set( - [...baseMcp, ...baseOpenapi, ...baseGraphql, ...baseCli] - .flatMap((r) => { - const domain = recordDomain(r); - return domain ? [domainKindKey(domain, r.kind)] : []; - }), + const discoveredBuild = buildDiscoveredEntries( + discoveredCatalog, + knownRawDomains, + knownDomainKinds, + knownDomains, + openDomainKinds, ); - const discoveredBuild = buildDiscovered(knownRawDomains, knownDomainKinds, knownDomains); const discovered = applyEndpointVerdicts(discoveredBuild.records.filter(isPublic)); + + // Every candidate has passed the same isPublic + endpoint-verdict rules as + // any other published record; an endpoint-less curated record yields to the + // first one with a URL, loudly, so the curated file gets fixed at the source. + const all = [...base, ...discovered]; + const hosted = new Map(); + for (const r of all) { + if (r.kind === "mcp" && r.mcp?.remoteUrl && !hosted.has(slotKey(r))) hosted.set(slotKey(r), r); + } + const warnings: string[] = []; + const records = all.filter((r) => { + if (!r.feeds.includes("curated") || !leavesMcpSlotOpen(r)) return true; + const replacement = hosted.get(slotKey(r)); + if (!replacement) return true; + warnings.push( + `curated-mcp-no-endpoint: ${r.id} has no endpoint; publishing ${replacement.id} (${replacement.mcp?.remoteUrl}) instead — add the verified endpoint to the curated file`, + ); + return false; + }); + + return { + records, + discoveredCount: discovered.length, + zeroSurfaceDomains: discoveredBuild.zeroSurfaceDomains, + warnings, + }; +} + +function main() { + // Order: build feed records → apply overrides (may add new records) → fill + // tools from cache → swap broken icons for domain-based fallbacks → keep only + // publicly-accessible records. + const assembled = assembleRecords( + buildCurated(), + { + mcp: applyEndpointVerdicts(applyFavicons(applyToolsCache("mcp", applyOverrides("mcp", buildMcp())))) + .filter(isPublic), + openapi: applyFavicons(applyToolsCache("openapi", applyOverrides("openapi", buildOpenapi()))).filter(isPublic), + graphql: applyFavicons(applyToolsCache("graphql", applyOverrides("graphql", buildGraphql()))).filter(isPublic), + cli: buildCli().filter(isPublic), + }, + readDomainCatalogTree(DOMAINS), + ); + for (const warning of assembled.warnings) console.warn(warning); const withIds = applyProviderIds( - [...baseMcp, ...baseOpenapi, ...baseGraphql, ...baseCli, ...discovered], + assembled.records, parseProviderIds(readJson(join(ROOT, "provider-ids.json"))), ); const mcp = withIds.filter((r) => r.kind === "mcp"); @@ -1343,7 +1432,7 @@ function main() { const index = buildIndex(all); writeFileSync(join(OUTPUT, "index.json"), JSON.stringify(index)); writeFileSync(join(OUTPUT, "catalog-seeds.json"), JSON.stringify(buildCatalogSeedData(all))); - writeFileSync(join(OUTPUT, "search-index.json"), JSON.stringify(buildSearchIndex(index, discoveredBuild.zeroSurfaceDomains))); + writeFileSync(join(OUTPUT, "search-index.json"), JSON.stringify(buildSearchIndex(index, assembled.zeroSurfaceDomains))); const mergedMcp = mcp.filter((r) => r.feeds.length > 1).length; const withTools = (rs: Integration[]) => @@ -1354,7 +1443,7 @@ function main() { console.log(`openapi: ${openapi.length.toString().padStart(5)} (${withTools(openapi)} with tools)`); console.log(`graphql: ${graphql.length.toString().padStart(5)} (${withTools(graphql)} with tools)`); console.log(`cli: ${cli.length.toString().padStart(5)}`); - console.log(`discovered: ${discovered.length.toString().padStart(5)}`); + console.log(`discovered: ${assembled.discoveredCount.toString().padStart(5)}`); console.log(`total: ${all.length.toString().padStart(5)}`); const validatedIcons = Object.keys(faviconCache).length; From 07aa32a339e8226ad47b0c2936c252faf5426999 Mon Sep 17 00:00:00 2001 From: Vaibhav Lajurkar Date: Tue, 20 Oct 2026 01:03:14 +0530 Subject: [PATCH 2/2] Publish discovered hosted MCP endpoints hidden behind unusable first entries; report withheld ones Audit of every discovered hosted MCP endpoint against the published search index (1,319 domains) found three causes of a missing or wrong MCP surface: - buildDiscoveredEntries kept the first MCP surface per domain even when it had no URL or an unusable {template} URL. That record was then dropped by the endpoint rules and the domain's real hosted server went with it. A hosted, publishable endpoint now beats an earlier unusable one (Algolia, Commerce Layer, LangChain/LangSmith, Snowplow, Vectorize, Halyard, PromptLayer, Loqate, Cosmic, Coverage Cat, Webiny, Weaviate, evlog). A promoted surface never becomes a second card for an endpoint another row already publishes; pre-existing duplicates are untouched. - A domain covered by another source with no MCP surface (an apis.guru spec, a GraphQL/CLI feed row, a curated file without an MCP interface) still has its discovered record skipped wholesale. Those hosted endpoints are now reported by the build (discovered-mcp-withheld) instead of disappearing silently; they are not auto-published, since there is no per-surface rejection if a crawled one is wrong. - The OpenAI directory feed lists Deel's staging host and AppDeploy's previous host. overrides/mcp/ points them at the production endpoints the vendors document. Co-Authored-By: Claude Opus 5.5 --- overrides/mcp/appdeploy.json | 5 ++ overrides/mcp/deel.json | 5 ++ scripts/normalize.test.ts | 139 +++++++++++++++++++++++++++++++++++ scripts/normalize.ts | 107 +++++++++++++++++++++++---- 4 files changed, 242 insertions(+), 14 deletions(-) create mode 100644 overrides/mcp/appdeploy.json create mode 100644 overrides/mcp/deel.json diff --git a/overrides/mcp/appdeploy.json b/overrides/mcp/appdeploy.json new file mode 100644 index 00000000..8546e9c3 --- /dev/null +++ b/overrides/mcp/appdeploy.json @@ -0,0 +1,5 @@ +{ + "mcp": { + "remoteUrl": "https://api-v2.appdeploy.ai/mcp" + } +} diff --git a/overrides/mcp/deel.json b/overrides/mcp/deel.json new file mode 100644 index 00000000..f408e730 --- /dev/null +++ b/overrides/mcp/deel.json @@ -0,0 +1,5 @@ +{ + "mcp": { + "remoteUrl": "https://api.letsdeel.com/mcp" + } +} diff --git a/scripts/normalize.test.ts b/scripts/normalize.test.ts index 69ad6523..e4f64dce 100644 --- a/scripts/normalize.test.ts +++ b/scripts/normalize.test.ts @@ -8,6 +8,7 @@ import { buildDiscoveredEntries, buildIndex, buildSearchIndex, + withheldDiscoveredMcp, } from "./normalize.ts"; import { denylistEntries } from "../src/lib/catalog-denylist.ts"; import type { Integration } from "../src/lib/types.ts"; @@ -363,3 +364,141 @@ describe("endpoint-less curated MCP entries", () => { ]); }); }); + +// ── Discovered MCP surfaces: one per domain, and the usable one ───────────── + +const mcpSurface = (slug: string, url?: string, authStatus: "none" | "required" | "unknown" = "required") => ({ + slug, + name: slug, + type: "mcp" as const, + ...(url ? { url } : {}), + authStatus, +}); +const discoveredOnly = (domain: string, surfaces: ReturnType[]) => ({ + domain, + summary: `${domain} fixture`, + surfaces: [ + { slug: "rest", name: "REST", type: "http" as const, url: `https://api.${domain}`, authStatus: "required" as const }, + ...surfaces, + ], +}); +const mcpOf = (records: Integration[], domain: string) => + records.filter((r) => r.kind === "mcp" && (r.raw.discovered as { domain?: string } | undefined)?.domain === domain); + +describe("discovered MCP surface selection", () => { + test("a hosted endpoint listed after an endpoint-less or templated one is published", () => { + // Shapes of domains/commercelayer.io, domains/langchain.com and domains/algolia.com. + const { records } = assembleRecords([], noFeeds, { + domains: [ + discoveredOnly("synthetic-commerce.com", [ + mcpSurface("docs-mcp", undefined, "none"), + mcpSurface("core-mcp", "https://core-mcp.synthetic-commerce.com/mcp"), + mcpSurface("metrics-mcp", "https://metrics-mcp.synthetic-commerce.com/mcp"), + ]), + discoveredOnly("synthetic-search.com", [ + mcpSurface("public-mcp", "https://{APP_ID}.synthetic-search.net/mcp/1/{UNIQ_ID}/mcp", "none"), + mcpSurface("productivity-mcp", "https://mcp.synthetic-search.com/mcp"), + ]), + ], + }); + + expect(mcpOf(records, "synthetic-commerce.com").map((r) => r.mcp?.remoteUrl)).toEqual([ + "https://core-mcp.synthetic-commerce.com/mcp", + ]); + expect(mcpOf(records, "synthetic-search.com").map((r) => r.mcp?.remoteUrl)).toEqual([ + "https://mcp.synthetic-search.com/mcp", + ]); + // The REST record and the domain's slugs are unaffected. + expect(records.filter((r) => r.kind === "openapi").map((r) => r.slug)).toEqual([ + "synthetic-commerce-com-openapi", + "synthetic-search-com-openapi", + ]); + }); + + test("the first listed hosted endpoint still wins, and an all-unusable domain still publishes no MCP", () => { + const { records } = assembleRecords([], noFeeds, { + domains: [ + discoveredOnly("synthetic-first.com", [ + mcpSurface("a", "https://a.synthetic-first.com/mcp"), + mcpSurface("b", "https://b.synthetic-first.com/mcp"), + ]), + discoveredOnly("synthetic-none.com", [ + mcpSurface("docs", undefined, "none"), + mcpSurface("local", "http://localhost:3000/mcp"), + ]), + ], + }); + expect(mcpOf(records, "synthetic-first.com").map((r) => r.mcp?.remoteUrl)).toEqual(["https://a.synthetic-first.com/mcp"]); + expect(mcpOf(records, "synthetic-none.com")).toEqual([]); + }); + + test("a promoted endpoint never becomes a second card for one another row already publishes", () => { + // domotz.app lists an unusable endpoint first, then domotz.com's own. + const shared = "https://mcp.synthetic-net.com/mcp"; + const { records } = assembleRecords([], noFeeds, { + domains: [ + discoveredOnly("synthetic-net.app", [mcpSurface("old", "https://{tenant}.synthetic-net.app/mcp"), mcpSurface("documented", shared)]), + discoveredOnly("synthetic-net.com", [mcpSurface("mcp", shared)]), + // Pre-existing duplicates (both first-listed) are left exactly as they were. + discoveredOnly("synthetic-twin-a.com", [mcpSurface("mcp", "https://mcp.synthetic-twin.com/mcp")]), + discoveredOnly("synthetic-twin-b.com", [mcpSurface("mcp", "https://mcp.synthetic-twin.com/mcp")]), + ], + }); + const byUrl = (url: string) => records.filter((r) => r.kind === "mcp" && r.mcp?.remoteUrl === url); + expect(byUrl(shared).map((r) => (r.raw.discovered as { domain: string }).domain)).toEqual(["synthetic-net.com"]); + expect(byUrl("https://mcp.synthetic-twin.com/mcp")).toHaveLength(2); + }); +}); + +describe("withheld discovered MCP endpoints", () => { + test("a hosted endpoint for a domain another source covers is reported, not published", () => { + // apis.guru covers synthetic-api.com with a spec; discovery also found its MCP server. + const spec: Integration = { + id: "openapi/synthetic-api-com", + kind: "openapi", + slug: "synthetic-api-com", + name: "Synthetic API", + description: "", + categories: [], + feeds: ["apis-guru"], + openapi: { provider: "synthetic-api.com", version: "1", specUrl: "https://synthetic-api.com/openapi.json", openapiVer: "3.0.0" }, + raw: {}, + }; + const legacySpec: Integration = { + ...spec, + id: "openapi/synthetic-legacy-com", + slug: "synthetic-legacy-com", + openapi: { ...spec.openapi!, provider: "synthetic-legacy.com" }, + }; + const catalog = { + domains: [ + discoveredOnly("synthetic-api.com", [mcpSurface("mcp", "https://mcp.synthetic-api.com/mcp")]), + // Already published under the vendor's current domain: not withheld. + discoveredOnly("synthetic-legacy.com", [mcpSurface("mcp", "https://mcp.synthetic-new.com/mcp")]), + discoveredOnly("synthetic-new.com", [mcpSurface("mcp", "https://mcp.synthetic-new.com/mcp")]), + ], + }; + const { records, withheldMcp } = assembleRecords([], { ...noFeeds, openapi: [spec, legacySpec] }, catalog); + + expect(records.filter((r) => r.kind === "mcp").map((r) => r.mcp?.remoteUrl)).toEqual(["https://mcp.synthetic-new.com/mcp"]); + expect(withheldMcp).toEqual([{ domain: "synthetic-api.com", url: "https://mcp.synthetic-api.com/mcp" }]); + // Nothing to report once the domain has an MCP surface of its own. + const ownMcp: Integration = { ...spec, id: "mcp/x", kind: "mcp", openapi: undefined, mcp: { remoteUrl: "https://other.synthetic-api.com/mcp" } }; + expect(withheldDiscoveredMcp([...records, ownMcp], catalog)).toEqual([]); + }); +}); + +describe("MCP overrides correcting feed URLs", () => { + // The OpenAI directory lists Deel's staging host and AppDeploy's previous + // host; the vendors document these instead. + const override = (slug: string) => + JSON.parse(readFileSync(join(import.meta.dir, "..", "overrides", "mcp", `${slug}.json`), "utf8")); + + test("Deel points at its documented production endpoint", () => { + expect(override("deel").mcp.remoteUrl).toBe("https://api.letsdeel.com/mcp"); + }); + + test("AppDeploy points at its documented production server", () => { + expect(override("appdeploy").mcp.remoteUrl).toBe("https://api-v2.appdeploy.ai/mcp"); + }); +}); diff --git a/scripts/normalize.ts b/scripts/normalize.ts index e94638a3..bc916250 100644 --- a/scripts/normalize.ts +++ b/scripts/normalize.ts @@ -226,6 +226,12 @@ function isHttpUrl(url: string | undefined): url is string { } } +/** Whether an MCP URL is a hosted endpoint a client can connect to, by the + * same rules the build publishes MCP endpoints with. */ +function isHostedMcpUrl(url: string | undefined): url is string { + return isHttpUrl(url) && isPublishableMcpUrl(url); +} + function isPlausibleSpecUrl(url: string | undefined): url is string { if (!isHttpUrl(url)) return false; const path = new URL(url).pathname; @@ -459,8 +465,14 @@ export function buildDiscoveredEntries( * discovered surface may fill one even though its domain is known — but * only with a URL, since an endpoint is the one thing the slot is missing. */ openDomainKinds: ReadonlySet = new Set(), -): { records: Integration[]; zeroSurfaceDomains: ZeroSurfaceDomain[] } { +): { + records: Integration[]; + zeroSurfaceDomains: ZeroSurfaceDomain[]; + /** MCP records whose surface beat an earlier, unusable one for its domain. */ + promoted: Set; +} { const recs: Integration[] = []; + const promoted = new Set(); const zeroSurfaceDomains = new Map(); for (const d of data.domains ?? []) { @@ -469,23 +481,38 @@ export function buildDiscoveredEntries( const canonical = canonicalDomain(domain); const knownRaw = knownRawDomains.has(domain); const fillsOpenSlot = (s: DiscoveredSurface) => - Boolean(s.url) && openDomainKinds.has(domainKindKey(domain, discoveredKind(s.type))); + isHostedMcpUrl(s.url) && openDomainKinds.has(domainKindKey(domain, discoveredKind(s.type))); if (knownRaw && !(d.surfaces ?? []).some(fillsOpenSlot)) continue; if (DISCOVERED_ALIAS_DEDUPE.has(domain) && knownDomains.has(canonical)) continue; const surfaces = d.surfaces ?? []; const byKind = new Map(); + // One surface per kind, first listed — except that for MCP a hosted, + // publishable endpoint beats an earlier entry without one. Otherwise a + // docs server listed with no URL, or a `{APP_ID}` template, wins the kind + // and is then dropped by the endpoint rules, taking the domain's real + // hosted server with it (Commerce Layer, LangChain, Algolia). + const promotedKinds = new Set(); + const take = (kind: Kind, surface: DiscoveredSurface) => { + const current = byKind.get(kind); + if (!current) { + byKind.set(kind, surface); + } else if (kind === "mcp" && !isHostedMcpUrl(current.url) && isHostedMcpUrl(surface.url)) { + byKind.set(kind, surface); + promotedKinds.add(kind); + } + }; for (const surface of surfaces) { // A client SDK/library mis-typed as `cli` is not a CLI surface — skip it. if (isSdkNotCli(surface)) continue; const kind = discoveredKind(surface.type); if (fillsOpenSlot(surface)) { - if (!byKind.has(kind)) byKind.set(kind, surface); + take(kind, surface); continue; } // A known domain contributes only the slots left open for it. if (knownRaw) continue; if (canonical === domain && knownDomainKinds.has(domainKindKey(domain, kind))) continue; - if (!byKind.has(kind)) byKind.set(kind, surface); + take(kind, surface); } const kinds = DISCOVERED_KIND_PRIORITY.filter((kind) => byKind.has(kind)); const description = discoveredDescription(d); @@ -513,6 +540,7 @@ export function buildDiscoveredEntries( feeds: ["discovered"], raw: { discovered: { domain, surface } }, }; + if (promotedKinds.has(kind)) promoted.add(rec); if (kind === "mcp") { rec.mcp = { remoteUrl: surface.url }; } else if (kind === "openapi") { @@ -539,7 +567,7 @@ export function buildDiscoveredEntries( }); } - return { records: dedupeSlugs(recs), zeroSurfaceDomains: [...zeroSurfaceDomains.values()] }; + return { records: dedupeSlugs(recs), zeroSurfaceDomains: [...zeroSurfaceDomains.values()], promoted }; } // ───────────────────────────────────────────────────────────────────────────── @@ -605,12 +633,6 @@ const CURATED_KIND: Record = { cli: "cli", }; -/** Whether a curated MCP interface names a hosted endpoint a client can - * connect to, by the same rules the build publishes MCP endpoints with. */ -function hasHostedMcpEndpoint(iface: CuratedInterface): boolean { - return isHttpUrl(iface.endpoint) && isPublishableMcpUrl(iface.endpoint); -} - /** The interface that represents each kind. The first one listed, except * that a hosted MCP endpoint beats an endpoint-less (local stdio) entry * listed before it — the picker's MCP surface is the one a client connects @@ -621,7 +643,7 @@ function primaryCuratedInterfaces(interfaces: CuratedInterface[]): Map, -): { records: Integration[]; discoveredCount: number; zeroSurfaceDomains: ZeroSurfaceDomain[]; warnings: string[] } { +): { + records: Integration[]; + discoveredCount: number; + zeroSurfaceDomains: ZeroSurfaceDomain[]; + warnings: string[]; + withheldMcp: { domain: string; url: string }[]; +} { const slotKey = (r: Integration) => domainKindKey(recordDomain(r), r.kind); const curatedKinds = new Set(curated.filter((r) => !leavesMcpSlotOpen(r)).map(slotKey)); // A crawled or third-party row for a (domain, kind) a human has already @@ -1369,7 +1397,18 @@ export function assembleRecords( knownDomains, openDomainKinds, ); - const discovered = applyEndpointVerdicts(discoveredBuild.records.filter(isPublic)); + // A promoted MCP surface must not become a second card for an endpoint + // another row already publishes: domotz.app's "documented endpoint" is the + // one domotz.com lists. Rows that were already duplicated are left alone. + const mcpUrlKey = (r: Integration) => (r.kind === "mcp" && r.mcp?.remoteUrl ? canonUrl(r.mcp.remoteUrl) : undefined); + const otherMcpUrls = new Set( + [...base, ...discoveredBuild.records.filter((r) => !discoveredBuild.promoted.has(r))].map(mcpUrlKey).filter(Boolean), + ); + const discovered = applyEndpointVerdicts( + discoveredBuild.records + .filter(isPublic) + .filter((r) => !discoveredBuild.promoted.has(r) || !otherMcpUrls.has(mcpUrlKey(r))), + ); // Every candidate has passed the same isPublic + endpoint-verdict rules as // any other published record; an endpoint-less curated record yields to the @@ -1395,9 +1434,42 @@ export function assembleRecords( discoveredCount: discovered.length, zeroSurfaceDomains: discoveredBuild.zeroSurfaceDomains, warnings, + withheldMcp: withheldDiscoveredMcp(records, discoveredCatalog), }; } +/** Hosted MCP endpoints discovery found for a domain that ends up with no MCP + * surface at all, and that no other row publishes either. Today that happens + * when another source (an apis.guru spec, a GraphQL or CLI feed row, a curated + * file without an MCP interface) already covers the domain: its discovered + * record is skipped wholesale. These are reported, not published — crawled + * data for a domain other sources already vouch for needs a maintainer's look, + * and the repo has no per-surface rejection if one turns out wrong. */ +export function withheldDiscoveredMcp( + records: Integration[], + discoveredCatalog: Pick, +): { domain: string; url: string }[] { + const published = new Set( + records.flatMap((r) => (r.kind === "mcp" && r.mcp?.remoteUrl ? [canonUrl(r.mcp.remoteUrl) ?? r.mcp.remoteUrl] : [])), + ); + const withMcp = new Set(records.filter((r) => r.kind === "mcp").map(recordDomain)); + const out: { domain: string; url: string }[] = []; + for (const d of discoveredCatalog.domains ?? []) { + const domain = d.domain.trim().toLowerCase(); + if (!domain || isJunkDomain(domain) || isDenylisted(domain)) continue; + if (withMcp.has(canonicalDomain(domain)) || out.some((w) => w.domain === domain)) continue; + const hosted = (d.surfaces ?? []).find( + (s) => + s.type === "mcp" && + isHostedMcpUrl(s.url) && + !GATED_HOST.test(new URL(s.url).hostname) && + !published.has(canonUrl(s.url) ?? s.url), + ); + if (hosted?.url) out.push({ domain, url: hosted.url }); + } + return out; +} + function main() { // Order: build feed records → apply overrides (may add new records) → fill // tools from cache → swap broken icons for domain-based fallbacks → keep only @@ -1455,6 +1527,13 @@ function main() { } printDomainMismatchWarnings(all); + + const withheld = assembled.withheldMcp; + if (withheld.length > 0) { + console.warn(`discovered-mcp-withheld: ${withheld.length} hosted MCP endpoints found by discovery are not published (domain covered by another source with no MCP surface)`); + for (const { domain, url } of withheld.slice(0, 50)) console.warn(`discovered-mcp-withheld: ${domain} ${url}`); + if (withheld.length > 50) console.warn(`discovered-mcp-withheld truncated: showing 50 of ${withheld.length}`); + } } if (import.meta.main) main();