feat(stocks): pick up new Coinbase stocks on Base from its registry

README.md

@@ -36,7 +36,7 @@ Creators choose a 0 / 1 / 3 % trading fee that is pushed **in full** to benefici
 - **Any quote asset.** ETH, USDG, USDC on Arc, **TWIG** ([Twigpine](https://twigpine.com)'s token on Base: a 1:1 wrapper of GITLAWB,
   wrap or unwrap any time with no fee; priced at GITLAWB's price; fees paid or burned in TWIG), **GITLAWB** ([Twigpine](https://twigpine.com)'s token, on Base and bridged 1:1 to
   Robinhood Chain; priced from its Base Uniswap v4 WETH pool; fees paid or burned in GITLAWB; the Base form offers TWIG in its place), or a **tokenized stock**: Coinbase's B20 stocks on Base
-  (every stock Coinbase lists: NVDAc, AAPLc, TSLAc, NFLXc, …; shown in USD where Chainlink publishes an on-chain feed) and Robinhood Stock Tokens on Robinhood Chain.
+  (every stock Coinbase lists: NVDAc, AAPLc, TSLAc, NFLXc, …, and new listings picked up from Coinbase's registry as they appear; shown in USD where Chainlink publishes an on-chain feed) and Robinhood Stock Tokens on Robinhood Chain.
 - **Trade in-page.** Buys and sells go straight to the Uniswap v4 pool through the Universal Router; the site
   never touches funds.
 - **Live everything.** One poller feeds the list, the trending strip, the tape, toasts and totals.

app/src/app/api/quotes/route.ts

@@ -1,7 +1,7 @@
 import { NextResponse } from "next/server";
 import { isChainKey } from "@/lib/chainPublic";
 import { launchpad } from "@/lib/launchpad/config";
-import { STOCK_SOURCE, ensureRegistry, stockIssued, stockList, stockPrices } from "@/lib/launchpad/stocksServer";
+import { STOCK_SOURCE, baseStockList, ensureRegistry, refreshBaseRegistry, stockIssued, stockList, stockPrices } from "@/lib/launchpad/stocksServer";
 import { searchStocks } from "@/lib/launchpad/stocks";
 import { BASE_STOCK_SEARCH_LIMIT, searchBaseStocks, stockTileSvg } from "@/lib/launchpad/baseStocks";
 import { memo } from "@/lib/launchpad/memo";
@@ -33,26 +33,33 @@ export async function GET(req: Request) {
   const { stocks, stocksTotal, stocksMatched } = await memo(`quotes:stocks:${chain}:${q.toUpperCase()}`, 30_000, async (): Promise<{ stocks: StockHit[]; stocksTotal: number; stocksMatched: number }> => {
     let hits: { address: string; symbol: string; name: string; decimals: number; logo: string | null }[];
     let matched: number;
+    // the total comes from the same list the search ran on: a registry refresh can land while prices are read below
+    let total: number;
     const source = STOCK_SOURCE[chain];
     if (source === null) return { stocks: [], stocksTotal: 0, stocksMatched: 0 };
     if (source === "coinbase-b20") {
-      const all = searchBaseStocks(q, Infinity);
+      void refreshBaseRegistry(); // background: the list as it stands answers now
+      const list = baseStockList();
+      const all = searchBaseStocks(q, Infinity, list);
       matched = all.length;
+      total = list.length;
       hits = all.slice(0, BASE_STOCK_SEARCH_LIMIT).map((s) => ({ ...s, logo: stockTileSvg(s.symbol) }));
     } else {
       if (!(await ensureRegistry())) return { stocks: [], stocksTotal: 0, stocksMatched: 0 };
+      const list = stockList(chain);
       const all = searchStocks(
-        stockList(chain).map((s) => ({ ...s, multiplier: 1 })),
+        list.map((s) => ({ ...s, multiplier: 1 })),
         q,
         Infinity,
       );
       matched = all.length;
+      total = list.length;
       hits = all.slice(0, 12);
     }
     const [px, issued] = await Promise.all([stockPrices(chain, hits.map((h) => h.address)), stockIssued(chain)]);
     return {
       stocks: hits.map((h) => ({ key: "stock" as const, address: h.address, symbol: h.symbol, name: h.name, decimals: h.decimals, usd: px.get(h.address) ?? null, logo: h.logo, issued: issued.get(h.address) ?? null })),
-      stocksTotal: stockList(chain).length,
+      stocksTotal: total,
       stocksMatched: matched,
     };
   });

app/src/lib/launchpad/base-registry.test.ts

@@ -0,0 +1,111 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { getMock } from "../db-mock.ts";
+import { BASE_STOCKS, BASE_STOCK_REGISTRY_URL, searchBaseStocks } from "./baseStocks.ts";
+import { baseStockList, ensureRegistry, refreshBaseRegistry, stockByAddress, stockIssued, stockList } from "./stocksServer.ts";
+
+// The Base list lives in the module, so this file walks it through one timeline: static, grown, cached, failed, retried.
+// fetch is mocked per URL: Coinbase's registry answers `reply`, anything else (Robinhood's registry) answers nothing useful.
+
+const NEW = "0xb200000000000000000000abcdef0123456789aa";
+const SPOOF = "0xb200000000000000000000abcdef0123456789ff";
+const NVDA = "0xb20000000000000000000078ee7ce2fe4908108c";
+const NFLX = "0xb20000000000000000000058b8c947e44011dfe6";
+const coinbaseRow = (symbol: string, address: string, name: string) => ({ contract_address: address, symbol, name, decimals: 8 });
+
+test("base list: static until a good registry read, then only grows; a failed read keeps it and is retried after a minute", async (t) => {
+  t.mock.timers.enable({ apis: ["Date"], now: 1_000_000 });
+  const warn = t.mock.method(console, "warn", () => {});
+  const info = t.mock.method(console, "info", () => {});
+  const coinbaseWarnings = () => warn.mock.calls.filter((c) => String(c.arguments[0]).includes("coinbase registry")).length; // Robinhood's registry warns on its own
+  let reply: (url: string) => Response = () => Response.json({ tokens: [...BASE_STOCKS.filter((s) => s.symbol !== "COINc").map((s) => coinbaseRow(s.symbol, s.address, `${s.name}, Inc.`))] });
+  const asked: string[] = [];
+  t.mock.method(globalThis, "fetch", async (url: unknown) => {
+    asked.push(String(url));
+    return String(url).startsWith(BASE_STOCK_REGISTRY_URL) ? reply(String(url)) : Response.json({ assets: [] });
+  });
+
+  // before any read: the static list, nothing fetched by a lookup
+  const before = stockList("base");
+  assert.equal(before.length, 146);
+  assert.equal(stockByAddress("base", NEW), null);
+  assert.deepEqual(asked, []);
+
+  // today's registry (145, no COINc): nothing new, so the very same rows
+  await refreshBaseRegistry();
+  assert.deepEqual(asked, [BASE_STOCK_REGISTRY_URL]);
+  assert.equal(stockList("base"), before);
+  assert.ok(stockByAddress("base", BASE_STOCKS.find((s) => s.symbol === "COINc")!.address), "COINc stays though the registry leaves it out");
+  assert.equal(stockByAddress("base", NFLX)?.name, "Netflix", "listed names are not replaced by the registry's");
+  assert.equal(info.mock.callCount(), 0);
+
+  // inside the 10 minutes: no read
+  await refreshBaseRegistry();
+  assert.equal(asked.length, 1);
+
+  // Coinbase lists a new stock (and the reply also carries a second "NVDAc" at another address)
+  reply = () => Response.json({ tokens: [coinbaseRow("NEWc", NEW, "New Robotics Corporation"), coinbaseRow("NVDAc", SPOOF, "NVIDIA Corporation")] });
+  t.mock.timers.tick(10 * 60_000);
+  await ensureRegistry(); // the page-data path starts it too
+  await refreshBaseRegistry();
+  assert.equal(asked.filter((u) => u === BASE_STOCK_REGISTRY_URL).length, 2);
+  const grown = stockList("base");
+  assert.equal(grown.length, 147);
+  assert.deepEqual(stockByAddress("base", NEW), { chain: "base", address: NEW, symbol: "NEWc", name: "New Robotics", decimals: 8, logo: grown.find((s) => s.address === NEW)!.logo, priced: false });
+  assert.ok(stockByAddress("base", NEW)!.logo?.startsWith("data:image/svg+xml,"));
+  assert.equal(stockByAddress("base", SPOOF), null, "a second NVDAc is never a stock");
+  assert.equal(stockByAddress("base", NVDA)?.priced, true);
+  assert.equal(stockByAddress("base", NVDA), before.find((s) => s.address === NVDA), "rows already built are reused, not rebuilt");
+  assert.deepEqual(searchBaseStocks("new", 13, baseStockList()).map((s) => s.symbol), ["NEWc"], "and search finds it");
+  assert.match(String(info.mock.calls[0]?.arguments[0]), /added 1 Base stock\(s\): NEWc/);
+
+  // the supply read covers the grown list
+  getMock().b20Supply = Object.fromEntries(stockList("base").map((s) => [s.address, 0n]));
+  assert.equal((await stockIssued("base")).get(NEW), false);
+
+  // registry down after the next 10 minutes: the list stays as it is; the failure is retried after a minute, not per request
+  reply = () => new Response("bad gateway", { status: 502 });
+  t.mock.timers.tick(10 * 60_000);
+  await refreshBaseRegistry();
+  assert.equal(stockList("base"), grown);
+  assert.equal(coinbaseWarnings(), 1);
+  await refreshBaseRegistry();
+  assert.equal(asked.filter((u) => u === BASE_STOCK_REGISTRY_URL).length, 3);
+  // a reply that is not the registry counts as a failure too, and never shrinks the list
+  reply = () => Response.json({ tokens: [] });
+  t.mock.timers.tick(60_000);
+  await refreshBaseRegistry();
+  assert.equal(asked.filter((u) => u === BASE_STOCK_REGISTRY_URL).length, 4);
+  assert.equal(stockList("base"), grown);
+  assert.equal(coinbaseWarnings(), 2);
+  // a later reply without NEWc does not remove it while the app runs
+  reply = () => Response.json({ tokens: [coinbaseRow("NVDAc", NVDA, "NVIDIA Corporation")] });
+  t.mock.timers.tick(60_000);
+  await refreshBaseRegistry();
+  assert.equal(asked.filter((u) => u === BASE_STOCK_REGISTRY_URL).length, 5);
+  assert.equal(stockList("base"), grown);
+  assert.ok(stockByAddress("base", NEW));
+
+  // a registry over two pages: the second is fetched with the first page's token, and its stock is added
+  const PAGED = "0xb200000000000000000000abcdef0123456789ab";
+  reply = (url) => Response.json(url === BASE_STOCK_REGISTRY_URL ? { tokens: [coinbaseRow("NVDAc", NVDA, "NVIDIA")], next_page_token: "MHhi+/2=" } : { tokens: [coinbaseRow("PAGEc", PAGED, "Paged Stock Inc.")] });
+  t.mock.timers.tick(10 * 60_000);
+  await refreshBaseRegistry();
+  assert.deepEqual(asked.slice(-2), [BASE_STOCK_REGISTRY_URL, `${BASE_STOCK_REGISTRY_URL}&page_token=MHhi%2B%2F2%3D`]);
+  assert.equal(stockByAddress("base", PAGED)?.name, "Paged Stock");
+  assert.equal(stockList("base").length, 148);
+  // ...and when a later page fails, nothing from the read is used (the list stays, retried after a minute)
+  const LOST = "0xb200000000000000000000abcdef0123456789ac";
+  reply = (url) => (url === BASE_STOCK_REGISTRY_URL ? Response.json({ tokens: [coinbaseRow("LOSTc", LOST, "Lost")], next_page_token: "p2" }) : new Response("bad gateway", { status: 502 }));
+  t.mock.timers.tick(10 * 60_000);
+  await refreshBaseRegistry();
+  assert.equal(stockByAddress("base", LOST), null);
+  assert.equal(stockList("base").length, 148);
+  assert.equal(coinbaseWarnings(), 3);
+});
+
+test("other chains never read Coinbase's registry and never see its stocks", async () => {
+  assert.equal(stockByAddress("robinhood", NEW), null);
+  assert.equal(stockByAddress("arc", NEW), null);
+  assert.deepEqual(stockList("arc"), []);
+});

app/src/lib/launchpad/baseStocks.test.ts

@@ -2,7 +2,7 @@ import { test } from "node:test";
 import assert from "node:assert/strict";
 import { createHash } from "node:crypto";
 import { isAddress } from "viem";
-import { BASE_STOCKS, BASE_STOCK_MAX_FEED_AGE_S, BASE_STOCK_SEARCH_LIMIT, baseStockByAddress, feedUsd, isB20Address, isB20RpcError, issuedFromSupply, parseRoundData, pricedBaseStocks, responseHasB20Error, searchBaseStocks, stockTileSvg, underlyingTicker } from "./baseStocks.ts";
+import { BASE_STOCKS, BASE_STOCK_MAX_FEED_AGE_S, BASE_STOCK_SEARCH_LIMIT, baseStockByAddress, BASE_STOCK_REGISTRY_URL, baseRegistryPageUrl, cleanStockName, feedUsd, isB20Address, isB20RpcError, issuedFromSupply, mergeBaseStocks, parseBaseRegistry, readBaseRegistry, parseRoundData, pricedBaseStocks, responseHasB20Error, searchBaseStocks, stockTileSvg, underlyingTicker } from "./baseStocks.ts";
 
 const symbols = (list: readonly { symbol: string }[]) => list.map((s) => s.symbol);
 /** The 13 stocks Chainlink publishes a "Coinbase <TICKER>" feed for on Base. */
@@ -125,6 +125,112 @@ test("feedUsd: 8-dec answer → USD; stale, future, zero or negative → null",
   assert.equal(feedUsd(null, now), null);
 });
 
+test("cleanStockName: the issuer's name the way the list writes it", () => {
+  const cases: [string, string][] = [
+    ["Tesla Inc.", "Tesla"],
+    ["Advanced Micro Devices, Inc.", "Advanced Micro Devices"],
+    ["Eli Lilly & Co", "Eli Lilly"],
+    ["Berkshire Hathaway Inc. Class B", "Berkshire Hathaway Class B"],
+    ["Eastman Kodak Company", "Eastman Kodak"],
+    ["The Hershey Company", "The Hershey Company"],
+    ["Papa John\u2019s International, Inc.", "Papa John's International"],
+    ["Novo Nordisk A/S", "Novo Nordisk"],
+    ["Anheuser-Busch InBev SA/NV", "Anheuser-Busch InBev"],
+    ["Banco Santander, S.A.", "Banco Santander"],
+    ["Planet Labs PBC", "Planet Labs"],
+    ["Arm Holdings plc", "Arm Holdings"],
+    ["  Sea   Limited ", "Sea"],
+  ];
+  for (const [raw, want] of cases) assert.equal(cleanStockName(raw), want, raw);
+  assert.equal(cleanStockName("A".repeat(60)).length, 40);
+});
+
+// one row of Coinbase's reply (api.coinbase.com/v1/tokenized-stocks), trimmed to the fields that are read
+const row = (symbol: string, address: string, extra: Record<string, unknown> = {}) => ({ contract_address: address, symbol, name: `${symbol.slice(0, -1)} Holdings, Inc.`, decimals: 8, isin: "US0000000000", ...extra });
+const NEW = "0xb200000000000000000000AbCdEf0123456789aa"; // a B20 address the static list does not have
+const NVDA = BASE_STOCKS.find((s) => s.symbol === "NVDAc")!;
+
+test("parseBaseRegistry: keeps well-formed B20 stock rows, lowercased, with no feed; drops everything else", () => {
+  const rows = parseBaseRegistry({
+    tokens: [
+      row("NEWc", NEW),
+      row("NEWc", "0xb200000000000000000000abcdef0123456789bb"), // repeated symbol: the first wins
+      row("DUPc", NEW), // repeated address
+      row("BADc", "0x1234567890123456789012345678901234567890"), // not B20
+      row("NVDA", "0xb200000000000000000000abcdef0123456789cc"), // no issuer "c"
+      row("nvdac", "0xb200000000000000000000abcdef0123456789cd"),
+      row("TOOLONGc", "0xb200000000000000000000abcdef0123456789ce"),
+      row("DECc", "0xb200000000000000000000abcdef0123456789cf", { decimals: 19 }),
+      row("DECc", "0xb200000000000000000000abcdef0123456789d0", { decimals: "8" }), // numeric string: accepted, and DECc is free again
+      row("ODDc", "0xb200000000000000000000abcdef0123456789d1", { name: "<img src=x onerror=alert(1)>" }),
+      row("NONc", "0xb200000000000000000000abcdef0123456789d2", { name: 42 }),
+      null,
+      "row",
+    ],
+  });
+  assert.deepEqual(rows, [
+    { symbol: "NEWc", name: "NEW Holdings", address: NEW.toLowerCase(), feed: null, decimals: 8 },
+    { symbol: "DECc", name: "DEC Holdings", address: "0xb200000000000000000000abcdef0123456789d0", feed: null, decimals: 8 },
+    { symbol: "ODDc", name: "ODD", address: "0xb200000000000000000000abcdef0123456789d1", feed: null, decimals: 8 },
+    { symbol: "NONc", name: "NON", address: "0xb200000000000000000000abcdef0123456789d2", feed: null, decimals: 8 },
+  ]);
+  for (const junk of [null, 42, "x", [], { tokens: "x" }, { data: [row("NEWc", NEW)] }]) assert.deepEqual(parseBaseRegistry(junk), [], JSON.stringify(junk));
+  // decimals must be a whole number or a string of digits: Number() would read null, "" and false as 0, true as 1, [8] as 8
+  for (const decimals of [null, undefined, "", " ", false, true, [8], 8.5, "8.0", "-1", -1, 19, "x"]) {
+    assert.deepEqual(parseBaseRegistry({ tokens: [row("NEWc", NEW, { decimals })] }), [], `decimals ${JSON.stringify(decimals)}`);
+  }
+  assert.equal(parseBaseRegistry({ tokens: [row("NEWc", NEW, { decimals: 0 })] })[0]?.decimals, 0, "a real 0 is still a number");
+  assert.deepEqual(parseBaseRegistry({ tokens: Array.from({ length: 1001 }, () => row("NEWc", NEW)) }), [], "a reply far larger than the registry is not the registry");
+});
+
+test("mergeBaseStocks: the registry can only add; it never relabels, unprices or shadows a listed stock", () => {
+  assert.equal(mergeBaseStocks(BASE_STOCKS, []), BASE_STOCKS);
+  assert.equal(mergeBaseStocks(BASE_STOCKS, BASE_STOCKS.map((s) => ({ ...s, name: "Renamed", feed: null }))), BASE_STOCKS, "every address known: same list back, static names and feeds untouched");
+  const added = { symbol: "NEWc", name: "New", address: NEW.toLowerCase(), feed: null, decimals: 8 };
+  const spoof = { ...added, symbol: "NVDAc", address: "0xb200000000000000000000abcdef0123456789ff" };
+  const lower = { ...added, symbol: "aaplc", address: "0xb200000000000000000000abcdef0123456789fe" }; // AAPLc in other case
+  const merged = mergeBaseStocks(BASE_STOCKS, [spoof, added, lower, { ...NVDA, name: "Not Nvidia", feed: null }]);
+  assert.equal(merged.length, BASE_STOCKS.length + 1, "only the new stock");
+  assert.ok(merged.includes(added));
+  assert.equal(merged.find((s) => s.symbol === "NVDAc"), NVDA, "NVDAc stays the listed row, feed and all");
+  assert.ok(!merged.some((s) => s.address === spoof.address), "a second NVDAc at another address is dropped");
+  const tickers = merged.map((s) => underlyingTicker(s.symbol));
+  assert.deepEqual(tickers, [...tickers].sort(), "still sorted by ticker");
+  assert.equal(mergeBaseStocks(merged, [added]), merged, "already there: nothing new");
+  // two new rows for one stock (say, across registry pages): the first wins, by address and by symbol
+  const twice = mergeBaseStocks(BASE_STOCKS, [added, { ...added }, { ...added, name: "Other", address: "0xb200000000000000000000abcdef0123456789fd" }]);
+  assert.equal(twice.length, BASE_STOCKS.length + 1);
+  assert.equal(twice.find((s) => s.symbol === "NEWc"), added);
+});
+
+test("readBaseRegistry: follows next_page_token through every page; a partial or looping read throws instead of passing for the registry", async () => {
+  assert.equal(baseRegistryPageUrl(null), BASE_STOCK_REGISTRY_URL);
+  assert.equal(baseRegistryPageUrl("ab+/="), `${BASE_STOCK_REGISTRY_URL}&page_token=ab%2B%2F%3D`, "the token is URL-encoded");
+  const NEW2 = "0xb200000000000000000000abcdef0123456789bb";
+  const pages: Record<string, unknown> = {
+    first: { tokens: [row("NEWc", NEW), row("AAPLc", BASE_STOCKS[1].address)], next_page_token: "p2" },
+    p2: { tokens: [row("NEXc", NEW2), row("NEWc", NEW)], next_page_token: "" },
+  };
+  const asked: (string | null)[] = [];
+  const rows = await readBaseRegistry(async (token) => {
+    asked.push(token);
+    return pages[token ?? "first"];
+  });
+  assert.deepEqual(asked, [null, "p2"]);
+  assert.deepEqual(rows.map((r) => r.symbol), ["NEWc", "AAPLc", "NEXc"], "both pages, NEWc once");
+  for (const end of [undefined, null, ""]) assert.equal((await readBaseRegistry(async () => ({ tokens: [row("NEWc", NEW)], next_page_token: end }))).length, 1, `next_page_token ${JSON.stringify(end)} ends it`);
+
+  const fails = async (name: string, getPage: (token: string | null) => Promise<unknown>, message: RegExp) => assert.rejects(readBaseRegistry(getPage), message, name);
+  await fails("a page without a token list", async (t) => (t === null ? { tokens: [], next_page_token: "p2" } : { error: "rate limited" }), /page 2 has no token list/);
+  await fails("a token that comes round again", async (t) => ({ tokens: [], next_page_token: t === "a" ? "b" : "a" }), /repeated next_page_token/);
+  await fails("a token that is not a string", async () => ({ tokens: [], next_page_token: 7 }), /bad or repeated next_page_token/);
+  let n = 0;
+  await fails("more than 10 pages", async () => ({ tokens: [], next_page_token: `p${++n}` }), /more than 10 pages/);
+  await fails("more than 1000 rows in all", async (t) => ({ tokens: Array.from({ length: 600 }, () => row("NEWc", NEW)), next_page_token: t === null ? "p2" : undefined }), /more than 1000 rows/);
+  await fails("a failed page fetch", async (t) => { if (t) throw new Error("base registry 502"); return { tokens: [], next_page_token: "p2" }; }, /502/);
+  assert.equal(n, 10, "stopped at the tenth page");
+});
+
 test("issuedFromSupply: supply above zero is issued, zero is not, a failed or odd read says nothing", () => {
   const [a, b, c, d] = BASE_STOCKS.slice(0, 4).map((s) => s.address);
   const out = issuedFromSupply([a.toUpperCase().replace("0X", "0x"), b, c, d], [