.env.example

# tsbench-forge — local secrets & knobs. Gitignored; never commit real keys.
# Nothing auto-loads this file: load it with `set -a; source .env; set +a`
# (or direnv / your process manager) before running the scraper or agent.

# ---------------------------------------------------------------- required
# Source-discovery agent, full run only (`python -m source_discovery --out ...`).
# --coverage / --dry-run / --vet / --assess work without it, and the bulk
# sweeps (--bulk-*, --wire, --grind) make no model call at all.
#
# Any OpenAI-compatible /chat/completions endpoint works. Provider-neutral
# names are preferred; the OPENROUTER_* names below still work unchanged.
#
#   engy.ai (verified 2026-08-13):
#     LLM_BASE_URL=https://api.engy.ai/v1/chat/completions
#     LLM_MODEL=glm-5.2
#     ENGY_API_KEY=...
#     OPENROUTER_REASONING_ENABLED=false   # or LLM_REASONING_ENABLED
#
# That last line is not optional for glm-5.2 on engy: it thinks by default,
# and without the disable it spends the whole budget in reasoning_content and
# returns content="" with finish_reason="length" — an HTTP 200 that reads as
# truncation, not misconfiguration. engy accepts OpenRouter's
# `reasoning:{enabled:false}`; `enable_thinking` and `thinking.type` are
# silently ignored there.
#
# LLM_BASE_URL=
# LLM_MODEL=
# LLM_API_KEY=          # or the provider's own name, e.g. ENGY_API_KEY
# ENGY_API_KEY=
#
# `reasoning` is an OpenRouter-only field; it is sent only when LLM_BASE_URL
# points at openrouter.ai. Override with LLM_SEND_REASONING=true|false.
# LLM_SEND_REASONING=

# OpenRouter (the original default). Get a key at https://openrouter.ai/keys
OPENROUTER_API_KEY=

# Optional overrides (defaults shown)
# OPENROUTER_MODEL=z-ai/GLM-5.2
# OPENROUTER_BASE_URL=https://openrouter.ai/api/v1/chat/completions
# OPENROUTER_TEMPERATURE=0.4
# Reasoning models (GLM, o-series, R1) spend most of the completion budget on
# hidden reasoning — with the proposal ledger in the prompt, 8000 reliably
# starves them (finish_reason=length). 32000 keeps the failure rate low; the
# client retries once at 2x on exhaustion.
OPENROUTER_MAX_TOKENS=32000
# Reasoning controls. GLM-5.2 occasionally deliberates through the entire
# completion budget — even the 2x retry — and returns no content
# (finish_reason=length). The z-ai provider IGNORES the max_tokens cap
# (verified 2026-07-10), so for GLM the only reliable fix is disabling
# reasoning outright:
OPENROUTER_REASONING_ENABLED=false
# For providers that honour it, a token cap keeps reasoning bounded instead:
# OPENROUTER_REASONING_MAX_TOKENS=16000
# OPENROUTER_TIMEOUT=120

# ------------------------------------------------- per-source scraper keys
# All optional and free; a missing key only disables that one source.

# EIA (Henry Hub natural gas requires it; NYIS demand falls back to DEMO_KEY)
# https://www.eia.gov/opendata/register.php
EIA_API_KEY=

# OpenAQ air quality — https://explore.openaq.org/register
OPENAQ_API_KEY=

# Banxico SIE FX/rates — https://www.banxico.org.mx/SieAPIRest/service/v1/token
BANXICO_TOKEN=

# GitHub personal access token (only raises rate limits; no scopes needed)
GITHUB_TOKEN=

# NYC Open Data app token — https://data.cityofnewyork.us/profile/edit/developer_settings
NYC_OPENDATA_TOKEN=

# Singapore LTA DataMall — https://datamall.lta.gov.sg/content/datamall/en/request-for-api.html
LTA_ACCOUNT_KEY=

# Not keys — polite User-Agent strings these APIs ask for, e.g.
# "tsbench-forge (you@tensor-link.com)"
REDDIT_USER_AGENT=
CRATESIO_USER_AGENT=
SEC_USER_AGENT=

# ------------------- keys for discovered candidate sources (2026-07-04 autoresearch)
# Needed to activate the top autoresearch candidates once they enter sources.yaml.

# PurpleAir PM2.5 sensor network (2-min cadence — fills the healthcare/sub-hourly gap)
# Free key: https://develop.purpleair.com/keys
PURPLEAIR_API_KEY=

# ENTSO-E Transparency Platform (energy outage/unavailability events)
# Register at https://transparency.entsoe.eu/ then My Account Settings -> Web API Security Token
ENTSOE_SECURITY_TOKEN=

# Global Fishing Watch (vessel encounter events)
# Free token: https://globalfishingwatch.org/our-apis/tokens
GFW_API_TOKEN=

# NIST NVD CVE API works keyless (5 req/30s); optional key raises to 50 req/30s:
# https://nvd.nist.gov/developers/request-an-api-key
# NVD_API_KEY=

# PJM Data Miner 2 — api.pjm.com is connection-blocked from the AU scrape host;
# only useful if the cron moves. https://apiportal.pjm.com/
# PJM_SUBSCRIPTION_KEY=

# ------------------------------------------------------------------- knobs
# TSBENCH_STRONG=statsforecast   # swap panel anchor to AutoETS (needs statsforecast)
# SCRAPER_LOG_LEVEL=INFO
# TSBENCH_FEED_CACHE=~/.cache/tsbench-forge/feeds
# HF_HOME=                       # pre-staged HF weight cache for TSFM adapters

# ------ keys for 2026-07-05 ledger-round candidates (all free registrations)
# abuse.ch platforms (URLhaus / MalwareBazaar / ThreatFox) share one Auth-Key:
# https://auth.abuse.ch/ -> profile -> Auth-Key  (sent as Auth-Key header)
ABUSECH_AUTH_KEY=

# Trakt TV (trending shows, PT15M-ish attention series)
# https://trakt.tv/oauth/applications -> create app -> use the Client ID
TRAKT_CLIENT_ID=

# Finnhub market news event stream — https://finnhub.io/register
FINNHUB_TOKEN=

# US BEA (GDP/personal income, quarterly) — https://apps.bea.gov/api/signup/
BEA_API_KEY=

# TriMet GTFS-RT service alerts — https://developer.trimet.org/appid/registration/
TRIMET_APP_ID=

# AEMET Spanish weather observations — https://opendata.aemet.es/centrodedescargas/altaUsuario
AEMET_API_KEY=

# OpenEnergyMonitor emoncms (needs a public feed id too) — https://emoncms.org
# EMONCMS_API_KEY=
# BoardGameGeek XMLAPI2 has no key program; its 401s are bot-walling — no var to add.

# ------ keys for sweep-2 candidates (2026-07-06 vetting; all free registrations)
# USDA NASS Quick Stats (agricultural production, weekly/monthly)
# https://quickstats.nass.usda.gov/api -> request API key
USDA_NASS_API_KEY=

# ODPT Japan public transport (train status, irregular) — https://developer.odpt.org
ODPT_ACCESS_TOKEN=

# Schiphol flight movements — https://developer.schiphol.nl (app id + key pair)
SCHIPHOL_APP_ID=
SCHIPHOL_APP_KEY=

# FAA NOTAM API — https://api.faa.gov (external API portal, client credentials)
FAA_NOTAM_CLIENT_ID=
FAA_NOTAM_CLIENT_SECRET=

# Sonatype OSS Index (vulnerability disclosures) — https://ossindex.sonatype.org
OSSINDEX_API_TOKEN=

# Heavier OAuth setups — only if wanted:
# EBAY_OAUTH_TOKEN=        # https://developer.ebay.com (browse API)
# METACULUS_API_TOKEN=     # metaculus.com account settings

# ------ keys for sweep-2 key-gated sources (2026-07-08 keyless-wiring batch)
# Probe-verified real but credential-gated; each is disabled in sources.yaml
# until its key is set. Names follow the env:VAR / {VAR} convention (a *_API_KEY
# name is auto-sent as the X-API-Key header).

# Ocean Networks Canada — Oceans 3.0 sensor data (nature/half-hour).
# Free account, then Profile -> Web Services API -> token. https://data.oceannetworks.ca
ONC_API_TOKEN=

# UptimeRobot — monitor uptime/response (web_cloudops). Free.
# My Settings -> API Settings -> Main API Key. https://uptimerobot.com
UPTIMEROBOT_API_KEY=

# American Meteor Society fireball reports (nature/irregular).
# Members-only api_key (passed as ?api_key=). https://www.amsmeteors.org (member account)
AMS_API_KEY=

# WattTime marginal emissions (energy). Register a user, then basic-auth login
# returns a bearer token; free tier is region-limited. https://watttime.org/data-plans/
WATTTIME_USERNAME=
WATTTIME_PASSWORD=

# poweroutage.us outage counts (energy) — COMMERCIAL/paid API; public /api/web/* -> 401.
# Contact for access: https://poweroutage.us/
# POWEROUTAGE_API_KEY=

# ESA Space Weather Service Network (nature) — Liferay portal registration,
# per-product access (no simple bearer key). https://swe.ssa.esa.int
# ESA_SWE_TOKEN=

# No env var to add (documented for completeness):
#   IAEA PRIS reactor status — web UI only, licensed data, no public API.
#   NERC reliability alerts   — no RSS/JSON/API feed (bot-walled).

# ------ keys for 2026-07-10 autoresearch candidates (all free registrations)
# Probe-verified key-gated endpoints from the 100-round sweep; see
# src/sources/discovery/2026-07-10-autoresearch-100.md.

# WMATA rail/bus realtime (transport/irregular) — free key (demo tier available)
# https://developer.wmata.com
WMATA_API_KEY=

# TransLink Vancouver GTFS-RT alerts (transport/irregular) — free key
# https://developer.translink.ca
TRANSLINK_API_KEY=

# Cloudflare Radar outages/traffic (web_cloudops/half-hour) — free API token
# with Radar read permission: https://dash.cloudflare.com/profile/api-tokens
CLOUDFLARE_RADAR_TOKEN=

# BART realtime ETD (transport/sub-min) — free key
# https://api.bart.gov/api/register.aspx
BART_API_KEY=

# Lichess event/TV streams (sales/few-min) — free personal access token
# https://lichess.org/account/oauth/token
LICHESS_API_TOKEN=

# NASA FIRMS active-fire detections (nature/irregular; Tier-2 since 2026-07-04)
# Free MAP_KEY: https://firms.modaps.eosdis.nasa.gov/api/map_key/
FIRMS_MAP_KEY=

# OpenChargeMap charge-session/POI events (transport/irregular; 403 since 07-05)
# Free key: https://openchargemap.org/site/develop/api
OCM_API_KEY=

# AbuseIPDB blacklist snapshots (web_cloudops/irregular) — free tier key
# https://www.abuseipdb.com/account/api
ABUSEIPDB_API_KEY=

# NVE Norway hydrology/hydropower observations (energy/hourly) — free key
# https://hydapi.nve.no/UserDocumentation/
NVE_API_KEY=

# ------ keys for 2026-07-25 unlock-wave key-gated sources (all free registrations)
# Probe-verified key-gated during the unlock-wave grind; wire once the key is set.

# Twitch Helix — live stream viewer counts (sales/few-min). Register an app to get
# a Client ID + Secret, then mint an app access token (client-credentials grant).
# https://dev.twitch.tv/console/apps/create
TWITCH_CLIENT_ID=
TWITCH_CLIENT_SECRET=

# WAQI / aqicn.org — World Air Quality Index real-time feed (healthcare/hourly).
# Free token: https://aqicn.org/data-platform/token/
WAQI_TOKEN=

# Elexon BMRS — UK grid system frequency / generation (energy/sub-min).
# The legacy api.bmreports.com needs a scripting key: register at
# https://www.elexonportal.co.uk/ -> My Profile -> Scripting Key.
# NOTE: the newer Elexon Insights API (https://bmrs.elexon.co.uk) is KEYLESS —
# prefer wiring that host and skip this key if the endpoint is available there.
BMRS_API_KEY=

# SEC EDGAR full-text-search filing arrivals reuses SEC_USER_AGENT (a contact UA,
# not a key): e.g. "tsbench-forge you@tensor-link.com". https://www.sec.gov/os/webmaster-faq

# eia_steering_committee_weekly_storage reuses the existing EIA_API_KEY.
# Not free / no public key program (documented so nobody re-proposes them):
#   PulsePoint CAD incidents — agency agreements only.
#   Flightradar24, poweroutage.us, DTCC, Port of Rotterdam — commercial.
#   Stocktwits — public API retired; 403 is permanent.
#   TreasuryDirect TA_WS, AEMO NEMWeb, GBFS capitalbikeshare, OpenPrescribing —
#   keyless but bot-walled from this host (UA/anti-bot, not credentials).

# ============================================================================
# 2026-07-25 proposal-grind key-gated sources — placeholders + sign-up URLs.
# Fill a value to enable that key; then ask Claude to WIRE the source into
# sources.yaml (adding the key here alone does not create the source).
# ============================================================================

# ---- Tier 1: free self-serve keys (REST) — will wire once keyed ----
# US Census economic indicators (EITS) — instant free key
# https://api.census.gov/data/key_signup.html
CENSUS_API_KEY=
# AirNow real-time AQI (EPA) — free, instant
# https://docs.airnowapi.org/account/request/
AIRNOW_API_KEY=
# UN Comtrade international trade flows — free tier
# https://comtradedeveloper.un.org/
COMTRADE_SUBSCRIPTION_KEY=
# FCA UK financial firm register — free developer key
# https://register.fca.org.uk/Developer/s/
FCA_API_KEY=
# PVOutput rooftop solar generation — free (account -> API)
# https://pvoutput.org/account.jsp
PVOUTPUT_API_KEY=
# api.data.gov (FBI NIBRS crime + many US gov APIs) — free
# https://api.data.gov/signup/
DATA_GOV_API_KEY=
# Product Hunt launches (GraphQL) — free dev token
# https://api.producthunt.com/v2/oauth/applications
PRODUCT_HUNT_TOKEN=
# Spotify new-releases/track metadata — free app
# https://developer.spotify.com/dashboard
SPOTIFY_CLIENT_ID=
# (Spotify app secret, same dashboard)
# https://developer.spotify.com/dashboard
SPOTIFY_CLIENT_SECRET=
# TMDB movie/TV metadata — free key
# https://www.themoviedb.org/settings/api
TMDB_API_KEY=
# Last.fm charts/scrobbles — free key
# https://www.last.fm/api/account/create
LASTFM_API_KEY=
# YouTube Data API (video/stream stats) — free via GCP
# https://console.cloud.google.com/apis/credentials
YOUTUBE_API_KEY=
# Jamendo music streaming stats — free client id
# https://devportal.jamendo.com/
JAMENDO_CLIENT_ID=
# setlist.fm concert setlists — free key
# https://www.setlist.fm/settings/api
SETLISTFM_API_KEY=
# PhishTank phishing reports — free app key
# https://phishtank.org/api_register.php
PHISHTANK_APP_KEY=
# Alpaca US stock trades — free paper key id
# https://alpaca.markets/
APCA_API_KEY_ID=
# (Alpaca secret, same account)
# https://alpaca.markets/
APCA_API_SECRET_KEY=
# OpenSea NFT sales/floors v2 — free key
# https://docs.opensea.io/reference/api-keys
OPENSEA_API_KEY=
# The Graph (Uniswap etc subgraphs) — free gateway key
# https://thegraph.com/studio/apikeys/
THEGRAPH_API_KEY=
# aviationstack flight status — free tier access key
# https://aviationstack.com/product
AVIATIONSTACK_KEY=

# ---- Tier 1b: free transit/utility agency keys (REST) ----
# Navitia multi-region transit — free token
# https://navitia.io/
NAVITIA_TOKEN=
# Transport for NSW (Sydney) — free key
# https://opendata.transport.nsw.gov.au/
TFNSW_API_KEY=
# Transport for Greater Manchester — free
# https://developer.tfgm.com/
TFGM_SUBSCRIPTION_KEY=
# National Transport Authority Ireland (TFI) — free
# https://developer.nationaltransport.ie/
TFI_SUBSCRIPTION_KEY=
# Wellington Metlink (NZ) — free key
# https://opendata.metlink.org.nz/
METLINK_API_KEY=
# Ile-de-France Mobilites PRIM (RATP/Paris) — free
# https://prim.iledefrance-mobilites.fr/
PRIM_API_KEY=
# opentransportdata.swiss (SBB Swiss rail) — free
# https://opentransportdata.swiss/en/register/
OPENTRANSPORTDATA_TOKEN=
# AESO Alberta electricity market — free key
# https://api.aeso.ca/web/api
AESO_API_KEY=
# NHS England A&E / GP data — free subscription key
# https://digital.nhs.uk/developer
NHS_SUBSCRIPTION_KEY=

# ---- Tier 2: free but need approval before issuing ----
# TCGplayer card pricing — dev program approval
# https://developer.tcgplayer.com/
TCGPLAYER_TOKEN=
# Untappd beer check-ins — app approval (id+secret)
# https://untappd.com/api/register
UNTAPPD_CLIENT_ID=
# (Untappd secret, same app)
# https://untappd.com/api/register
UNTAPPD_CLIENT_SECRET=
# Met Office DataHub (DataPoint retired) — free account
# https://datahub.metoffice.gov.uk/
MET_OFFICE_API_KEY=
# Censys internet scans — account api id
# https://search.censys.io/account/api
CENSYS_API_ID=
# (Censys secret, same account)
# https://search.censys.io/account/api
CENSYS_API_SECRET=

# ---- Tier 3: keyed but ALSO blocked (scraper gap / commercial) — FYI only ----
# Etsy (needs custom x-api-key header - scraper gap): ETSY_API_KEY
# LooksRare (needs X-Looks-Api-Key header - scraper gap): LOOKSRARE_API_KEY
# Keepa Amazon price history - PAID: KEEPA_API_KEY
# Swiftly LA Metro/SF Muni realtime - agency/commercial: SWIFTLY_API_KEY
# Port of Antwerp NxtPort / Port of Rotterdam - OAuth commercial: NXTPORT_TOKEN / POR_API_KEY
# Copernicus CAMS air quality (ADS) - async CDS job API, needs cdsapi client: ADS_API_KEY
# aisstream.io / Blitzortung - WebSocket only (not REST): AISSTREAM_API_KEY

# ---- Tier 3 (commented): keyed but ALSO blocked — uncomment + fill only if the
#      blocker is resolved (scraper feature added, or commercial access bought) ----
# Copernicus CAMS air quality — free account, but async CDS job API (needs cdsapi client, not plain REST)
# https://ads.atmosphere.copernicus.eu/
# ADS_API_KEY=
# aisstream.io AIS vessels — free key, but WebSocket-only (scraper is HTTP GET/POST)
# https://aisstream.io/
# AISSTREAM_API_KEY=
# Etsy listings — free key, but requires a custom x-api-key header (scraper gap)
# https://www.etsy.com/developers/register
# ETSY_API_KEY=
# LooksRare NFT — key needs custom X-Looks-Api-Key header (scraper gap)
# https://docs.looksrare.org/developers/public-api-documentation
# LOOKSRARE_API_KEY=
# Keepa Amazon price history — PAID subscription API
# https://keepa.com/#!api
# KEEPA_API_KEY=
# Swiftly (LA Metro / SF Muni realtime) — agency/commercial, contact required
# https://www.goswift.ly/
# SWIFTLY_API_KEY=
# Port of Antwerp NxtPort — OAuth, commercial data marketplace
# https://www.nxtport.com/
# NXTPORT_TOKEN=
# Port of Rotterdam — commercial/contact for API access
# https://www.portofrotterdam.com/en/building-port/smartport
# POR_API_KEY=

# ------------------------------------------------ storage & infrastructure
# NOT source keys. These are what a restore needs FIRST: without the Hippius
# block nothing is mirrored off the box, and the scraped tree becomes the only
# copy of the data. See docs/RUNBOOK.md.

# Hippius S3 mirror — scrape_all.sh pushes data/ + sources.yaml here every
# 15 min, and sync_storage.py --pull restores from it. Append-only.
HIPPIUS_S3_ACCESS_KEY=
HIPPIUS_S3_SECRET_KEY=
# Optional overrides (defaults shown in src/sources/sync_storage.py)
# HIPPIUS_S3_BUCKET=tsbench-forge-sources
# HIPPIUS_S3_ENDPOINT=
# HIPPIUS_S3_REGION=decentralized

# Eval-pool publish target. Belongs to the cascade side, which pulls FROM the
# Hippius bucket; listed here only so a restore knows the name exists.
TSFORGE_S3_ACCESS_KEY=
TSFORGE_S3_SECRET_KEY=
CASCADE_S3_ACCESS_KEY=
CASCADE_S3_SECRET_KEY=

# Model / compute accounts used by the comparison harness, not by scraping.
HF_TOKEN=
LIUM_API_KEY=
TABPFN_TOKEN=

# ------------------------------------------- per-source keys added since v1
# Same rule as above: a missing key disables just that source.

# PJM (US grid operator) — https://apiportal.pjm.com/
PJM_SUBSCRIPTION_KEY=
# Emoncms self-hosted energy monitoring — https://emoncms.org/
EMONCMS_API_KEY=
# ESA space-weather service — https://swe.ssa.esa.int/
ESA_SWE_TOKEN=
# NVD CVE feed; without a key the rate limit is severe
# https://nvd.nist.gov/developers/request-an-api-key
NVD_API_KEY=
# Metaculus forecast series — https://www.metaculus.com/accounts/signup/
METACULUS_API_TOKEN=
# eBay Browse API — OAuth token, expires and must be refreshed
# https://developer.ebay.com/
EBAY_OAUTH_TOKEN=