feat(research): SearXNG is the default provider + keyless local web_fetch

.env.example

@@ -417,16 +417,15 @@ ROBOCO_EXTERNAL_PR_REQUIRE_HUMAN_CONFIRM=true
 # =============================================================================
 # Web research
 # =============================================================================
-# Gives Board/PM roles web_search + web_fetch. INERT (empty results) without a
-# provider key; set one to make it live (Tavily / Brave / Exa / SearXNG, per
-# your config).
+# Gives Board/PM roles web_search + web_fetch. DEFAULT is the bundled
+# self-hosted SearXNG container (no API key, no cost); the compose files wire
+# ROBOCO_RESEARCH_SEARXNG_BASE_URL to it out of the box.
 # ROBOCO_RESEARCH_ENABLED=true
-# tavily | brave | exa | searxng | null : provider adapter.
-# ROBOCO_RESEARCH_PROVIDER=tavily
+# searxng (default) | tavily | brave | exa | null : provider adapter.
+# ROBOCO_RESEARCH_PROVIDER=searxng
+# Only needed for the paid adapters (tavily/brave/exa):
 # ROBOCO_RESEARCH_API_KEY=
-# Self-hosted SearXNG (keyless, free): point this at the compose service and
-# flip the provider above. Requires the searxng container (in the compose
-# files) and no API key.
+# Override only if you run SearXNG somewhere other than the bundled container:
 # ROBOCO_RESEARCH_SEARXNG_BASE_URL=http://roboco-searxng:8080
 
 # =============================================================================

CHANGELOG.md

@@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 
 ## [Unreleased]
 
+### Added
+
+- **Self-hosted SearXNG web research (the new default provider).** The research capability (Board + PM `web_search` / `web_fetch`) no longer requires a paid API: a keyless `searxng` provider talks to a bundled `roboco-searxng` container (all three compose files, internal-only, no host port, no nginx route; `deploy/searxng/settings.yml` enables `format=json` and disables the public-instance limiter; ~150-300 MB RAM). `web_fetch` on SearXNG downloads the page server-side and extracts readable text locally via trafilatura, guarded against SSRF: agent-supplied URLs must resolve to public addresses, so the orchestrator's internal network (DB, Redis, metadata) is unreachable through it. Tavily/Brave/Exa remain opt-in paid fallbacks via `.env`.
+
 ## [0.31.0] - 2026-09-17
 
 ### Added

docker-compose.registry.yml

@@ -194,8 +194,8 @@ x-orchestrator-env: &orchestrator-env
   # the key/token passthrough — without them a real value set in .env
   # never reaches the container and both stay inert regardless.
   ROBOCO_RESEARCH_API_KEY: ${ROBOCO_RESEARCH_API_KEY:-}
-  ROBOCO_RESEARCH_PROVIDER: ${ROBOCO_RESEARCH_PROVIDER:-tavily}
-  ROBOCO_RESEARCH_SEARXNG_BASE_URL: ${ROBOCO_RESEARCH_SEARXNG_BASE_URL:-}
+  ROBOCO_RESEARCH_PROVIDER: ${ROBOCO_RESEARCH_PROVIDER:-searxng}
+  ROBOCO_RESEARCH_SEARXNG_BASE_URL: ${ROBOCO_RESEARCH_SEARXNG_BASE_URL:-http://roboco-searxng:8080}
   ROBOCO_PROVISIONING_TOKEN: ${ROBOCO_PROVISIONING_TOKEN:-}
   ROBOCO_PROVISIONING_ORG: ${ROBOCO_PROVISIONING_ORG:-}
   # Spawn preflight — inert in practice (every real delivery role is

docker-compose.yaml

@@ -228,17 +228,17 @@ x-orchestrator-env: &orchestrator-env
   ROBOCO_STRATEGY_ENGINE_ENABLED: ${ROBOCO_STRATEGY_ENGINE_ENABLED:-true}
   ROBOCO_INTERNAL_PR_ENABLED: ${ROBOCO_INTERNAL_PR_ENABLED:-true}
   # Remaining default-OFF capabilities, ARMED here (personal deploy): web
-  # research (inert without ROBOCO_RESEARCH_API_KEY → empty results — the
-  # two lines below are what actually gets the .env key/provider into the
-  # container; without them .env values never reach it), pitch
+  # research (live by default via the bundled searxng container; the
+  # three lines below are what actually gets the .env key/provider/URL into
+  # the container; without them .env values never reach it), pitch
   # auto-provisioning (inert until a pitch is approved + a git token is
   # set — same deal for ROBOCO_PROVISIONING_TOKEN/_ORG below), and
   # transcript pruning (retention maintenance). All left OFF in the
   # registry compose. Override any via .env.
   ROBOCO_RESEARCH_ENABLED: ${ROBOCO_RESEARCH_ENABLED:-true}
   ROBOCO_RESEARCH_API_KEY: ${ROBOCO_RESEARCH_API_KEY:-}
-  ROBOCO_RESEARCH_PROVIDER: ${ROBOCO_RESEARCH_PROVIDER:-tavily}
-  ROBOCO_RESEARCH_SEARXNG_BASE_URL: ${ROBOCO_RESEARCH_SEARXNG_BASE_URL:-}
+  ROBOCO_RESEARCH_PROVIDER: ${ROBOCO_RESEARCH_PROVIDER:-searxng}
+  ROBOCO_RESEARCH_SEARXNG_BASE_URL: ${ROBOCO_RESEARCH_SEARXNG_BASE_URL:-http://roboco-searxng:8080}
   ROBOCO_PROVISIONING_ENABLED: ${ROBOCO_PROVISIONING_ENABLED:-true}
   ROBOCO_PROVISIONING_TOKEN: ${ROBOCO_PROVISIONING_TOKEN:-}
   ROBOCO_PROVISIONING_ORG: ${ROBOCO_PROVISIONING_ORG:-}
@@ -588,9 +588,9 @@ services:
   # ==========================================================================
   # SearXNG - Self-hosted metasearch for the web-research capability
   # ==========================================================================
-  # Keyless alternative to Tavily/Brave/Exa (set
-  # ROBOCO_RESEARCH_PROVIDER=searxng + ROBOCO_RESEARCH_SEARXNG_BASE_URL to
-  # use it). Internal-only by design: no host port, never proxied by nginx:
+  # The DEFAULT research provider (config default is searxng; the orchestrator
+  # env block points ROBOCO_RESEARCH_SEARXNG_BASE_URL at this container).
+  # Tavily/Brave/Exa remain opt-in via .env. Internal-only by design: no host port, never proxied by nginx:
   # an exposed open metasearch instance is abuse bait. The JSON format and
   # limiter-off live in deploy/searxng/settings.yml. The secret guards the
   # instance's own cookies only; on an internal bridge with no auth surface

docker-compose.yml

@@ -228,17 +228,17 @@ x-orchestrator-env: &orchestrator-env
   ROBOCO_STRATEGY_ENGINE_ENABLED: ${ROBOCO_STRATEGY_ENGINE_ENABLED:-true}
   ROBOCO_INTERNAL_PR_ENABLED: ${ROBOCO_INTERNAL_PR_ENABLED:-true}
   # Remaining default-OFF capabilities, ARMED here (personal deploy): web
-  # research (inert without ROBOCO_RESEARCH_API_KEY → empty results — the
-  # two lines below are what actually gets the .env key/provider into the
-  # container; without them .env values never reach it), pitch
+  # research (live by default via the bundled searxng container; the
+  # three lines below are what actually gets the .env key/provider/URL into
+  # the container; without them .env values never reach it), pitch
   # auto-provisioning (inert until a pitch is approved + a git token is
   # set — same deal for ROBOCO_PROVISIONING_TOKEN/_ORG below), and
   # transcript pruning (retention maintenance). All left OFF in the
   # registry compose. Override any via .env.
   ROBOCO_RESEARCH_ENABLED: ${ROBOCO_RESEARCH_ENABLED:-true}
   ROBOCO_RESEARCH_API_KEY: ${ROBOCO_RESEARCH_API_KEY:-}
-  ROBOCO_RESEARCH_PROVIDER: ${ROBOCO_RESEARCH_PROVIDER:-tavily}
-  ROBOCO_RESEARCH_SEARXNG_BASE_URL: ${ROBOCO_RESEARCH_SEARXNG_BASE_URL:-}
+  ROBOCO_RESEARCH_PROVIDER: ${ROBOCO_RESEARCH_PROVIDER:-searxng}
+  ROBOCO_RESEARCH_SEARXNG_BASE_URL: ${ROBOCO_RESEARCH_SEARXNG_BASE_URL:-http://roboco-searxng:8080}
   ROBOCO_PROVISIONING_ENABLED: ${ROBOCO_PROVISIONING_ENABLED:-true}
   ROBOCO_PROVISIONING_TOKEN: ${ROBOCO_PROVISIONING_TOKEN:-}
   ROBOCO_PROVISIONING_ORG: ${ROBOCO_PROVISIONING_ORG:-}
@@ -588,9 +588,9 @@ services:
   # ==========================================================================
   # SearXNG - Self-hosted metasearch for the web-research capability
   # ==========================================================================
-  # Keyless alternative to Tavily/Brave/Exa (set
-  # ROBOCO_RESEARCH_PROVIDER=searxng + ROBOCO_RESEARCH_SEARXNG_BASE_URL to
-  # use it). Internal-only by design: no host port, never proxied by nginx:
+  # The DEFAULT research provider (config default is searxng; the orchestrator
+  # env block points ROBOCO_RESEARCH_SEARXNG_BASE_URL at this container).
+  # Tavily/Brave/Exa remain opt-in via .env. Internal-only by design: no host port, never proxied by nginx:
   # an exposed open metasearch instance is abuse bait. The JSON format and
   # limiter-off live in deploy/searxng/settings.yml. The secret guards the
   # instance's own cookies only; on an internal bridge with no auth surface

docs/rag/architecture/config-reference.md

@@ -120,9 +120,9 @@ Pluggable external search/fetch for the Board + PM roles (`cell_pm`, `main_pm`,
 | Variable | Default | Description |
 |----------|---------|-------------|
 | `ROBOCO_RESEARCH_ENABLED` | `true` | Master switch. Ships default-**on** (unlike most feature flags): the `roboco-search` MCP server is mounted for the four research roles unless explicitly disabled. Panel-toggleable. Both NAS composes also set `ROBOCO_RESEARCH_ENABLED:-true` explicitly (redundant with the code default, but keeps the deploy's env self-documenting). |
-| `ROBOCO_RESEARCH_PROVIDER` | `tavily` | Adapter: `tavily` (LLM-native cited results + extract), `brave` (independent index, no fetch endpoint), `exa` (neural search + contents), `searxng` (self-hosted metasearch, keyless, no fetch endpoint, needs `ROBOCO_RESEARCH_SEARXNG_BASE_URL`), or `null` (always-empty stub). Swapping providers is a config change only. |
-| `ROBOCO_RESEARCH_API_KEY` | (unset) | API key for the selected provider. Server-side only. Unset ⇒ graceful `NullProvider` (empty results, never a hard fail). Not used by `searxng`, which keys off the base URL instead. |
-| `ROBOCO_RESEARCH_SEARXNG_BASE_URL` | (unset) | Base URL of the self-hosted SearXNG instance for the `searxng` adapter (e.g. `http://roboco-searxng:8080`, the container defined in the compose files). The instance must list `application/json` in its `search.formats` (the shipped `deploy/searxng/settings.yml` does). Unset with `searxng` selected ⇒ `NullProvider`. |
+| `ROBOCO_RESEARCH_PROVIDER` | `searxng` | Adapter. DEFAULT is `searxng` (self-hosted metasearch over the bundled `roboco-searxng` container: keyless, no per-query cost, `web_fetch` via a local SSRF-guarded trafilatura extractor). Paid fallbacks, opt-in via `.env`: `tavily` (LLM-native cited results + extract), `brave` (independent index, no fetch endpoint), `exa` (neural search + contents). `null` is the always-empty stub. Swapping providers is a config change only. |
+| `ROBOCO_RESEARCH_API_KEY` | (unset) | API key for the selected provider. Server-side only. Not used by `searxng`, which keys off the base URL instead. Unset with a paid adapter ⇒ graceful `NullProvider` (empty results, never a hard fail). |
+| `ROBOCO_RESEARCH_SEARXNG_BASE_URL` | (unset in code; composes default it to `http://roboco-searxng:8080`) | Base URL of the self-hosted SearXNG instance for the `searxng` adapter. The bundled container's settings (shipped `deploy/searxng/settings.yml`) already list `application/json` in `search.formats`. Unset with `searxng` selected ⇒ `NullProvider`. |
 | `ROBOCO_RESEARCH_MAX_RESULTS` | `5` | Hard cap (1-20) on `web_search` results per call. |
 | `ROBOCO_RESEARCH_FETCH_MAX_CHARS` | `20000` | Hard cap on characters `web_fetch` returns; content past this is truncated. |
 | `ROBOCO_RESEARCH_TIMEOUT_SECONDS` | `15.0` | Per-request timeout for outbound provider HTTP calls. |

docs/rag/tools/research-tools.md

@@ -2,7 +2,7 @@
 
 Two tools on the `roboco-search` MCP server: `web_search` and `web_fetch`. Mounted only for **Board + PM roles** — `cell_pm`, `main_pm`, `product_owner`, `head_marketing` — and only when `ROBOCO_RESEARCH_ENABLED` is true (default **on**; see `docs/rag/architecture/config-reference.md`). Every other role has no access.
 
-Calls flow agent → `roboco-search` MCP → `/api/research/search` or `/api/research/fetch` → `ResearchService` → the configured provider. The provider API key lives only in the server-side orchestrator process — it is never injected into an agent container, and the agent itself never egresses; the provider's own API does.
+Calls flow agent → `roboco-search` MCP → `/api/research/search` or `/api/research/fetch` → `ResearchService` → the configured provider. Any provider credentials live only in the server-side orchestrator process; they are never injected into an agent container, and the agent itself never egresses; the orchestrator's own outbound calls do (the provider's API, or, for SearXNG's `web_fetch`, a direct SSRF-guarded page download).
 
 ## `web_search`
 
@@ -15,7 +15,7 @@ web_search(query="competitor pricing for AI agent platforms", max_results=5)
 | `query` | yes | The search query. |
 | `max_results` | no | Cap on results, clamped server-side to `ROBOCO_RESEARCH_MAX_RESULTS` (default 5, hard ceiling 20). |
 
-Returns cited results (`title`, `url`, `snippet`, optional `score`) and, when the provider supports it (Tavily), a short synthesized `answer`. A `guidance` field reminds you to cite the URL for anything you rely on and to persist key findings — see below.
+Returns cited results (`title`, `url`, `snippet`, optional `score`) and, when the provider supports it (Tavily's synthesized answer, SearXNG instant answers), a short `answer`. A `guidance` field reminds you to cite the URL for anything you rely on and to persist key findings (see below).
 
 ## `web_fetch`
 
@@ -28,11 +28,11 @@ web_fetch(url="https://example.com/pricing", max_chars=5000)
 | `url` | yes | The page to extract readable content from. |
 | `max_chars` | no | Cap on returned characters, clamped server-side to `ROBOCO_RESEARCH_FETCH_MAX_CHARS` (default 20000). |
 
-Only works with providers that support content extraction (Tavily, Exa); the response's `truncated` field tells you whether the content was cut at the cap. Brave and SearXNG have no extraction endpoint; calling `web_fetch` against them returns a "does not support web_fetch" error (HTTP 501 at the route, surfaced as an error envelope to the tool).
+Works with Tavily and Exa (remote extraction endpoints) and with SearXNG, which downloads the page server-side and extracts readable text locally (trafilatura); the response's `truncated` field tells you whether the content was cut at the cap. Brave has no extraction path; calling `web_fetch` against it returns a "does not support web_fetch" error (HTTP 501 at the route, surfaced as an error envelope to the tool). SearXNG's local fetch refuses URLs that are not plain http(s) resolving to a public address, so it can never be aimed at the deployment's internal services.
 
-## Self-hosted provider (SearXNG)
+## Self-hosted provider (SearXNG, the default)
 
-`ROBOCO_RESEARCH_PROVIDER=searxng` uses the self-hosted SearXNG container (`roboco-searxng`, defined in the compose files) instead of a paid API. It needs **no API key**: set `ROBOCO_RESEARCH_SEARXNG_BASE_URL` (e.g. `http://roboco-searxng:8080`) and it is live. It is a metasearch aggregator (Google, Bing, DuckDuckGo, Brave, ...) run inside the deployment's own network: no external search account, no per-query cost. Because there is no extraction endpoint, `web_fetch` follows the 501 pattern above. Without the base URL configured, the provider degrades to the same always-empty behavior as `null`.
+`ROBOCO_RESEARCH_PROVIDER=searxng` (the **default**) uses the self-hosted SearXNG container (`roboco-searxng`, defined in the compose files) instead of a paid API. It needs **no API key**: the compose files point `ROBOCO_RESEARCH_SEARXNG_BASE_URL` at the bundled container (`http://roboco-searxng:8080`), so a stock stack is live out of the box. It is a metasearch aggregator (Google, Bing, DuckDuckGo, Brave, ...) run inside the deployment's own network: no external search account, no per-query cost. `web_fetch` works via the local extractor described above. Tavily/Brave/Exa remain opt-in paid fallbacks via `.env`. Without the base URL configured, the provider degrades to the same always-empty behavior as `null`.
 
 ## Cite-and-persist rules
 
@@ -48,7 +48,7 @@ Each agent gets `ROBOCO_RESEARCH_DAILY_QUOTA_PER_AGENT` (default 50) combined `w
 
 ## Graceful degradation
 
-With `ROBOCO_RESEARCH_PROVIDER=null` or no `ROBOCO_RESEARCH_API_KEY` configured, both tools still respond (never a hard failure) — `web_search` returns zero results with `provider: "null"` and a guidance string telling you to proceed without external sources or ask the CEO to configure a key; `web_fetch` returns empty content. Check the `provider` field in the response rather than assuming a key is set.
+With `ROBOCO_RESEARCH_PROVIDER=null`, or `searxng` selected with no base URL, or a paid adapter selected with no `ROBOCO_RESEARCH_API_KEY`, both tools still respond (never a hard failure): `web_search` returns zero results with `provider: "null"` and a guidance string telling you to proceed without external sources or ask the CEO to configure a provider; `web_fetch` returns empty content. Check the `provider` field in the response rather than assuming a provider is configured.
 
 ## CEO access
 

pyproject.toml

@@ -56,6 +56,7 @@ dependencies = [
     "fastapi-users[sqlalchemy]",  # Cloud auth: cookie session for the seeded CEO login
     "fastapi-users-db-sqlalchemy",  # imported directly (SQLAlchemyUserDatabase); declare it
     "pyjwt",  # imported directly as `jwt` in the cloud-auth JWT strategy
+    "trafilatura",  # services/research.py: SearXNG provider's local content extraction for web_fetch
 ]
 
 [project.optional-dependencies]

roboco/config.py

@@ -758,14 +758,16 @@ def _local_llm_base_url_internal_only(cls, v: str) -> str:
         ),
     )
     research_provider: str = Field(
-        default="tavily",
+        default="searxng",
         pattern="^(tavily|brave|exa|searxng|null)$",
         description=(
-            "Web-search provider adapter. 'tavily' (LLM-native cited results "
-            "+ extract), 'brave' (independent index; no fetch), 'exa' "
-            "(neural search + contents), 'searxng' (self-hosted metasearch, "
-            "no API key, no fetch), or 'null' (always-empty stub). Swapping "
-            "providers is a config change only."
+            "Web-search provider adapter. 'searxng' (DEFAULT: self-hosted "
+            "metasearch via the bundled roboco-searxng container, no API "
+            "key, needs research_searxng_base_url), 'tavily' (LLM-native "
+            "cited results + extract; opt-in paid fallback), 'brave' "
+            "(independent index; no fetch), 'exa' (neural search + "
+            "contents), or 'null' (always-empty stub). Swapping providers "
+            "is a config change only."
         ),
     )
     research_api_key: str | None = Field(
@@ -781,8 +783,9 @@ def _local_llm_base_url_internal_only(cls, v: str) -> str:
         description=(
             "Base URL of the self-hosted SearXNG instance (e.g. "
             "http://roboco-searxng:8080) for the 'searxng' provider. The "
-            "instance must allow the JSON format in its search.formats. "
-            "Server-side only; set/unset together with research_provider."
+            "compose files default this to the bundled roboco-searxng "
+            "container so the stock stack is live out of the box. Server-"
+            "side only; set/unset together with research_provider."
         ),
     )
     research_max_results: int = Field(

roboco/mcp/search_server.py

@@ -111,9 +111,9 @@ async def web_search(query: str, max_results: int | None = None) -> dict[str, An
     async def web_fetch(url: str, max_chars: int | None = None) -> dict[str, Any]:
         """Fetch the readable content of a specific web page.
 
-        Uses the configured provider's content-extraction endpoint, so it works
-        only with providers that support extraction (Tavily, Exa; SearXNG and
-        Brave do not). Content is truncated to the server's character cap.
+        Uses the configured provider's content extraction: a remote endpoint
+        (Tavily, Exa) or SearXNG's local extractor. Brave has none. Content
+        is truncated to the server's character cap.
         Args:
             url: The page URL to extract.
             max_chars: Optional cap on returned characters (clamped server-side).