From 734065e99ff1d70c2f246510faa223f634f30e71 Mon Sep 17 00:00:00 2001 From: Fabio Leitao Date: Mon, 19 Oct 2026 11:22:17 -0300 Subject: [PATCH 01/12] fix(security): quote non-simple Oracle catalog identifiers (Closes #2094) Unquoted emission stays only for simple [A-Z][A-Z0-9_$#]* names, including reflected lowercase objects. Any other catalog name is quoted with embedded quotes doubled. --- connectors/sql_connector.py | 6 ++++-- connectors/sql_sampling.py | 16 ++++++++++++++-- tests/test_sql_sampling.py | 8 ++++++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/connectors/sql_connector.py b/connectors/sql_connector.py index ea87cf042..360382535 100644 --- a/connectors/sql_connector.py +++ b/connectors/sql_connector.py @@ -819,8 +819,10 @@ def sample( ) use_limit = resolve_sql_sample_limit(base) dialect = self.engine.dialect.name if self.engine else "" - # Escape identifier per dialect to prevent SQL injection (identifiers come from discover(), not user input). - # Double-quote for SQLite/Postgres/Oracle; backtick for MySQL. + # Escape identifiers per dialect. Names come from the scanned catalog + # (discover()), which is not a trusted input. Oracle quoting is + # oracle_sql_identifier; other dialects double the quote character. + # Double-quote for SQLite/Postgres; backtick for MySQL. safe_col = column_name.replace('"', '""') safe_table = table.replace('"', '""') safe_schema = (schema or "").replace('"', '""') diff --git a/connectors/sql_sampling.py b/connectors/sql_sampling.py index b3916aff6..7010f7636 100644 --- a/connectors/sql_sampling.py +++ b/connectors/sql_sampling.py @@ -22,7 +22,8 @@ `sql_connector.sample`); this module only composes SQL shape. **Oracle** is the exception: pass optional ``raw_col`` / ``raw_table`` / ``raw_schema`` from discovery (may be ``quoted_name``) so case-sensitive quoted objects are preserved; -unquoted dictionary names are emitted without double quotes (uppercase). +simple unquoted dictionary names are emitted without double quotes +(uppercase); any other catalog name is quoted. Architecture (orchestrator vs fixed strings): - `SamplingManager` picks a **strategy label** and builds a `TextClause` from @@ -53,6 +54,7 @@ from __future__ import annotations import os +import re from dataclasses import dataclass from typing import Any @@ -220,6 +222,10 @@ def _ansi_quoted_table(safe_schema: str, safe_table: str, schema: str | None) -> return f'"{safe_schema}"."{safe_table}"' if schema else f'"{safe_table}"' +# Unquoted Oracle identifiers are uppercase and start with a letter (#2094). +_ORACLE_SIMPLE_IDENTIFIER = re.compile(r"^[A-Z][A-Z0-9_$#]*$") + + def oracle_sql_identifier(name: Any) -> str: """ Render an Oracle SQL identifier from a discovery/reflection name. @@ -228,6 +234,9 @@ def oracle_sql_identifier(name: Any) -> str: those in double quotes forces case-sensitive resolution and yields ORA-00942. Objects created with quoted lowercase identifiers are reflected as ``quoted_name(..., quote=True)`` and must keep exact case inside quotes. + Any other name that is not a simple unquoted identifier (``[A-Z][A-Z0-9_$#]*``) + is emitted quoted, with embedded ``"`` doubled, so catalog syntax cannot + splice into the sampling statement. """ raw = str(name) safe = raw.replace('"', '""') @@ -235,7 +244,10 @@ def oracle_sql_identifier(name: Any) -> str: return f'"{safe}"' if raw != raw.lower() and raw != raw.upper(): return f'"{safe}"' - return safe.upper() + upper = safe.upper() + if _ORACLE_SIMPLE_IDENTIFIER.fullmatch(upper): + return upper + return f'"{safe}"' def _oracle_table_ref(schema_name: Any, table_name: Any, schema: str | None) -> str: diff --git a/tests/test_sql_sampling.py b/tests/test_sql_sampling.py index 337adb0e0..990c4a066 100644 --- a/tests/test_sql_sampling.py +++ b/tests/test_sql_sampling.py @@ -97,6 +97,14 @@ def test_oracle_sql_identifier_mixed_case_quoted(): assert oracle_sql_identifier("Lab_Customers") == '"Lab_Customers"' +def test_oracle_sql_identifier_quotes_uppercase_names_with_sql_syntax(): + """#2094: quote catalog names that are not a simple unquoted Oracle identifier.""" + assert oracle_sql_identifier("ATTACKER.EVIL()") == '"ATTACKER.EVIL()"' + assert oracle_sql_identifier('SAY "HI"') == '"SAY ""HI"""' + assert oracle_sql_identifier("COL$ID") == "COL$ID" + assert oracle_sql_identifier("A#1") == "A#1" + + def test_build_oracle_reflected_lowercase_emits_unquoted_upper(): """Regression #1370: quoted lowercase breaks ORA-00942 on default Oracle objects.""" q = SqlColumnSampleQueryBuilder.build( From a50cb022c913779ce59ff6ee00405a784b717551 Mon Sep 17 00:00:00 2001 From: Fabio Leitao Date: Mon, 19 Oct 2026 11:32:39 -0300 Subject: [PATCH 02/12] fix(security): fail closed on non-REST pass_from_env names (Closes #2086) Share the #1977 env-name prefixes and an explicit destination allowlist in the loader. Non-REST targets and findings_sink copy an env credential only after both checks pass. --- config/loader.py | 83 +++++++++++++++++++++++++- connectors/rest_connector.py | 17 ++---- core/findings_sink.py | 3 + tests/test_credential_env_allowlist.py | 73 ++++++++++++++++++++++ tests/test_main_validate_config.py | 8 ++- tests/test_security.py | 26 +++++--- 6 files changed, 186 insertions(+), 24 deletions(-) create mode 100644 tests/test_credential_env_allowlist.py diff --git a/config/loader.py b/config/loader.py index 7a0ad644e..c3615d7a4 100644 --- a/config/loader.py +++ b/config/loader.py @@ -15,6 +15,7 @@ import warnings from pathlib import Path from typing import Any +from urllib.parse import urlparse import yaml @@ -53,6 +54,77 @@ def _warn_if_env_from_key_unset_or_empty( ) +# Same env-name prefixes as the REST credential gate (#1977 / #2086). +CREDENTIAL_ENV_PREFIXES = ("DATA_BOAR_", "REST_API_", "API_") +_CREDENTIAL_ENV_TAG = "#1977" +_REST_TARGET_TYPES = frozenset({"api", "rest", "rest_api", "http", "https"}) + + +def assert_credential_env_name(env_name: str, label: str) -> None: + """Fail closed unless the env var name uses a #1977 credential prefix.""" + name = env_name.strip() + if any(name.startswith(prefix) for prefix in CREDENTIAL_ENV_PREFIXES): + return + raise ValueError( + f"{label} must use one of the prefixes " + f"{', '.join(CREDENTIAL_ENV_PREFIXES)} ({_CREDENTIAL_ENV_TAG})." + ) + + +def _explicit_allowed_hosts(cfg: dict[str, Any]) -> set[str]: + raw = cfg.get("allowed_hosts") + if not raw and isinstance(cfg.get("auth"), dict): + raw = cfg["auth"].get("allowed_hosts") or [] + if isinstance(raw, str): + raw = [raw] + if not isinstance(raw, list): + return set() + return {str(host).lower().strip() for host in raw if str(host).strip()} + + +def _destination_host(cfg: dict[str, Any]) -> str: + host = str(cfg.get("host") or "").strip().lower() + if host: + return host + for key in ("base_url", "url"): + raw = str(cfg.get(key) or "").strip() + if not raw: + continue + parsed = (urlparse(raw).hostname or "").lower() + if parsed: + return parsed + return "" + + +def assert_explicit_destination_allowlisted(cfg: dict[str, Any], label: str) -> None: + """Fail closed unless the credential destination is explicitly allowlisted.""" + host = _destination_host(cfg) + allowed = _explicit_allowed_hosts(cfg) + if host and host in allowed: + return + raise ValueError( + f"{label} requires an explicit allowed_hosts entry " + f"for the destination ({_CREDENTIAL_ENV_TAG})." + ) + + +def assert_non_rest_env_credential( + cfg: dict[str, Any], env_name: str, label: str +) -> None: + """Prefix plus destination allowlist for non-REST env credentials (#2086).""" + assert_credential_env_name(env_name, label) + assert_explicit_destination_allowlisted(cfg, label) + + +def _gate_non_rest_env_credential( + target: dict[str, Any], env_name: str, label: str +) -> None: + kind = str(target.get("type") or "").strip().lower() + if kind in _REST_TARGET_TYPES: + return + assert_non_rest_env_credential(target, env_name, label) + + def _notification_env_value(val: Any) -> str | None: """ Resolve optional notification secrets: inline string or ${ENV_VAR} from the environment. @@ -737,10 +809,11 @@ def _normalize_max_expansion_ratio(val: Any) -> float | None: t.get("pass_from_env") or t.get("password_from_env") or "" ).strip() if env_pass_key: - _warn_if_env_from_key_unset_or_empty( - env_pass_key, - ("pass_from_env" if t.get("pass_from_env") else "password_from_env"), + pass_label = ( + "pass_from_env" if t.get("pass_from_env") else "password_from_env" ) + _gate_non_rest_env_credential(t, env_pass_key, pass_label) + _warn_if_env_from_key_unset_or_empty(env_pass_key, pass_label) t["pass"] = ( (os.environ.get(env_pass_key) or "").strip() or t.get("pass") @@ -749,6 +822,7 @@ def _normalize_max_expansion_ratio(val: Any) -> float | None: ) env_user_key = (t.get("user_from_env") or "").strip() if env_user_key: + _gate_non_rest_env_credential(t, env_user_key, "user_from_env") _warn_if_env_from_key_unset_or_empty(env_user_key, "user_from_env") t["user"] = ( (os.environ.get(env_user_key) or "").strip() @@ -760,12 +834,14 @@ def _normalize_max_expansion_ratio(val: Any) -> float | None: if isinstance(auth, dict): token_env = (auth.get("token_from_env") or "").strip() if token_env: + _gate_non_rest_env_credential(t, token_env, "auth.token_from_env") _warn_if_env_from_key_unset_or_empty(token_env, "auth.token_from_env") auth["token"] = ( (os.environ.get(token_env) or "").strip() or auth.get("token") or "" ) cs_env = (auth.get("client_secret_from_env") or "").strip() if cs_env: + _gate_non_rest_env_credential(t, cs_env, "auth.client_secret_from_env") _warn_if_env_from_key_unset_or_empty( cs_env, "auth.client_secret_from_env" ) @@ -776,6 +852,7 @@ def _normalize_max_expansion_ratio(val: Any) -> float | None: ) env_cs_key = (t.get("client_secret_from_env") or "").strip() if env_cs_key: + _gate_non_rest_env_credential(t, env_cs_key, "client_secret_from_env") _warn_if_env_from_key_unset_or_empty(env_cs_key, "client_secret_from_env") t["client_secret"] = ( (os.environ.get(env_cs_key) or "").strip() diff --git a/connectors/rest_connector.py b/connectors/rest_connector.py index de6bdec94..6383a9d5f 100644 --- a/connectors/rest_connector.py +++ b/connectors/rest_connector.py @@ -41,7 +41,6 @@ # Credential-bearing REST calls must not target arbitrary public hosts (#1977). # url_guard blocks private/metadata IPs only; compare HubSpot host allowlist (#1607). _REST_AUTH_HOST_TAG = "#1977" -_REST_ENV_VAR_PREFIXES = ("DATA_BOAR_", "REST_API_", "API_") def _host_from_url(url: str) -> str: @@ -107,11 +106,9 @@ def _validate_rest_env_var_names(auth: dict[str, Any]) -> None: env_name = (auth.get(key) or "").strip() if not env_name: continue - if not any(env_name.startswith(prefix) for prefix in _REST_ENV_VAR_PREFIXES): - raise ValueError( - f"{label} must use one of the prefixes " - f"{', '.join(_REST_ENV_VAR_PREFIXES)} ({_REST_AUTH_HOST_TAG})." - ) + from config.loader import assert_credential_env_name + + assert_credential_env_name(env_name, label) def _validate_target_env_var_names(target: dict[str, Any]) -> None: @@ -123,11 +120,9 @@ def _validate_target_env_var_names(target: dict[str, Any]) -> None: env_name = (target.get(key) or "").strip() if not env_name: continue - if not any(env_name.startswith(prefix) for prefix in _REST_ENV_VAR_PREFIXES): - raise ValueError( - f"{label} must use one of the prefixes " - f"{', '.join(_REST_ENV_VAR_PREFIXES)} ({_REST_AUTH_HOST_TAG})." - ) + from config.loader import assert_credential_env_name + + assert_credential_env_name(env_name, label) def _credential_endpoint_urls(target: dict[str, Any]) -> list[tuple[str, str]]: diff --git a/core/findings_sink.py b/core/findings_sink.py index 8a4d45da1..2984ec957 100644 --- a/core/findings_sink.py +++ b/core/findings_sink.py @@ -145,6 +145,9 @@ def redacted_sink_label(cfg: dict[str, Any]) -> str: def _env_or_inline(cfg: dict[str, Any], env_key: str, inline_key: str) -> str: env_name = str(cfg.get(env_key) or "").strip() if env_name: + from config.loader import assert_non_rest_env_credential + + assert_non_rest_env_credential(cfg, env_name, env_key) return (os.environ.get(env_name) or "").strip() return str(cfg.get(inline_key) or "").strip() diff --git a/tests/test_credential_env_allowlist.py b/tests/test_credential_env_allowlist.py new file mode 100644 index 000000000..8487dbaa0 --- /dev/null +++ b/tests/test_credential_env_allowlist.py @@ -0,0 +1,73 @@ +"""Non-REST pass_from_env must reuse the #1977 prefix and host allowlist (#2086).""" + +from __future__ import annotations + +import pytest + +from config.loader import normalize_config +from core.findings_sink import _sqlalchemy_url + + +def _database_target(**extra: object) -> dict: + target: dict = { + "name": "db1", + "type": "database", + "host": "attacker.example", + "pass_from_env": "LAB_PG_PASSWORD", + } + target.update(extra) + return { + "targets": [target], + "api": {}, + "report": {"output_dir": "."}, + } + + +@pytest.mark.parametrize("kind", ["database", "mongodb", "redis", "smb"]) +def test_non_rest_pass_from_env_rejects_non_prefixed_name( + kind: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("LAB_PG_PASSWORD", "db-secret") + data = _database_target(type=kind, allowed_hosts=["attacker.example"]) + with pytest.raises(ValueError, match="pass_from_env.*#1977"): + normalize_config(data) + + +def test_sql_pass_from_env_rejects_host_outside_allowlist( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("API_DB_PASSWORD", "db-secret") + data = _database_target( + pass_from_env="API_DB_PASSWORD", + allowed_hosts=["db.trusted.example"], + ) + with pytest.raises(ValueError, match="allowed_hosts.*#1977"): + normalize_config(data) + + +def test_sql_pass_from_env_copies_only_allowlisted_prefixed_name( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("API_DB_PASSWORD", "db-secret") + data = _database_target( + pass_from_env="API_DB_PASSWORD", + allowed_hosts=["attacker.example"], + ) + out = normalize_config(data) + assert out["targets"][0]["pass"] == "db-secret" + + +def test_findings_sink_pass_from_env_rejects_non_prefixed_name( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LAB_PG_PASSWORD", "sink-secret") + cfg = { + "type": "postgresql", + "host": "attacker.example", + "database": "audit", + "user": "scanner", + "pass_from_env": "LAB_PG_PASSWORD", + "allowed_hosts": ["attacker.example"], + } + with pytest.raises(ValueError, match="pass_from_env.*#1977"): + _sqlalchemy_url(cfg) diff --git a/tests/test_main_validate_config.py b/tests/test_main_validate_config.py index 949705f43..c79932055 100644 --- a/tests/test_main_validate_config.py +++ b/tests/test_main_validate_config.py @@ -137,12 +137,14 @@ def test_mask_env_name_sensitive_vs_non_sensitive() -> None: def test_validate_config_unset_env_var_warns_but_ok(tmp_path): - env_name = "DB_USER_TEST_VALIDATE_CONFIG_UNUSED" + env_name = "API_DB_USER_TEST_VALIDATE_CONFIG_UNUSED" cfg = _base_config( tmp_path, f""" - name: prod-postgres type: database driver: postgresql+psycopg2 + host: db.example + allowed_hosts: ["db.example"] user_from_env: {env_name}""", ) child_env = os.environ.copy() @@ -170,12 +172,14 @@ def test_validate_config_unset_env_var_warns_but_ok(tmp_path): def test_validate_config_masks_sensitive_env_name_in_warning(tmp_path): - env_name = "PG_PASS_TEST_VALIDATE_CONFIG_MASK" + env_name = "API_PG_PASS_TEST_VALIDATE_CONFIG_MASK" cfg = _base_config( tmp_path, f""" - name: prod-postgres type: database driver: postgresql+psycopg2 + host: db.example + allowed_hosts: ["db.example"] pass_from_env: {env_name}""", ) child_env = os.environ.copy() diff --git a/tests/test_security.py b/tests/test_security.py index b1fa00807..0b51cf33f 100644 --- a/tests/test_security.py +++ b/tests/test_security.py @@ -702,16 +702,24 @@ def test_normalize_config_resolves_pass_from_env_and_user_from_env(monkeypatch): """Loader resolves pass_from_env, password_from_env, user_from_env for targets.""" from config.loader import normalize_config - monkeypatch.setenv("DB_PASS", "env-pass") - monkeypatch.setenv("DB_USER", "env-user") + monkeypatch.setenv("API_DB_PASS", "env-pass") + monkeypatch.setenv("API_DB_USER", "env-user") data = { "targets": [ - {"name": "db1", "type": "database", "pass_from_env": "DB_PASS"}, + { + "name": "db1", + "type": "database", + "host": "db.example", + "allowed_hosts": ["db.example"], + "pass_from_env": "API_DB_PASS", + }, { "name": "db2", "type": "database", - "password_from_env": "DB_PASS", - "user_from_env": "DB_USER", + "host": "db.example", + "allowed_hosts": ["db.example"], + "password_from_env": "API_DB_PASS", + "user_from_env": "API_DB_USER", }, ], "api": {}, @@ -788,14 +796,16 @@ def test_normalize_config_warns_when_pass_from_env_unset_then_falls_back(monkeyp from config.loader import normalize_config - monkeypatch.delenv("MISSING_DB_PASS", raising=False) + monkeypatch.delenv("API_MISSING_DB_PASS", raising=False) data = { "targets": [ { "name": "db1", "type": "database", + "host": "db.example", + "allowed_hosts": ["db.example"], "pass": "inline-secret", - "pass_from_env": "MISSING_DB_PASS", + "pass_from_env": "API_MISSING_DB_PASS", } ], "api": {}, @@ -806,7 +816,7 @@ def test_normalize_config_warns_when_pass_from_env_unset_then_falls_back(monkeyp out = normalize_config(data) assert out["targets"][0]["pass"] == "inline-secret" assert any("pass_from_env" in str(w.message) for w in recorded) - assert any("MISSING_DB_PASS" in str(w.message) for w in recorded) + assert any("API_MISSING_DB_PASS" in str(w.message) for w in recorded) # --------------------------------------------------------------------------- From 6e01f20468bc2d1d4d1f626617f48584c4e9a89f Mon Sep 17 00:00:00 2001 From: Fabio Leitao Date: Mon, 19 Oct 2026 11:36:47 -0300 Subject: [PATCH 03/12] fix(security): restrict notification env interpolation (Closes #2087) Notification ${VAR} names must use the #1977 prefixes. public_base_url is display text and no longer expands an environment variable. --- config/loader.py | 18 +++++++- tests/test_database.py | 4 +- tests/test_notification_env_allowlist.py | 53 ++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 4 deletions(-) create mode 100644 tests/test_notification_env_allowlist.py diff --git a/config/loader.py b/config/loader.py index c3615d7a4..be3013cf1 100644 --- a/config/loader.py +++ b/config/loader.py @@ -125,9 +125,14 @@ def _gate_non_rest_env_credential( assert_non_rest_env_credential(target, env_name, label) -def _notification_env_value(val: Any) -> str | None: +def _notification_env_value( + val: Any, *, interpolate: bool = True, label: str = "notification" +) -> str | None: """ Resolve optional notification secrets: inline string or ${ENV_VAR} from the environment. + + ``${VAR}`` names must use the #1977 credential prefixes. ``public_base_url`` + is display text and never expands an environment variable (#2087). """ if val is None: return None @@ -137,7 +142,12 @@ def _notification_env_value(val: Any) -> str | None: if not t: return None if t.startswith("${") and t.endswith("}"): + if not interpolate: + raise ValueError( + f"{label} must not interpolate environment variables (#2087)." + ) var = t[2:-1].strip() + assert_credential_env_name(var, label) return (os.environ.get(var) or "").strip() or None return t @@ -1384,7 +1394,11 @@ def _jh_int(key: str, default: int) -> int: n_raw.get("notify_only_if_high_or_critical", False) ), "notify_on_failure": bool(n_raw.get("notify_on_failure", True)), - "public_base_url": _notification_env_value(n_raw.get("public_base_url")), + "public_base_url": _notification_env_value( + n_raw.get("public_base_url"), + interpolate=False, + label="public_base_url", + ), "dedupe_scan_complete_per_session": bool( n_raw.get("dedupe_scan_complete_per_session", True) ), diff --git a/tests/test_database.py b/tests/test_database.py index 7ffe9dcb9..2ac496bef 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -487,13 +487,13 @@ def test_normalize_config_notifications_defaults(): def test_normalize_config_notifications_env(monkeypatch): - monkeypatch.setenv("TEST_NOTIFY_SLACK", "https://hooks.example/test") + monkeypatch.setenv("API_TEST_NOTIFY_SLACK", "https://hooks.example/test") out = normalize_config( { "targets": [], "notifications": { "enabled": True, - "operator": {"slack_webhook_url": "${TEST_NOTIFY_SLACK}"}, + "operator": {"slack_webhook_url": "${API_TEST_NOTIFY_SLACK}"}, }, } ) diff --git a/tests/test_notification_env_allowlist.py b/tests/test_notification_env_allowlist.py new file mode 100644 index 000000000..0b4e627b8 --- /dev/null +++ b/tests/test_notification_env_allowlist.py @@ -0,0 +1,53 @@ +"""Notification ${VAR} interpolation uses the #1977 prefixes (#2087).""" + +from __future__ import annotations + +import pytest + +from config.loader import normalize_config + + +def test_notification_env_rejects_non_prefixed_name( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LAB_NOTIFY_SLACK", "https://hooks.attacker.example/hook") + with pytest.raises(ValueError, match="#1977"): + normalize_config( + { + "targets": [], + "notifications": { + "operator": {"slack_webhook_url": "${LAB_NOTIFY_SLACK}"}, + }, + } + ) + + +def test_public_base_url_does_not_expand_env( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("API_PUBLIC_SECRET", "super-secret") + with pytest.raises(ValueError, match="public_base_url"): + normalize_config( + { + "targets": [], + "notifications": {"public_base_url": "${API_PUBLIC_SECRET}"}, + } + ) + + +def test_notification_env_expands_prefixed_name( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("API_NOTIFY_SLACK", "https://hooks.example/test") + out = normalize_config( + { + "targets": [], + "notifications": { + "operator": {"slack_webhook_url": "${API_NOTIFY_SLACK}"}, + }, + } + ) + assert ( + out["notifications"]["operator"]["slack_webhook_url"] + == "https://hooks.example/test" + ) From 8d1109b5380b22d51083195ab7fec5f9099449f9 Mon Sep 17 00:00:00 2001 From: Fabio Leitao Date: Mon, 19 Oct 2026 11:51:01 -0300 Subject: [PATCH 04/12] fix(security): stop scan webhooks from following redirects (Closes #2088) HTTPS checks stay on the configured URL. The opener refuses redirects so a 302 cannot move the POST onto plain HTTP or an internal address. --- tests/test_notify.py | 56 +++++++++++++++++++++++++++++++++++++++----- utils/notify.py | 20 +++++++++++++--- 2 files changed, 67 insertions(+), 9 deletions(-) diff --git a/tests/test_notify.py b/tests/test_notify.py index 8699f21bf..9ffb43ec9 100644 --- a/tests/test_notify.py +++ b/tests/test_notify.py @@ -3,7 +3,9 @@ from __future__ import annotations import json +import threading import urllib.error +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import pytest @@ -113,7 +115,7 @@ def __exit__(self, *a): return R() - monkeypatch.setattr("urllib.request.urlopen", fake_urlopen) + monkeypatch.setattr("utils.notify._open_webhook", fake_urlopen) n = { "operator": { "slack_webhook_url": "https://hooks.slack.example/services/XXX", @@ -154,7 +156,7 @@ def __exit__(self, *a): return R() - monkeypatch.setattr("urllib.request.urlopen", fake_urlopen) + monkeypatch.setattr("utils.notify._open_webhook", fake_urlopen) monkeypatch.setattr("utils.notify.time.sleep", lambda *_a, **_k: None) send_slack_webhook("https://hooks.example/hook", "hi") assert calls["n"] == 2 @@ -178,7 +180,7 @@ def __exit__(self, *a): return R() - monkeypatch.setattr("urllib.request.urlopen", fake_urlopen) + monkeypatch.setattr("utils.notify._open_webhook", fake_urlopen) n = { "operator": { "channels": [ @@ -212,7 +214,7 @@ def __exit__(self, *a): return R() - monkeypatch.setattr("urllib.request.urlopen", fake_urlopen) + monkeypatch.setattr("utils.notify._open_webhook", fake_urlopen) class FakeDB: def get_session_scan_summary_for_notification(self, sid): @@ -262,7 +264,7 @@ def __exit__(self, *a): return R() - monkeypatch.setattr("urllib.request.urlopen", fake_urlopen) + monkeypatch.setattr("utils.notify._open_webhook", fake_urlopen) cfg = { "notifications": { "enabled": True, @@ -301,7 +303,7 @@ def __exit__(self, *a): return R() - monkeypatch.setattr("urllib.request.urlopen", fake_urlopen) + monkeypatch.setattr("utils.notify._open_webhook", fake_urlopen) class FakeDB: def get_session_scan_summary_for_notification(self, sid): @@ -334,3 +336,45 @@ def get_session_scan_summary_for_notification(self, sid): } notify_scan_complete_sync(cfg, FakeDB(), "sess_tenant_1") assert calls["n"] == 2 + + +def test_post_json_does_not_follow_http_redirect( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """#2088: a 302 from an HTTPS webhook must not be fetched.""" + from utils.notify import _post_json + + hits: list[str] = [] + + class Handler(BaseHTTPRequestHandler): + def log_message(self, fmt: str, *args: object) -> None: + return + + def do_POST(self) -> None: + hits.append(self.path) + self.send_response(302) + location = f"http://127.0.0.1:{self.server.server_address[1]}/landed" + self.send_header("Location", location) + self.end_headers() + + def do_GET(self) -> None: + hits.append(self.path) + self.send_response(200) + self.end_headers() + self.wfile.write(b"ok") + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + monkeypatch.setattr("utils.notify._require_https_webhook_url", lambda _url: None) + try: + with pytest.raises(urllib.error.HTTPError): + _post_json( + f"http://127.0.0.1:{server.server_address[1]}/hook", + {"text": "x"}, + timeout_s=2, + ) + finally: + server.shutdown() + server.server_close() + assert hits == ["/hook"] diff --git a/utils/notify.py b/utils/notify.py index 01933acad..4c1072a53 100644 --- a/utils/notify.py +++ b/utils/notify.py @@ -4,7 +4,7 @@ Default: disabled in config. Secrets come from env or config; never log full URLs. See docs/USAGE.md and PLAN_NOTIFICATIONS_OFFBAND_AND_SCAN_COMPLETE.md. -``_post_json`` / ``_post_form`` require an ``https`` URL before ``urlopen`` (Bandit B310). +``_post_json`` / ``_post_form`` require an ``https`` URL and do not follow redirects (Bandit B310, #2088). Operator webhooks and the Telegram API URL are HTTPS-only paths. """ @@ -30,6 +30,20 @@ _MAX_DEDUPE_SESSIONS = 10_000 +class _RefuseRedirects(urllib.request.HTTPRedirectHandler): + """Do not follow webhook redirects (#2088). A 302 stays an HTTP error.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): # noqa: ANN001 + return None + + +_WEBHOOK_OPENER = urllib.request.build_opener(_RefuseRedirects) + + +def _open_webhook(req: urllib.request.Request, timeout: float): + return _WEBHOOK_OPENER.open(req, timeout=timeout) + + def _require_https_webhook_url(url: str) -> None: """Reject non-HTTPS URLs before urllib opens the request (Bandit B310 contract).""" parsed = urllib.parse.urlparse(url) @@ -53,7 +67,7 @@ def _post_json(url: str, payload: dict[str, Any], timeout_s: float = 15.0) -> No ) try: # B310: HTTPS-only URL enforced above via _require_https_webhook_url. - with urllib.request.urlopen(req, timeout=timeout_s) as resp: # nosec B310 + with _open_webhook(req, timeout_s) as resp: # nosec B310 resp.read() return except urllib.error.HTTPError as e: @@ -80,7 +94,7 @@ def _post_form(url: str, body: dict[str, str], timeout_s: float = 15.0) -> None: ) try: # B310: HTTPS-only URL enforced above via _require_https_webhook_url. - with urllib.request.urlopen(req, timeout=timeout_s) as resp: # nosec B310 + with _open_webhook(req, timeout_s) as resp: # nosec B310 resp.read() return except urllib.error.HTTPError as e: From 51c5d83181571806fcfe875e58b4316292cd09cf Mon Sep 17 00:00:00 2001 From: Fabio Leitao Date: Mon, 19 Oct 2026 12:01:38 -0300 Subject: [PATCH 05/12] fix(security): make the non-REST credential gate satisfiable per connector (Refs #2086) Derive the credential destination from an explicit per-type key so SharePoint (site_url), Snowflake (account or .snowflakecomputing.com), WebDAV, NFS, and the host-based engines can satisfy allowed_hosts. HubSpot and the default Power BI token host stay on their existing vendor allowlists. allowed_hosts is read from the same YAML the config author writes, same limit as #1977. Against that author only the env-name prefix is an independent control; the host list is a brake, not a second authority. --- config/loader.py | 117 ++++++++++++-- tests/test_credential_env_allowlist.py | 204 +++++++++++++++++++++++++ 2 files changed, 306 insertions(+), 15 deletions(-) diff --git a/config/loader.py b/config/loader.py index be3013cf1..4c9edd64c 100644 --- a/config/loader.py +++ b/config/loader.py @@ -82,25 +82,112 @@ def _explicit_allowed_hosts(cfg: dict[str, Any]) -> set[str]: return {str(host).lower().strip() for host in raw if str(host).strip()} -def _destination_host(cfg: dict[str, Any]) -> str: - host = str(cfg.get("host") or "").strip().lower() - if host: - return host - for key in ("base_url", "url"): - raw = str(cfg.get(key) or "").strip() - if not raw: - continue - parsed = (urlparse(raw).hostname or "").lower() - if parsed: - return parsed - return "" +def _hostname_of(value: str) -> str: + raw = value.strip() + if not raw: + return "" + if "://" in raw: + return (urlparse(raw).hostname or "").lower() + return raw.split("/")[0].split(":")[0].lower() + + +def _credential_target_kind(cfg: dict[str, Any]) -> str: + kind = str(cfg.get("type") or "").strip().lower() + if kind != "database": + return kind + return str(cfg.get("driver") or "").split("+")[0].strip().lower() or "database" + + +# Explicit destination key per non-REST type (#2086). Not a fallback chain. +# Kinds absent here fail closed when an env credential is set. +_HOST_DESTINATION_KINDS = frozenset( + { + "database", + "postgresql", + "postgres", + "mysql", + "mariadb", + "mssql", + "sqlserver", + "oracle", + "mongodb", + "mongo", + "redis", + "smb", + "cifs", + } +) +# Secret is posted only to a host the connector already allowlists. +_VENDOR_FIXED_SECRET_KINDS = frozenset({"hubspot", "powerbi"}) + + +def _credential_destination_identities(cfg: dict[str, Any]) -> set[str] | None: + """Identities that must intersect ``allowed_hosts``. + + ``None`` means a vendor allowlist already binds where the secret is sent + (#1607 HubSpot, #2007 Power BI default token host). An empty set fails closed. + """ + kind = _credential_target_kind(cfg) + auth = cfg.get("auth") if isinstance(cfg.get("auth"), dict) else {} + token_url = str(auth.get("token_url") or "").strip() + + if kind == "hubspot": + return None + if kind == "powerbi": + if not token_url: + return None + host = _hostname_of(token_url) + return {host} if host else set() + if kind in {"dataverse", "powerapps"}: + org = str(cfg.get("org_url") or cfg.get("environment_url") or "") + idents = {host} if (host := _hostname_of(org)) else set() + if token_url: + token_host = _hostname_of(token_url) + if not token_host: + return set() + idents.add(token_host) + return idents + if kind == "snowflake": + account = str(cfg.get("account") or "").strip().lower() + if not account: + return set() + derived = ( + account + if account.endswith(".snowflakecomputing.com") + else f"{account}.snowflakecomputing.com" + ) + return {account, derived} + if kind == "sharepoint": + host = _hostname_of(str(cfg.get("site_url") or cfg.get("base_url") or "")) + return {host} if host else set() + if kind == "webdav": + host = _hostname_of( + str(cfg.get("base_url") or cfg.get("url") or cfg.get("host") or "") + ) + return {host} if host else set() + if kind == "nfs": + host = str(cfg.get("host") or cfg.get("server") or "").strip().lower() + return {host} if host else set() + if kind in _HOST_DESTINATION_KINDS: + host = str(cfg.get("host") or "").strip().lower() + if not host: + host = _hostname_of(str(cfg.get("url") or cfg.get("base_url") or "")) + return {host} if host else set() + return set() def assert_explicit_destination_allowlisted(cfg: dict[str, Any], label: str) -> None: - """Fail closed unless the credential destination is explicitly allowlisted.""" - host = _destination_host(cfg) + """Fail closed unless the credential destination is explicitly allowlisted. + + ``allowed_hosts`` comes from the same YAML the config author writes (same + limit as #1977). Against that author only the env-name prefix is an + independent control; the host list is a brake, not a second authority. + """ + identities = _credential_destination_identities(cfg) + if identities is None: + return allowed = _explicit_allowed_hosts(cfg) - if host and host in allowed: + if identities and identities & allowed: return raise ValueError( f"{label} requires an explicit allowed_hosts entry " diff --git a/tests/test_credential_env_allowlist.py b/tests/test_credential_env_allowlist.py index 8487dbaa0..2259b486b 100644 --- a/tests/test_credential_env_allowlist.py +++ b/tests/test_credential_env_allowlist.py @@ -71,3 +71,207 @@ def test_findings_sink_pass_from_env_rejects_non_prefixed_name( } with pytest.raises(ValueError, match="pass_from_env.*#1977"): _sqlalchemy_url(cfg) + + +def test_sharepoint_pass_from_env_accepts_site_host( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("API_SP_PASSWORD", "sp-secret") + data = { + "targets": [ + { + "name": "sp", + "type": "sharepoint", + "site_url": "https://sharepoint.example/sites/hr", + "pass_from_env": "API_SP_PASSWORD", + "allowed_hosts": ["sharepoint.example"], + } + ], + "api": {}, + "report": {"output_dir": "."}, + } + out = normalize_config(data) + assert out["targets"][0]["pass"] == "sp-secret" + + +def test_snowflake_pass_from_env_accepts_account_or_derived_host( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("API_SF_PASSWORD", "sf-secret") + base = { + "name": "sf", + "type": "snowflake", + "account": "xy12345.us-east-1", + "user": "loader", + "pass_from_env": "API_SF_PASSWORD", + } + for allowed in ( + ["xy12345.us-east-1"], + ["xy12345.us-east-1.snowflakecomputing.com"], + ): + data = { + "targets": [{**base, "allowed_hosts": allowed}], + "api": {}, + "report": {"output_dir": "."}, + } + out = normalize_config(data) + assert out["targets"][0]["pass"] == "sf-secret" + + +@pytest.mark.parametrize( + ("kind", "fields", "allowed"), + [ + ("database", {"host": "db.example"}, ["db.example"]), + ("mongodb", {"host": "mongo.example"}, ["mongo.example"]), + ("redis", {"host": "redis.example"}, ["redis.example"]), + ("smb", {"host": "files.example"}, ["files.example"]), + ("cifs", {"host": "files.example"}, ["files.example"]), + ("nfs", {"server": "nas.example"}, ["nas.example"]), + ( + "sharepoint", + {"site_url": "https://sharepoint.example/sites/hr"}, + ["sharepoint.example"], + ), + ( + "webdav", + {"base_url": "https://dav.example/remote.php"}, + ["dav.example"], + ), + ( + "snowflake", + {"account": "xy12345.us-east-1"}, + ["xy12345.us-east-1.snowflakecomputing.com"], + ), + ( + "dataverse", + {"org_url": "https://org.crm.dynamics.com"}, + ["org.crm.dynamics.com"], + ), + ( + "powerapps", + {"environment_url": "https://org.crm.dynamics.com"}, + ["org.crm.dynamics.com"], + ), + ], +) +def test_non_rest_env_credential_copies_when_destination_allowlisted( + kind: str, fields: dict, allowed: list[str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("API_TARGET_SECRET", "copied-secret") + target = { + "name": kind, + "type": kind, + "pass_from_env": "API_TARGET_SECRET", + "allowed_hosts": allowed, + **fields, + } + out = normalize_config( + {"targets": [target], "api": {}, "report": {"output_dir": "."}} + ) + assert out["targets"][0]["pass"] == "copied-secret" + + +@pytest.mark.parametrize( + ("kind", "fields"), + [ + ("database", {"host": "db.example"}), + ("mongodb", {"host": "mongo.example"}), + ("redis", {"host": "redis.example"}), + ("smb", {"host": "files.example"}), + ("cifs", {"host": "files.example"}), + ("nfs", {"server": "nas.example"}), + ("webdav", {"base_url": "https://dav.example/remote.php"}), + ("sharepoint", {"site_url": "https://sharepoint.example/sites/hr"}), + ("snowflake", {"account": "xy12345.us-east-1"}), + ("dataverse", {"org_url": "https://org.crm.dynamics.com"}), + ], +) +def test_non_rest_env_credential_rejects_other_host( + kind: str, fields: dict, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("API_TARGET_SECRET", "copied-secret") + target = { + "name": kind, + "type": kind, + "pass_from_env": "API_TARGET_SECRET", + "allowed_hosts": ["other.example"], + **fields, + } + with pytest.raises(ValueError, match="allowed_hosts.*#1977"): + normalize_config( + {"targets": [target], "api": {}, "report": {"output_dir": "."}} + ) + + +def test_hubspot_token_from_env_keeps_vendor_host_allowlist( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("API_HUBSPOT_TOKEN", "pat-secret") + out = normalize_config( + { + "targets": [ + { + "name": "crm", + "type": "hubspot", + "auth": {"token_from_env": "API_HUBSPOT_TOKEN"}, + } + ], + "api": {}, + "report": {"output_dir": "."}, + } + ) + assert out["targets"][0]["auth"]["token"] == "pat-secret" + + +def test_findings_sink_pass_from_env_rejects_other_host( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("API_SINK_PASSWORD", "sink-secret") + cfg = { + "type": "postgresql", + "host": "attacker.example", + "database": "audit", + "user": "scanner", + "pass_from_env": "API_SINK_PASSWORD", + "allowed_hosts": ["db.trusted.example"], + } + with pytest.raises(ValueError, match="allowed_hosts.*#1977"): + _sqlalchemy_url(cfg) + + +def test_findings_sink_pass_from_env_accepts_allowlisted_host( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("API_SINK_PASSWORD", "sink-secret") + cfg = { + "type": "postgresql", + "host": "db.trusted.example", + "database": "audit", + "user": "scanner", + "pass_from_env": "API_SINK_PASSWORD", + "allowed_hosts": ["db.trusted.example"], + } + url = _sqlalchemy_url(cfg) + assert "sink-secret" in url + assert "db.trusted.example" in url + + +def test_powerbi_client_secret_from_env_uses_vendor_token_host( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("API_PBI_SECRET", "pbi-secret") + data = { + "targets": [ + { + "name": "pbi", + "type": "powerbi", + "tenant_id": "tenant", + "client_id": "client", + "client_secret_from_env": "API_PBI_SECRET", + } + ], + "api": {}, + "report": {"output_dir": "."}, + } + out = normalize_config(data) + assert out["targets"][0]["client_secret"] == "pbi-secret" From 1e97d20aacaf396833bffee51c733202514b1c23 Mon Sep 17 00:00:00 2001 From: Fabio Leitao Date: Mon, 19 Oct 2026 12:12:44 -0300 Subject: [PATCH 06/12] docs: migrate shipped examples and document the credential env gate (Refs #2086, #2087) Rename distributed pass_from_env examples to DATA_BOAR_* and list each destination in allowed_hosts so they load. Document both requirements and the residual risk that allowed_hosts is same-YAML as the config author. Note the #2087 public_base_url break in the changelog. --- CHANGELOG.md | 5 ++- .../compat-matrix-config-ref/config-db.yaml | 18 +++++++---- deploy/lab-smoke-stack/docker-compose.yml | 14 ++++---- deploy/lab-smoke-stack/env.example | 8 ++--- deploy/samples/config.starter-lgpd-eval.yaml | 3 +- docs/SECURITY.md | 1 + docs/SECURITY.pt_BR.md | 1 + docs/TROUBLESHOOTING_CREDENTIALS_AND_AUTH.md | 2 +- ...UBLESHOOTING_CREDENTIALS_AND_AUTH.pt_BR.md | 2 +- docs/USAGE.md | 32 +++++++++++++------ docs/USAGE.pt_BR.md | 16 +++++++--- docs/ops/OPERATOR_CREDENTIALS_FROM_ENV.md | 4 ++- .../OPERATOR_CREDENTIALS_FROM_ENV.pt_BR.md | 4 ++- docs/ops/SCOPE_IMPORT_QUICKSTART.md | 2 +- docs/ops/SCOPE_IMPORT_QUICKSTART.pt_BR.md | 2 +- tests/test_shipped_credential_examples.py | 30 +++++++++++++++++ 16 files changed, 106 insertions(+), 38 deletions(-) create mode 100644 tests/test_shipped_credential_examples.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a20c16818..bea849687 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,10 @@ Human-readable summary of user-facing changes. **Detailed release notes:** [docs ## Unreleased (`main`) -Empty while the working tree carries **`1.8.0-rc`** (see section below). +Migration notes on the **`1.8.0-rc`** working line (no new semver cut): + +- **Credential env names (#2086):** `pass_from_env`, `password_from_env`, `user_from_env`, `token_from_env`, and `client_secret_from_env` fail at load when the name is outside the `DATA_BOAR_` / `REST_API_` / `API_` prefixes, or when the destination is missing from that target’s `allowed_hosts`. Shipped examples now use prefixed names (`DATA_BOAR_DEMO_DB_PASSWORD`, `DATA_BOAR_LAB_*`). +- **Notification `public_base_url` (#2087):** `public_base_url: ${VAR}` raises `ValueError` and is not expanded. Other notification `${VAR}` values still expand only for the same prefixes. --- diff --git a/deploy/compat-matrix-config-ref/config-db.yaml b/deploy/compat-matrix-config-ref/config-db.yaml index 8c04c5722..d218dba82 100644 --- a/deploy/compat-matrix-config-ref/config-db.yaml +++ b/deploy/compat-matrix-config-ref/config-db.yaml @@ -27,9 +27,10 @@ targets: type: database driver: postgresql+psycopg2 host: REPLACE_WITH_LAB_HOST_IP + allowed_hosts: ["REPLACE_WITH_LAB_HOST_IP"] port: 55432 user: lab_smoke - pass_from_env: LAB_PG_PASSWORD + pass_from_env: DATA_BOAR_LAB_PG_PASSWORD database: lab_smoke_pg connect_timeout_seconds: 25 read_timeout_seconds: 90 @@ -38,9 +39,10 @@ targets: type: database driver: mysql+pymysql host: REPLACE_WITH_LAB_HOST_IP + allowed_hosts: ["REPLACE_WITH_LAB_HOST_IP"] port: 33306 user: lab_smoke - pass_from_env: LAB_MY_PASSWORD + pass_from_env: DATA_BOAR_LAB_MY_PASSWORD database: lab_smoke_my connect_timeout_seconds: 25 read_timeout_seconds: 90 @@ -49,9 +51,10 @@ targets: type: database driver: mssql+pymssql host: REPLACE_WITH_LAB_HOST_IP + allowed_hosts: ["REPLACE_WITH_LAB_HOST_IP"] port: 14333 user: sa - pass_from_env: LAB_MSSQL_SA_PASSWORD + pass_from_env: DATA_BOAR_LAB_MSSQL_SA_PASSWORD database: lab_smoke_mssql connect_timeout_seconds: 25 read_timeout_seconds: 90 @@ -60,9 +63,10 @@ targets: type: database driver: oracle+oracledb host: REPLACE_WITH_LAB_HOST_IP + allowed_hosts: ["REPLACE_WITH_LAB_HOST_IP"] port: 15211 user: lab_smoke - pass_from_env: LAB_ORACLE_PASSWORD + pass_from_env: DATA_BOAR_LAB_ORACLE_PASSWORD database: XEPDB1 connect_timeout_seconds: 25 read_timeout_seconds: 90 @@ -73,16 +77,18 @@ targets: host: REPLACE_WITH_LAB_HOST_IP port: 56379 # Redis lab image has no auth by default; leave pass_from_env unset unless - # you enable requirepass. When set, export LAB_REDIS_PASSWORD. + # you enable requirepass. When set, export DATA_BOAR_LAB_REDIS_PASSWORD + # and add allowed_hosts for this host. # Optional remote latency lane (#1237 / #1337) — fill from operator vault, not git: # - name: Cloud_Postgres_Remote # type: database # driver: postgresql+psycopg2 # host: REPLACE_WITH_REMOTE_PG_HOST + # allowed_hosts: ["REPLACE_WITH_REMOTE_PG_HOST"] # port: 6543 # user: REPLACE_WITH_REMOTE_PG_USER - # pass_from_env: DATABOAR_REMOTE_PG_PASS + # pass_from_env: DATA_BOAR_REMOTE_PG_PASS # database: postgres # connect_timeout_seconds: 25 # read_timeout_seconds: 90 diff --git a/deploy/lab-smoke-stack/docker-compose.yml b/deploy/lab-smoke-stack/docker-compose.yml index 88172e310..c750a6a5d 100644 --- a/deploy/lab-smoke-stack/docker-compose.yml +++ b/deploy/lab-smoke-stack/docker-compose.yml @@ -20,7 +20,7 @@ services: image: postgres:16-alpine environment: POSTGRES_USER: ${LAB_PG_USER:-lab_smoke} - POSTGRES_PASSWORD: ${LAB_PG_PASSWORD:-lab_smoke_change_me} + POSTGRES_PASSWORD: ${DATA_BOAR_LAB_PG_PASSWORD:-lab_smoke_change_me} POSTGRES_DB: ${LAB_PG_DATABASE:-lab_smoke_pg} ports: - "${LAB_PG_PORT:-55432}:5432" @@ -39,7 +39,7 @@ services: MARIADB_ROOT_PASSWORD: ${LAB_MY_ROOT_PASSWORD:-lab_smoke_root_change_me} MARIADB_DATABASE: ${LAB_MY_DATABASE:-lab_smoke_my} MARIADB_USER: ${LAB_MY_USER:-lab_smoke} - MARIADB_PASSWORD: ${LAB_MY_PASSWORD:-lab_smoke_change_me} + MARIADB_PASSWORD: ${DATA_BOAR_LAB_MY_PASSWORD:-lab_smoke_change_me} ports: - "${LAB_MY_PORT:-33306}:3306" volumes: @@ -55,7 +55,7 @@ services: image: mcr.microsoft.com/mssql/server:2022-latest environment: ACCEPT_EULA: "Y" - MSSQL_SA_PASSWORD: ${LAB_MSSQL_SA_PASSWORD:-Lab_Smoke_Change_Me1!} + MSSQL_SA_PASSWORD: ${DATA_BOAR_LAB_MSSQL_SA_PASSWORD:-Lab_Smoke_Change_Me1!} MSSQL_PID: Developer ports: - "${LAB_MSSQL_PORT:-14333}:1433" @@ -78,7 +78,7 @@ services: environment: MSSQL_HOST: lab-mssql MSSQL_PORT: "1433" - MSSQL_SA_PASSWORD: ${LAB_MSSQL_SA_PASSWORD:-Lab_Smoke_Change_Me1!} + MSSQL_SA_PASSWORD: ${DATA_BOAR_LAB_MSSQL_SA_PASSWORD:-Lab_Smoke_Change_Me1!} volumes: - ./init/mssql:/seed:ro entrypoint: ["/bin/bash", "/seed/apply_lab_smoke.sh"] @@ -87,9 +87,9 @@ services: lab-oracle: image: gvenzl/oracle-xe:21-slim environment: - ORACLE_PASSWORD: ${LAB_ORACLE_PASSWORD:-lab_smoke_oracle_change_me} + ORACLE_PASSWORD: ${DATA_BOAR_LAB_ORACLE_PASSWORD:-lab_smoke_oracle_change_me} APP_USER: ${LAB_ORACLE_USER:-lab_smoke} - APP_USER_PASSWORD: ${LAB_ORACLE_PASSWORD:-lab_smoke_change_me} + APP_USER_PASSWORD: ${DATA_BOAR_LAB_ORACLE_PASSWORD:-lab_smoke_change_me} ports: - "${LAB_ORACLE_PORT:-15211}:1521" healthcheck: @@ -109,7 +109,7 @@ services: ORACLE_PORT: "1521" ORACLE_SERVICE: XEPDB1 APP_USER: ${LAB_ORACLE_USER:-lab_smoke} - APP_USER_PASSWORD: ${LAB_ORACLE_PASSWORD:-lab_smoke_change_me} + APP_USER_PASSWORD: ${DATA_BOAR_LAB_ORACLE_PASSWORD:-lab_smoke_change_me} volumes: - ./init/oracle:/seed:ro entrypoint: ["/bin/bash", "/seed/apply_lab_smoke_pdb.sh"] diff --git a/deploy/lab-smoke-stack/env.example b/deploy/lab-smoke-stack/env.example index d1c2bf822..0a0c99d0f 100644 --- a/deploy/lab-smoke-stack/env.example +++ b/deploy/lab-smoke-stack/env.example @@ -2,23 +2,23 @@ # Never commit .env with real secrets. LAB_PG_USER=lab_smoke -LAB_PG_PASSWORD=lab_smoke_change_me +DATA_BOAR_LAB_PG_PASSWORD=lab_smoke_change_me LAB_PG_DATABASE=lab_smoke_pg LAB_PG_PORT=55432 LAB_MY_ROOT_PASSWORD=lab_smoke_root_change_me LAB_MY_DATABASE=lab_smoke_my LAB_MY_USER=lab_smoke -LAB_MY_PASSWORD=lab_smoke_change_me +DATA_BOAR_LAB_MY_PASSWORD=lab_smoke_change_me LAB_MY_PORT=33306 # MSSQL (SA password must meet SQL Server complexity rules) -LAB_MSSQL_SA_PASSWORD=Lab_Smoke_Change_Me1! +DATA_BOAR_LAB_MSSQL_SA_PASSWORD=Lab_Smoke_Change_Me1! LAB_MSSQL_PORT=14333 # Oracle XE (gvenzl/oracle-xe; APP_USER schema) LAB_ORACLE_USER=lab_smoke -LAB_ORACLE_PASSWORD=lab_smoke_oracle_change_me +DATA_BOAR_LAB_ORACLE_PASSWORD=lab_smoke_oracle_change_me LAB_ORACLE_PORT=15211 # Redis diff --git a/deploy/samples/config.starter-lgpd-eval.yaml b/deploy/samples/config.starter-lgpd-eval.yaml index 7162c6902..3d004d10a 100644 --- a/deploy/samples/config.starter-lgpd-eval.yaml +++ b/deploy/samples/config.starter-lgpd-eval.yaml @@ -27,10 +27,11 @@ targets: type: database driver: postgresql+psycopg2 host: 192.0.2.10 + allowed_hosts: ["192.0.2.10"] port: 5432 database: demo_app user: demo_reader - pass_from_env: DEMO_DB_PASSWORD + pass_from_env: DATA_BOAR_DEMO_DB_PASSWORD # --- file_scan.extensions (filesystem / SMB / WebDAV / SharePoint) --- # This sample keeps a short LGPD-style pilot list below. For the full built-in suffix diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 8aa74bf2a..1dc3f45b8 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -34,6 +34,7 @@ The table above is guarded by **`tests/test_security.py`** plus the named module - **Logging policy:** API keys, passwords, and connection strings are never written to logs. **`get_logger()`** always attaches **`SanitizeLogFilter`** (#1722 / ADR-0036) so every record runs **`sanitize_log_text`**. Persistence still uses **`save_failure`**. See **`utils/logger.py`** and **`tests/test_logger_pii_filter.py`**. - **Audit log vs report:** Text logs (`audit_YYYYMMDD.log`) use **sanitized, unique** per-target labels (`audit_log_name`) and **redacted** filesystem locations as described in **docs/USAGE.md**. Do not treat the log’s third column as a verbatim OS path when it shows `name?hex`; use the database/report row for the exact stored path. If the `target?hash` form for out-of-root files is too opaque in your environment, a future option could switch to **filename-only** plus hash—file an issue if you need that operational toggle. - **Passwords with special characters:** You can safely use database or MongoDB passwords that contain `@`, `:`, `/`, or `#`. The application encodes them when building connection URLs. No extra configuration is required. +- **Credential env names (#2086):** `pass_from_env`, `password_from_env`, `user_from_env`, `token_from_env`, and `client_secret_from_env` must use a `DATA_BOAR_` / `REST_API_` / `API_` name, and the destination must be listed in that target’s `allowed_hosts` (vendor-fixed hosts such as HubSpot and the default Power BI token URL keep their own allowlist). `allowed_hosts` is read from the **same YAML the config author writes**, same limit as #1977. Against that author, only the **prefix** is an independent control; the host list is a brake, not a second authority. An operator-controlled allowlist is not shipped. - **Protecting the config in the UI:** The **Configuration** page (GET `/config`) shows and allows editing the main config file, which may contain credentials. If the API is exposed to untrusted users or networks, set **`api.require_api_key: true`** and provide a strong API key (or `api.api_key_from_env`). Then only requests that send a valid **X-API-Key** or **Authorization: Bearer** can access `/config`. - **API key vs /health:** **`GET /health`** stays **unauthenticated** (liveness for orchestrators). Other routes require the key when enabled. **401** = wrong/missing key; **503** = `require_api_key` true but no key resolved; **`main.py --web` exits 2** before listen in that case. See root **[SECURITY.md](../SECURITY.md)** § Optional API key, **[USAGE.md](USAGE.md)** (Authentication), and **[SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.md](ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.md)** ([pt-BR](ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.pt_BR.md)). - **Host header vs bind:** Clients must send a `Host` that is in `trusted_api_hosts()`. Add the public DNS name via **`api.host`** or **`api.trusted_hosts`**. Binding `0.0.0.0` is not enough. Untrusted Host is **HTTP 400** (not 401). Restart `--web` after YAML changes. diff --git a/docs/SECURITY.pt_BR.md b/docs/SECURITY.pt_BR.md index 79564ef37..f3568ff63 100644 --- a/docs/SECURITY.pt_BR.md +++ b/docs/SECURITY.pt_BR.md @@ -34,6 +34,7 @@ A tabela acima é coberta por **`tests/test_security.py`** e pelos módulos nome - **Política de logging:** Chaves de API, senhas e strings de conexão nunca são gravadas em log. **`get_logger()`** aplica **`SanitizeLogFilter`** (#1722 / ADR-0036) em todo registro. Persistência continua em **`save_failure`**. Veja **`utils/logger.py`** e **`tests/test_logger_pii_filter.py`**. - **Audit log vs relatório:** Os logs de texto (`audit_YYYYMMDD.log`) usam rótulos **únicos e higienizados** por target (`audit_log_name`) e localizações de filesystem **redigidas**, conforme **docs/USAGE.pt_BR.md**. Não interprete a terceira coluna do log como caminho literal do SO quando aparecer `nome?hex`; use a linha no banco/relatório para o caminho armazenado. Se a forma `target?hash` para arquivos fora da raiz for pouco clara na operação, uma opção futura poderia ser **só nome de arquivo** mais hash — abra uma issue se precisar desse toggle. - **Senhas com caracteres especiais:** Você pode usar com segurança senhas de banco ou MongoDB que contenham `@`, `:`, `/` ou `#`. A aplicação as codifica ao montar as URLs de conexão. Nenhuma configuração extra é necessária. +- **Nomes de env de credencial (#2086):** `pass_from_env`, `password_from_env`, `user_from_env`, `token_from_env` e `client_secret_from_env` precisam de um nome com prefixo `DATA_BOAR_` / `REST_API_` / `API_`, e o destino precisa estar em `allowed_hosts` daquele alvo (hosts fixos de fornecedor, como HubSpot e a URL de token padrão do Power BI, seguem a allowlist própria). `allowed_hosts` é lido do **mesmo YAML que o autor do config escreve**, o mesmo limite do #1977. Contra esse autor, só o **prefixo** é controle independente; a lista de host é um freio, não uma segunda autoridade. Não há allowlist controlada pelo operador nesta versão. - **Proteger o config na interface:** A página **Configuração** (GET `/config`) exibe e permite editar o arquivo de config principal. **GET `/config` sempre redige valores secretos** (senhas, chave de API, tokens, client_secret, etc.) antes de enviar o YAML ao navegador, para que a interface nunca exiba nem transmita segredos em texto claro; ao salvar, placeholders são mesclados com o arquivo atual para não sobrescrever os segredos reais. Se a API estiver exposta a usuários ou redes não confiáveis, defina **`api.require_api_key: true`** e forneça uma chave de API forte (ou `api.api_key_from_env`). - **Chave de API vs /health:** **`GET /health`** permanece **sem autenticação** (liveness para orquestradores). Demais rotas exigem a chave quando habilitado. **401** = chave ausente ou inválida; **503** = `require_api_key` true porém chave não resolvida; **`main.py --web` encerra com código 2** antes de escutar nesse caso. Veja **[SECURITY.pt_BR.md](../SECURITY.pt_BR.md)** (chave opcional), **[USAGE.pt_BR.md](USAGE.pt_BR.md)** (Autenticação) e **[SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.pt_BR.md](ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.pt_BR.md)** ([EN](ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.md)). - **Host vs bind:** o cliente precisa enviar um `Host` que esteja em `trusted_api_hosts()`. Inclua o DNS público via **`api.host`** ou **`api.trusted_hosts`**. Bind `0.0.0.0` não basta. Host não confiável é **HTTP 400** (não 401). Reinicie `--web` após mudar o YAML. diff --git a/docs/TROUBLESHOOTING_CREDENTIALS_AND_AUTH.md b/docs/TROUBLESHOOTING_CREDENTIALS_AND_AUTH.md index 520da82c8..7e65f6492 100644 --- a/docs/TROUBLESHOOTING_CREDENTIALS_AND_AUTH.md +++ b/docs/TROUBLESHOOTING_CREDENTIALS_AND_AUTH.md @@ -85,7 +85,7 @@ If that is not enough, use the sections below. This is **not** `auth_failed` from the remote API. `connectors/rest_connector.py` refuses to attach credentials when: 1. Secrets come from the environment (`token_from_env`, `client_secret_from_env`, `client_secret: "${VAR}"`, **or** REST `pass_from_env` / `password_from_env` / `user_from_env`) and **`auth.allowed_hosts`** is missing. The loader may already have copied the env value into `pass`; the gate still fires while `*_from_env` remains (#2006). -2. Those REST `*_from_env` names do not start with **`API_`**, **`REST_API_`**, or **`DATA_BOAR_`**. Database/share `pass_from_env` names are **not** on this prefix list. +2. A `*_from_env` name (REST or SQL / NoSQL / share) does not start with **`API_`**, **`REST_API_`**, or **`DATA_BOAR_`**, or the non-REST destination is missing from `allowed_hosts`. Both checks fail closed at load (#2086). 3. `base_url` or `auth.token_url` hostname is **not** in the allowlist (exact lowercase hostname; no wildcards). **Fix:** List every host that receives the secret (API + token endpoint). For inline tokens, the implicit list is `base_url` / `token_url` on the same target. Full contract: [USAGE.md](USAGE.md) (*Targets: APIs*). Tests: `tests/test_rest_connector_auth_allowlist.py`, `tests/test_rest_connector_pass_from_env_loader.py`. diff --git a/docs/TROUBLESHOOTING_CREDENTIALS_AND_AUTH.pt_BR.md b/docs/TROUBLESHOOTING_CREDENTIALS_AND_AUTH.pt_BR.md index 8eb03a2b3..540df351a 100644 --- a/docs/TROUBLESHOOTING_CREDENTIALS_AND_AUTH.pt_BR.md +++ b/docs/TROUBLESHOOTING_CREDENTIALS_AND_AUTH.pt_BR.md @@ -34,7 +34,7 @@ Verifique usuário/senha no config (`user`, `pass` ou `password`); conta bloquea **Basic/Bearer/API key:** Use apenas um (Basic com user/pass, ou Bearer, ou header de API key). Evite enviar a mesma chave no header e no body. **OAuth2 (ex. Power BI, Dataverse):** Verifique tenant_id, client_id, client_secret e permissões do app no IdP; não envie o mesmo token no fluxo OAuth e em um header customizado. -**Allow-list de host (`#1977` / `#2006`):** se **Details** contém `#1977`, o conector recusou anexar credencial **antes** do HTTP. Segredo de ambiente (`token_from_env`, `client_secret_from_env`, `client_secret: "${VAR}"` **ou** `pass_from_env` / `password_from_env` / `user_from_env` em REST) exige `auth.allowed_hosts`. O loader pode já ter copiado o valor para `pass`; o gate ainda dispara enquanto `*_from_env` existir (#2006). Nomes `*_from_env` **REST** precisam do prefixo **`API_`**, **`REST_API_`** ou **`DATA_BOAR_`** — `pass_from_env` de banco/share **não**. O hostname de `base_url` / `auth.token_url` precisa estar na lista (exato, minúsculas, sem wildcard). Contrato: [USAGE.pt_BR.md](USAGE.pt_BR.md) (*Alvos: APIs*). Testes: `tests/test_rest_connector_auth_allowlist.py`, `tests/test_rest_connector_pass_from_env_loader.py`. +**Allow-list de host (`#1977` / `#2006`):** se **Details** contém `#1977`, o conector recusou anexar credencial **antes** do HTTP. Segredo de ambiente (`token_from_env`, `client_secret_from_env`, `client_secret: "${VAR}"` **ou** `pass_from_env` / `password_from_env` / `user_from_env` em REST) exige `auth.allowed_hosts`. O loader pode já ter copiado o valor para `pass`; o gate ainda dispara enquanto `*_from_env` existir (#2006). Nomes `*_from_env` (REST ou SQL / NoSQL / share) precisam do prefixo **`API_`**, **`REST_API_`** ou **`DATA_BOAR_`**, e o destino não-REST precisa estar em `allowed_hosts`. Os dois falham fechado no load (#2086). O hostname de `base_url` / `auth.token_url` precisa estar na lista (exato, minúsculas, sem wildcard). Contrato: [USAGE.pt_BR.md](USAGE.pt_BR.md) (*Alvos: APIs*). Testes: `tests/test_rest_connector_auth_allowlist.py`, `tests/test_rest_connector_pass_from_env_loader.py`. **Host de login Microsoft (`#2007`):** Power BI e Dataverse só fazem POST do client secret para **`https://login.microsoftonline.com`**. `auth.token_url` customizado para outro host (inclusive nuvem soberana Azure) falha fechado **antes** do POST (`#2007` em **Details**). Testes: `tests/test_powerbi_dataverse_http_surfaces.py`. diff --git a/docs/USAGE.md b/docs/USAGE.md index 2ea9d2557..57058f209 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -97,10 +97,11 @@ findings_sink: enabled: true type: postgresql # postgresql | mysql | mssql | mongodb | sqlite (lab) host: db.example.com + allowed_hosts: ["db.example.com"] port: 5432 database: data_governance_db - user_from_env: SINK_DB_USER - pass_from_env: SINK_DB_PASS + user_from_env: DATA_BOAR_SINK_DB_USER + pass_from_env: DATA_BOAR_SINK_DB_PASS schema: data_boar # documented for PostgreSQL; unused by the echo client today on_conflict: upsert # upsert | skip | fail include_sample_content: false @@ -525,7 +526,7 @@ If you are **new to the product** (counsel, DPO, audit lead, or anyone validatin 1. Open the **hub** **[docs/samples/README.md](samples/README.md)** ([pt-BR](samples/README.pt_BR.md)) — it links the starter YAML, the minimal Docker template, the scope CSV, and explains **`config/config.json`** (legacy JSON only). 1. Copy **[deploy/samples/config.starter-lgpd-eval.yaml](../deploy/samples/config.starter-lgpd-eval.yaml)** to your machine as **`config.yaml`** (any writable folder). The file is **Runnable defaults** (two folders + one database + rate limit + ML/DL terms) plus **commented optional** sections (e.g. `scan_compressed`, `file_passwords`, `learned_patterns`, external pattern files) so you can enable features gradually. 1. Replace the **fictitious** paths (`/opt/lgpd-demo/...`) and the database host **`192.0.2.10`** (documentation-only TEST-NET) with your lab paths and hostname. -1. Set the environment variable referenced by **`pass_from_env`** (example: `DEMO_DB_PASSWORD`) **before** starting the process — do not paste real passwords into YAML for production. +1. Set the environment variable referenced by **`pass_from_env`** (example: `DATA_BOAR_DEMO_DB_PASSWORD`) **before** starting the process — do not paste real passwords into YAML for production. The name must use a `DATA_BOAR_` / `REST_API_` / `API_` prefix, and `allowed_hosts` must list `192.0.2.10` (already in the starter). 1. Run a one-shot scan: `uv run python main.py --config config.yaml` or start the dashboard with the same `--config` path. The starter is **not** as exhaustive as **`docs/compliance-samples/`** (those are for **framework-specific** tuning). @@ -593,11 +594,24 @@ To keep secrets **out of the config file**, use **`*_from_env`** keys so the app **Requesting access from IT:** When you need to ask the IT team for permissions (e.g. shared folders, database accounts, API tokens), use the **minimal** access required. See [OPERATOR_IT_REQUIREMENTS.md](ops/OPERATOR_IT_REQUIREMENTS.md) for a per-source checklist of what to ask for (read-only, no admin), what we do *not* need, and a short justification so the request aligns with zero-trust or strict IAM. ([pt-BR](ops/OPERATOR_IT_REQUIREMENTS.pt_BR.md)) - **API key:** `api.api_key_from_env: "AUDIT_API_KEY"` (see Authentication above). -- **Targets (databases, REST, Power BI, etc.):** -- **Password:** `pass_from_env: "DB_PASS"` or `password_from_env: "DB_PASS"` — the app reads the password from the named env var. -- **User:** `user_from_env: "DB_USER"` — username from env. -- **REST / OAuth:** In the target’s `auth` block: `token_from_env: "REST_TOKEN"`, `client_secret_from_env: "CLIENT_SECRET"`. -- **Power BI / Dataverse:** At target level: `client_secret_from_env: "PBI_SECRET"`, or in `auth`: `client_secret_from_env: "PBI_SECRET"`. +- **Targets (databases, REST, Power BI, etc.):** two checks run at load (#2086). The env **name** must start with **`DATA_BOAR_`**, **`REST_API_`**, or **`API_`**. The destination must appear in that target’s **`allowed_hosts`** (or `auth.allowed_hosts`). A name outside the prefix, or a destination that is not on the list, raises `ValueError` and the secret is not copied. +- **Password:** `pass_from_env: "DATA_BOAR_DB_PASS"` or `password_from_env: "DATA_BOAR_DB_PASS"`. +- **User:** `user_from_env: "DATA_BOAR_DB_USER"`. +- **REST / OAuth:** In the target’s `auth` block: `token_from_env: "API_REST_TOKEN"`, `client_secret_from_env: "API_CLIENT_SECRET"`, plus `allowed_hosts` for the API host and the token host. +- **Power BI / Dataverse:** `client_secret_from_env: "API_PBI_SECRET"`. Power BI’s default token host is already vendor-allowlisted; Dataverse still needs `allowed_hosts` for `org_url` / `environment_url`. +- **Example (database):** + +```yaml +- name: payroll + type: database + driver: postgresql+psycopg2 + host: db.example.com + allowed_hosts: ["db.example.com"] + user: reader + pass_from_env: DATA_BOAR_DB_PASS +``` + +`allowed_hosts` is written in the same YAML as the rest of the target. Against a config author, only the prefix is an independent control; the host list is a brake. See [SECURITY.md](SECURITY.md). When a `*_from_env` key is set, the resolved value is used for the connection; the config file can omit the literal secret. Restrict config file permissions and **do not commit** config files that contain credentials; see [SECURITY.md](../SECURITY.md) (Config file and secrets). Bitwarden as human source of truth: [OPERATOR_SECRETS_BITWARDEN.md](ops/OPERATOR_SECRETS_BITWARDEN.md). @@ -889,7 +903,7 @@ Use `type: api` or `type: rest`. Required: `name`, `base_url` (or `url`). Option **Credential host allowlist (#1977 / #2006):** Before the connector attaches Basic, Bearer, OAuth client secrets, or custom `Authorization` / `X-API-Key` / `api-key` headers, it checks every credential endpoint host (`base_url` / `url` and `auth.token_url`). Matching is the **exact hostname** (lowercased). No wildcards, no ports, no path. - **Secrets from the environment** (`auth.token_from_env`, `auth.client_secret_from_env`, `client_secret: "${VAR}"`, **or** target-level `pass_from_env` / `password_from_env` / `user_from_env`) **require** an explicit list: `auth.allowed_hosts` (or top-level `allowed_hosts`). Include **both** the API host and the OAuth token host when they differ. The config loader may copy the env value into `pass` / `user`; the connector still treats the target as env-loaded while those `*_from_env` keys remain (#2006). -- Names in `token_from_env`, `client_secret_from_env`, `pass_from_env`, `password_from_env`, and `user_from_env` on **REST/API** targets must start with **`API_`**, **`REST_API_`**, or **`DATA_BOAR_`**. Other names fail closed (`ValueError` containing `#1977`) before any `Authorization` header is set. **SQL / NoSQL / share** targets do **not** use this prefix list (a database `pass_from_env: DEMO_DB_PASSWORD` stays valid). +- Names in `token_from_env`, `client_secret_from_env`, `pass_from_env`, `password_from_env`, and `user_from_env` must start with **`API_`**, **`REST_API_`**, or **`DATA_BOAR_`** on REST **and** on SQL, NoSQL, and share targets. Other names fail closed (`ValueError` containing `#1977`) and the secret is not copied. Non-REST targets also need the destination in `allowed_hosts` (see *Credentials from environment* above). Example: `pass_from_env: DATA_BOAR_DB_PASS` with `allowed_hosts: ["db.example.com"]`. - **Inline** tokens/passwords (no env keys) default to the hosts already in `base_url` / `auth.token_url` on the **same** target. An explicit list still wins and can reject a mismatched `base_url`. - Failure is connect-time **`ValueError`**. Scan failures usually show reason **`error`** with `#1977` in **Details** — not a remote 401. See [TROUBLESHOOTING_CREDENTIALS_AND_AUTH.md](TROUBLESHOOTING_CREDENTIALS_AND_AUTH.md). diff --git a/docs/USAGE.pt_BR.md b/docs/USAGE.pt_BR.md index f01b56b80..6d9a7b979 100644 --- a/docs/USAGE.pt_BR.md +++ b/docs/USAGE.pt_BR.md @@ -106,10 +106,11 @@ findings_sink: enabled: true type: postgresql # postgresql | mysql | mssql | mongodb | sqlite (lab) host: db.example.com + allowed_hosts: ["db.example.com"] port: 5432 database: data_governance_db - user_from_env: SINK_DB_USER - pass_from_env: SINK_DB_PASS + user_from_env: DATA_BOAR_SINK_DB_USER + pass_from_env: DATA_BOAR_SINK_DB_PASS schema: data_boar # documentado para PostgreSQL; o cliente de eco ainda não usa on_conflict: upsert # upsert | skip | fail include_sample_content: false @@ -600,6 +601,13 @@ Defina `licensing.effective_tier: enterprise` em lab. Mesmo CLI do Pro (`--gover Para manter segredos **fora do arquivo de config**, use chaves **`*_from_env`**: o YAML guarda só o **nome** da variável; o valor entra no **ambiente do processo** na subida. Esse é o contrato estável — vaults (Bitwarden hoje; Fase B `@vault:` / vaults corporativos depois) devem **injetar na mesma camada de env**. Ponte opcional: `~/.config/databoar/*.env` + `scripts/databoar-env-load.sh` / `.ps1`. Detalhe: [OPERATOR_CREDENTIALS_FROM_ENV.pt_BR.md](ops/OPERATOR_CREDENTIALS_FROM_ENV.pt_BR.md) ([EN](ops/OPERATOR_CREDENTIALS_FROM_ENV.md)). Bitwarden como fonte humana: [OPERATOR_SECRETS_BITWARDEN.pt_BR.md](ops/OPERATOR_SECRETS_BITWARDEN.pt_BR.md). +Dois requisitos no load (#2086), para `pass_from_env`, `password_from_env`, `user_from_env` e `token_from_env`: + +- O **nome** começa com **`DATA_BOAR_`**, **`REST_API_`** ou **`API_`**. +- O destino do alvo está em **`allowed_hosts`** (ou `auth.allowed_hosts`). + +Nome fora do prefixo, ou destino fora da lista, levanta `ValueError` e o segredo não é copiado. Exemplo: `pass_from_env: DATA_BOAR_DB_PASS` com `host: db.example.com` e `allowed_hosts: ["db.example.com"]`. `allowed_hosts` vem do mesmo YAML que o autor do config escreve: contra esse autor, só o prefixo é controle independente; a lista de host é um freio. Veja [SECURITY.pt_BR.md](SECURITY.pt_BR.md). + ### Config inicial (copiar e colar) e onde começar {#config-inicial-exemplos} Se você está **começando agora** (jurídico, DPO, liderança de auditoria ou **validação de capacidades**) e precisa de **um único arquivo** para alinhar com a TI **sem** ler todos os guias primeiro: @@ -607,7 +615,7 @@ Se você está **começando agora** (jurídico, DPO, liderança de auditoria ou 1. Abra o **hub** **[docs/samples/README.pt_BR.md](samples/README.pt_BR.md)** ([EN](samples/README.md)) — ele aponta para o YAML *starter*, o modelo mínimo Docker, o CSV de escopo e explica **`config/config.json`** (JSON legado só para referência). 1. Copie **[deploy/samples/config.starter-lgpd-eval.yaml](../deploy/samples/config.starter-lgpd-eval.yaml)** para a sua máquina como **`config.yaml`**. O arquivo traz **valores padrão utilizáveis** (duas pastas + um banco + timeouts + limite de taxa + termos ML/DL) e **blocos opcionais comentados** (ex.: arquivos compactados, `file_passwords`, `learned_patterns`) para ir habilitando aos poucos. 1. Substitua os caminhos **fictícios** (`/opt/lgpd-demo/...`) e o host **`192.0.2.10`** (rede só de documentação) pelos valores de laboratório reais. -1. Defina a variável de ambiente de **`pass_from_env`** (exemplo: `DEMO_DB_PASSWORD`) **antes** de subir o processo — não coloque senhas reais no YAML em produção. +1. Defina a variável de ambiente de **`pass_from_env`** (exemplo: `DATA_BOAR_DEMO_DB_PASSWORD`) **antes** de subir o processo — não coloque senhas reais no YAML em produção. O nome precisa do prefixo `DATA_BOAR_` / `REST_API_` / `API_`, e `allowed_hosts` precisa listar `192.0.2.10` (já está no starter). 1. Execute uma varredura one-shot: `uv run python main.py --config config.yaml` ou suba o painel com o mesmo `--config`. O *starter* **não** substitui os grandes YAML de **`docs/compliance-samples/`** (ajuste fino por norma). @@ -907,7 +915,7 @@ Use `type: api` ou `type: rest`. Obrigatório: `name`, `base_url` (ou `url`). Op **Allow-list de host de credencial (#1977 / #2006):** Antes de anexar Basic, Bearer, segredo OAuth ou cabeçalhos customizados `Authorization` / `X-API-Key` / `api-key`, o conector confere cada host de endpoint de credencial (`base_url` / `url` e `auth.token_url`). A comparação é o **hostname exato** (minúsculas). Sem wildcard, sem porta, sem path. - **Segredos do ambiente** (`auth.token_from_env`, `auth.client_secret_from_env`, `client_secret: "${VAR}"` **ou** `pass_from_env` / `password_from_env` / `user_from_env` no alvo) **exigem** lista explícita: `auth.allowed_hosts` (ou `allowed_hosts` no alvo). Inclua **os dois** hosts quando a API e o token OAuth diferem. O loader pode copiar o valor da env para `pass` / `user`; o conector ainda trata o alvo como carregado do ambiente enquanto essas chaves `*_from_env` existem (#2006). -- Nomes em `token_from_env`, `client_secret_from_env`, `pass_from_env`, `password_from_env` e `user_from_env` em alvos **REST/API** precisam começar com **`API_`**, **`REST_API_`** ou **`DATA_BOAR_`**. Outros nomes falham fechado (`ValueError` com `#1977`) antes de qualquer cabeçalho `Authorization`. Alvos **SQL / NoSQL / share** **não** usam essa lista de prefixo (um banco com `pass_from_env: DEMO_DB_PASSWORD` continua válido). +- Nomes em `token_from_env`, `client_secret_from_env`, `pass_from_env`, `password_from_env` e `user_from_env` precisam começar com **`API_`**, **`REST_API_`** ou **`DATA_BOAR_`** em REST **e** em alvos SQL, NoSQL e share. Outros nomes falham fechado (`ValueError` com `#1977`) e o segredo não é copiado. Alvos não-REST também precisam do destino em `allowed_hosts` (veja *Credenciais a partir do ambiente* acima). Exemplo: `pass_from_env: DATA_BOAR_DB_PASS` com `allowed_hosts: ["db.example.com"]`. - Tokens/senhas **inline** (sem chaves env) usam por padrão os hosts de `base_url` / `auth.token_url` no **mesmo** alvo. Lista explícita vence e pode rejeitar um `base_url` fora da lista. - A falha é **`ValueError`** no `connect`. Em Scan failures o motivo costuma ser **`error`** com `#1977` em **Details** — não é 401 remoto. Veja [TROUBLESHOOTING_CREDENTIALS_AND_AUTH.pt_BR.md](TROUBLESHOOTING_CREDENTIALS_AND_AUTH.pt_BR.md). diff --git a/docs/ops/OPERATOR_CREDENTIALS_FROM_ENV.md b/docs/ops/OPERATOR_CREDENTIALS_FROM_ENV.md index b51a6342d..4e2968c55 100644 --- a/docs/ops/OPERATOR_CREDENTIALS_FROM_ENV.md +++ b/docs/ops/OPERATOR_CREDENTIALS_FROM_ENV.md @@ -20,6 +20,8 @@ **Rule:** Do not invent a second secret channel in YAML that bypasses **A→B**. New vault products should **populate environment variables** (or future `@vault:` resolution that the loader already planned) with the **same names** your `*_from_env` keys point to. +**Load gate (#2086):** `pass_from_env`, `password_from_env`, `user_from_env`, `token_from_env`, and `client_secret_from_env` names must start with `DATA_BOAR_`, `REST_API_`, or `API_`, and the target’s `allowed_hosts` must include the destination. Example: `pass_from_env: DATA_BOAR_DEMO_DB_PASSWORD` with `allowed_hosts: ["192.0.2.10"]`. `allowed_hosts` lives in the same YAML the config author writes; only the prefix is an independent control against that author. + ```text Bitwarden / HCP Vault / K8s Secret / systemd EnvironmentFile │ @@ -99,7 +101,7 @@ Tracked pattern for the dashboard API key: [API_KEY_FROM_ENV_OPERATOR_STEPS.md]( ```text ~/.config/databoar/ # mkdir; chmod 700 hubspot.env # chmod 0600 — HUBSPOT_PRIVATE_APP_TOKEN=… - postgres-lab.env # DEMO_DB_PASSWORD=… + postgres-lab.env # DATA_BOAR_DEMO_DB_PASSWORD=… agent-sessions.env # optional; used by primary-linux-agent-sessions.sh ``` diff --git a/docs/ops/OPERATOR_CREDENTIALS_FROM_ENV.pt_BR.md b/docs/ops/OPERATOR_CREDENTIALS_FROM_ENV.pt_BR.md index 84c2203f3..3fdf990f0 100644 --- a/docs/ops/OPERATOR_CREDENTIALS_FROM_ENV.pt_BR.md +++ b/docs/ops/OPERATOR_CREDENTIALS_FROM_ENV.pt_BR.md @@ -20,6 +20,8 @@ **Regra:** Não inventar um segundo canal de segredo no YAML que contorne **A→B**. Novos produtos de vault devem **preencher variáveis de ambiente** (ou a resolução `@vault:` já planejada no loader) com os **mesmos nomes** apontados pelos `*_from_env`. +**Gate no load (#2086):** nomes em `pass_from_env`, `password_from_env`, `user_from_env`, `token_from_env` e `client_secret_from_env` começam com `DATA_BOAR_`, `REST_API_` ou `API_`, e `allowed_hosts` do alvo inclui o destino. Exemplo: `pass_from_env: DATA_BOAR_DEMO_DB_PASSWORD` com `allowed_hosts: ["192.0.2.10"]`. `allowed_hosts` vive no mesmo YAML que o autor do config escreve; contra esse autor, só o prefixo é controle independente. + ```text Bitwarden / HCP Vault / Secret K8s / systemd EnvironmentFile │ @@ -96,7 +98,7 @@ Padrão da chave de API do painel: [API_KEY_FROM_ENV_OPERATOR_STEPS.md](API_KEY_ ```text ~/.config/databoar/ # mkdir; chmod 700 hubspot.env # chmod 0600 — HUBSPOT_PRIVATE_APP_TOKEN=… - postgres-lab.env # DEMO_DB_PASSWORD=… + postgres-lab.env # DATA_BOAR_DEMO_DB_PASSWORD=… agent-sessions.env # opcional; usado por primary-linux-agent-sessions.sh ``` diff --git a/docs/ops/SCOPE_IMPORT_QUICKSTART.md b/docs/ops/SCOPE_IMPORT_QUICKSTART.md index e769a5719..8b299621d 100644 --- a/docs/ops/SCOPE_IMPORT_QUICKSTART.md +++ b/docs/ops/SCOPE_IMPORT_QUICKSTART.md @@ -35,7 +35,7 @@ This path is for **counsel, DPOs, or programme leads** who have a **list of syst 1. Add **one row per data source** (folder, database, or share). One row = one place the organisation stores files or tables that might contain personal data. 1. In **`type`**, use only values the product supports in the importer today (examples: `filesystem` for a folder path, `postgresql` or `mysql` for SQL, `smb` / `nfs` for shares). If unsure, put `filesystem` for “a folder we need to scan” and fill **`path`** with the path IT will confirm. 1. **`name`** is a free label (e.g. “HR shared drive”) so people can recognise the row in reviews — it is not a technical hostname. -1. **Never** type real passwords in the sheet. Put the **name of an environment variable** in **`pass_from_env`** (e.g. `HR_DB_PASSWORD`) and ask IT to set that variable on the server before running scans. +1. **Never** type real passwords in the sheet. Put the **name of an environment variable** in **`pass_from_env`** (e.g. `DATA_BOAR_HR_DB_PASSWORD`) and ask IT to set that variable on the server before running scans. The name must start with `DATA_BOAR_`, `REST_API_`, or `API_`, and the finished YAML needs `allowed_hosts` listing that row’s host. 1. **Save As** → **CSV UTF-8** (comma-separated). If the tool offers only “CSV (comma delimited)”, use **Excel UTF-8** or **LibreOffice** with UTF-8 so Portuguese accents do not break. 1. Give the CSV file to the person who runs Data Boar (or run yourself if you have the repo): `uv run python scripts/scope_import_csv.py path/to/your.csv -o targets.fragment.yaml` diff --git a/docs/ops/SCOPE_IMPORT_QUICKSTART.pt_BR.md b/docs/ops/SCOPE_IMPORT_QUICKSTART.pt_BR.md index 81153cbc7..6766a3dd1 100644 --- a/docs/ops/SCOPE_IMPORT_QUICKSTART.pt_BR.md +++ b/docs/ops/SCOPE_IMPORT_QUICKSTART.pt_BR.md @@ -35,7 +35,7 @@ Para **jurídico, DPO ou gestão de programa** que tem uma **lista de sistemas** 1. Inclua **uma linha por fonte de dados** (pasta, base ou compartilhamento). Uma linha = um local onde a organização guarda arquivos ou tabelas que possam conter dados pessoais. 1. Em **`type`**, use só valores que o importador suporta hoje (ex.: `filesystem` para pasta, `postgresql` ou `mysql` para SQL, `smb` / `nfs` para compartilhamentos). Em dúvida, use `filesystem` para “uma pasta a analisar” e preencha **`path`** com o caminho que a TI confirmar. 1. **`name`** é um rótulo livre (ex.: “compartilhamento RH”) para reconhecer a linha na revisão — não é hostname técnico. -1. **Nunca** coloque senhas reais na planilha. Coloque o **nome de uma variável de ambiente** em **`pass_from_env`** (ex.: `HR_DB_PASSWORD`) e peça à TI para definir essa variável no servidor antes das varreduras. +1. **Nunca** coloque senhas reais na planilha. Coloque o **nome de uma variável de ambiente** em **`pass_from_env`** (ex.: `DATA_BOAR_HR_DB_PASSWORD`) e peça à TI para definir essa variável no servidor antes das varreduras. O nome começa com `DATA_BOAR_`, `REST_API_` ou `API_`, e o YAML final precisa de `allowed_hosts` com o host daquela linha. 1. **Salvar como** → **CSV UTF-8** (vírgulas). Se só existir “CSV (delimitado por vírgulas)”, use **Excel UTF-8** ou **LibreOffice** em UTF-8 para não corromper acentos. 1. Entregue o CSV a quem executa o Data Boar (ou execute você, se tiver o repositório): `uv run python scripts/scope_import_csv.py caminho/para/sua.csv -o targets.fragment.yaml` diff --git a/tests/test_shipped_credential_examples.py b/tests/test_shipped_credential_examples.py new file mode 100644 index 000000000..0b0808d3b --- /dev/null +++ b/tests/test_shipped_credential_examples.py @@ -0,0 +1,30 @@ +"""Shipped sample YAML must load under the #2086 credential env gate.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +from config.loader import normalize_config + +_REPO = Path(__file__).resolve().parents[1] +_SAMPLES = ( + _REPO / "deploy" / "samples" / "config.starter-lgpd-eval.yaml", + _REPO / "deploy" / "compat-matrix-config-ref" / "config-db.yaml", +) + + +def test_shipped_credential_examples_normalize( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("DATA_BOAR_DEMO_DB_PASSWORD", "demo-secret") + monkeypatch.setenv("DATA_BOAR_LAB_PG_PASSWORD", "pg-secret") + monkeypatch.setenv("DATA_BOAR_LAB_MY_PASSWORD", "my-secret") + monkeypatch.setenv("DATA_BOAR_LAB_MSSQL_SA_PASSWORD", "mssql-secret") + monkeypatch.setenv("DATA_BOAR_LAB_ORACLE_PASSWORD", "oracle-secret") + for path in _SAMPLES: + raw = yaml.safe_load(path.read_text(encoding="utf-8")) + out = normalize_config(raw) + assert out["targets"] From e807dc7cd76b50e09120a2d8022ebf9a4676bf8e Mon Sep 17 00:00:00 2001 From: Fabio Leitao Date: Mon, 19 Oct 2026 12:39:00 -0300 Subject: [PATCH 07/12] fix(security): disable FastAPI native OTel auto-config (Refs #2089) FastAPI 0.142.x exports OTLP from the environment unless auto_configure is off. Keep product export behind DATA_BOAR_OTEL_ENABLED. The version bump to 0.143.0 stays on the dependency PR. --- api/routes.py | 3 +++ tests/test_otel_setup_1500.py | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/api/routes.py b/api/routes.py index f91d865e6..764fa6159 100644 --- a/api/routes.py +++ b/api/routes.py @@ -801,6 +801,9 @@ async def _lifespan(app: FastAPI): title="LGPD/GDPR/CCPA Audit API", version=get_about_info()["version"], lifespan=_lifespan, + # FastAPI 0.142.x auto-exports OTLP when the env is set. Product export + # stays behind DATA_BOAR_OTEL_ENABLED (core.otel_setup, #1500 / #2089). + telemetry={"auto_configure": False}, ) # OpenTelemetry: opt-in via DATA_BOAR_OTEL_ENABLED (see core/otel_setup.py / #1500). diff --git a/tests/test_otel_setup_1500.py b/tests/test_otel_setup_1500.py index 193aa94b9..89978f717 100644 --- a/tests/test_otel_setup_1500.py +++ b/tests/test_otel_setup_1500.py @@ -41,6 +41,16 @@ def test_otel_disabled_by_default(monkeypatch: pytest.MonkeyPatch) -> None: assert maybe_setup_otel(app=None) is False +def test_fastapi_app_does_not_auto_configure_otel( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Native FastAPI export stays off unless DATA_BOAR_OTEL_ENABLED (#2089).""" + monkeypatch.delenv("DATA_BOAR_OTEL_ENABLED", raising=False) + import api.routes as routes + + assert routes.app._telemetry["auto_configure"] is False + + @pytest.mark.parametrize("truthy", ["1", "true", "YES", "on"]) def test_otel_enabled_truthy(monkeypatch: pytest.MonkeyPatch, truthy: str) -> None: monkeypatch.setenv("DATA_BOAR_OTEL_ENABLED", truthy) From adb5e3ff18f6c6e53a0cef38444a3d4faf1611ad Mon Sep 17 00:00:00 2001 From: Fabio Leitao Date: Mon, 19 Oct 2026 13:18:28 -0300 Subject: [PATCH 08/12] fix(security): keep unauthenticated /health to a liveness body (Refs #2077) Probes still skip the API key and now receive only {"status":"ok"}. License, trust, transport, and integrity stay on GET /status. No separate not-ready 503 existed on this handler, so that path was left unchanged. --- CHANGELOG.md | 1 + api/routes.py | 25 ++++++------------- docs/SECURITY.md | 4 +-- docs/SECURITY.pt_BR.md | 4 +-- docs/USAGE.md | 8 +++--- docs/USAGE.pt_BR.md | 8 +++--- ..._INSPIRED_ENTERPRISE_TRUST_ACCELERATORS.md | 2 +- tests/test_api_key.py | 4 +-- tests/test_licensing.py | 6 +++-- 9 files changed, 26 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bea849687..3f3491b3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ Human-readable summary of user-facing changes. **Detailed release notes:** [docs Migration notes on the **`1.8.0-rc`** working line (no new semver cut): - **Credential env names (#2086):** `pass_from_env`, `password_from_env`, `user_from_env`, `token_from_env`, and `client_secret_from_env` fail at load when the name is outside the `DATA_BOAR_` / `REST_API_` / `API_` prefixes, or when the destination is missing from that target’s `allowed_hosts`. Shipped examples now use prefixed names (`DATA_BOAR_DEMO_DB_PASSWORD`, `DATA_BOAR_LAB_*`). +- **`GET /health` (#2077):** the unauthenticated body is only `{"status":"ok"}`. License, trust, transport, and integrity moved to authenticated `GET /status`. - **Notification `public_base_url` (#2087):** `public_base_url: ${VAR}` raises `ValueError` and is not expanded. Other notification `${VAR}` values still expand only for the same prefixes. --- diff --git a/api/routes.py b/api/routes.py index 764fa6159..f16b118eb 100644 --- a/api/routes.py +++ b/api/routes.py @@ -128,7 +128,7 @@ class LocaleSlug(str, Enum): def _integrity_snapshot() -> dict: - """#856: integrity anchor snapshot for /health and /status (ensure once, lazily).""" + """#856: integrity anchor snapshot for /status (ensure once, lazily).""" from core.integrity_anchor import ensure_integrity_anchor, get_integrity_snapshot snap = get_integrity_snapshot() @@ -1340,27 +1340,15 @@ async def locale_html_middleware(request: Request, call_next): @app.get("/health") -async def health(request: Request): +async def health(): """ Liveness/readiness probe for Docker, Swarm and Kubernetes. - Semantics: **always unauthenticated** (no API key). Returns minimal JSON for - orchestrators; see SECURITY.md / USAGE.md for difference vs protected routes. + Semantics: **always unauthenticated** (no API key). Body is only + ``{"status": "ok"}`` (#2077). License, trust, and transport live on + authenticated ``GET /status``. """ - cfg = _get_config() - canonical = get_canonical_trust_snapshot(cfg) - eff = effective_dashboard_transport(request, cfg) - body: dict = {"status": "ok"} - body["license"] = _license_public_dict() - body["trust_state"] = canonical["trust_state"] - body["trust_reasons"] = canonical["trust_reasons"] - body["output_confidence"] = canonical["output_confidence"] - body["dashboard_transport"] = get_dashboard_transport_snapshot() - body["forwarded_headers"] = eff["forwarded"] - body["effective_external_transport"] = eff["effective_external_transport"] - body["enterprise_surface"] = get_enterprise_surface_posture(cfg) - body["integrity"] = _integrity_snapshot() - return body + return {"status": "ok"} @app.get("/about/json") @@ -1578,6 +1566,7 @@ async def get_status(request: Request): "enterprise_surface": get_enterprise_surface_posture(cfg), "maturity_assessment_integrity": maturity_integrity, "integrity": _integrity_snapshot(), + "license": _license_public_dict(), } diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 1dc3f45b8..829b8f2f0 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -21,7 +21,7 @@ For full policy, supported versions, dependency audit, and how to report vulnera | **Audit text logs (`audit_*.log`)** | After config normalization, each target gets a unique **`audit_log_name`** (sanitized from `name`) so log lines never collide. Filesystem **Connected** and **Finding** lines avoid raw absolute host paths: scan roots use `folder?8-hex`, file paths use POSIX paths **relative to the resolved scan root**, or `audit_log_name?12-hex` when a file resolves **outside** that root (e.g. symlink). **`GET /logs`** runs a second-layer PII self-scan (`core/log_self_scan.py`, #877) before serving attachments: cleartext shape hits block export (HTTP **422**, category counts only) and record an **Audit Trail** finding — complements write-time **`sanitize_log_text`** ([ADR-0036](adr/ADR-0036-exception-and-log-pii-redaction-pipeline.md)). The SQLite/Excel report remains the authoritative evidence; see **docs/USAGE.md** (audit log format). | `tests/test_audit_log_display.py`, `tests/test_database.py` (`test_normalize_config_sets_unique_audit_log_names`), `tests/test_log_self_scan.py` | | **Host header (`TrustedHostMiddleware`)** | Allowed Host names come from `trusted_api_hosts()` at **import**: always `127.0.0.1`, `localhost`, `testserver`; plus `api.host` when set; plus `api.trusted_hosts` extras. Wildcards (`*`) are ignored. Binding `0.0.0.0` does not add a public DNS name. Dashboard YAML save does **not** refresh the list — restart `--web`. Untrusted Host → **HTTP 400**. | `tests/test_host_resolution.py` (`test_api_rejects_untrusted_host_header`) | | **Findings CSV formula injection** | `GET /findings/csv` and `GET /findings/{session_id}/csv` run every string cell through `excel_sanitize_cell` (prefix `'` when the value starts with `=`, `+`, `-`, `@`, TAB, or CR — CWE-1236, same helper as XLSX / #547 / #1723). JSON `/findings` is unchanged. | `tests/test_api_findings_export.py` (`test_build_findings_csv_sanitizes_formula_prefixes`); `tests/test_report_formula_injection.py` | -| **Integrity snapshot public `error`** | Fail-soft `ensure_integrity_anchor` JSON on **`/health`** and **`/status`** stores **`type(e).__name__` only** (#1721). `str(e)` stays in the operator log via `SanitizeLogFilter`. | `tests/test_integrity_anchor.py` (`test_ensure_exception_public_error_is_type_name_only`) | +| **Integrity snapshot public `error`** | Fail-soft `ensure_integrity_anchor` JSON on **`/status`** stores **`type(e).__name__` only** (#1721). `str(e)` stays in the operator log via `SanitizeLogFilter`. | `tests/test_integrity_anchor.py` (`test_ensure_exception_public_error_is_type_name_only`) | The table above is guarded by **`tests/test_security.py`** plus the named modules. Running `pytest tests/test_security.py tests/test_audit_log_display.py tests/test_host_resolution.py tests/test_api_findings_export.py tests/test_integrity_anchor.py -v` regularly (e.g. in CI) helps ensure these protections are not accidentally removed. @@ -36,7 +36,7 @@ The table above is guarded by **`tests/test_security.py`** plus the named module - **Passwords with special characters:** You can safely use database or MongoDB passwords that contain `@`, `:`, `/`, or `#`. The application encodes them when building connection URLs. No extra configuration is required. - **Credential env names (#2086):** `pass_from_env`, `password_from_env`, `user_from_env`, `token_from_env`, and `client_secret_from_env` must use a `DATA_BOAR_` / `REST_API_` / `API_` name, and the destination must be listed in that target’s `allowed_hosts` (vendor-fixed hosts such as HubSpot and the default Power BI token URL keep their own allowlist). `allowed_hosts` is read from the **same YAML the config author writes**, same limit as #1977. Against that author, only the **prefix** is an independent control; the host list is a brake, not a second authority. An operator-controlled allowlist is not shipped. - **Protecting the config in the UI:** The **Configuration** page (GET `/config`) shows and allows editing the main config file, which may contain credentials. If the API is exposed to untrusted users or networks, set **`api.require_api_key: true`** and provide a strong API key (or `api.api_key_from_env`). Then only requests that send a valid **X-API-Key** or **Authorization: Bearer** can access `/config`. -- **API key vs /health:** **`GET /health`** stays **unauthenticated** (liveness for orchestrators). Other routes require the key when enabled. **401** = wrong/missing key; **503** = `require_api_key` true but no key resolved; **`main.py --web` exits 2** before listen in that case. See root **[SECURITY.md](../SECURITY.md)** § Optional API key, **[USAGE.md](USAGE.md)** (Authentication), and **[SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.md](ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.md)** ([pt-BR](ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.pt_BR.md)). +- **API key vs /health:** **`GET /health`** stays **unauthenticated** (liveness for orchestrators) and returns only `{"status":"ok"}` (#2077). License, trust, and transport are on **`GET /status`**. Other routes require the key when enabled. **401** = wrong/missing key; **503** = `require_api_key` true but no key resolved; **`main.py --web` exits 2** before listen in that case. See root **[SECURITY.md](../SECURITY.md)** § Optional API key, **[USAGE.md](USAGE.md)** (Authentication), and **[SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.md](ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.md)** ([pt-BR](ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.pt_BR.md)). - **Host header vs bind:** Clients must send a `Host` that is in `trusted_api_hosts()`. Add the public DNS name via **`api.host`** or **`api.trusted_hosts`**. Binding `0.0.0.0` is not enough. Untrusted Host is **HTTP 400** (not 401). Restart `--web` after YAML changes. - **Findings CSV vs Excel:** CSV downloads use the same formula-prefix sanitizer as the workbook. JSON `/findings` does not. - **Where config is stored:** The config file path is set at startup (default `config.yaml` or `CONFIG_PATH`). Ensure the file and directory permissions restrict read/write to trusted users only. diff --git a/docs/SECURITY.pt_BR.md b/docs/SECURITY.pt_BR.md index f3568ff63..05ecae56c 100644 --- a/docs/SECURITY.pt_BR.md +++ b/docs/SECURITY.pt_BR.md @@ -21,7 +21,7 @@ Para política completa, versões suportadas, auditoria de dependências e como | **Logs de texto de auditoria (`audit_*.log`)** | Depois da normalização do config, cada target recebe um **`audit_log_name`** único (higienizado a partir de `name`) para não colidir em linhas de log. Em filesystem, linhas **Connected** e **Finding** evitam caminho absoluto bruto: raiz de varredura como `pasta?8-hex`, arquivos como caminho POSIX **relativo** à raiz resolvida, ou `audit_log_name?12-hex` quando o arquivo resolve **fora** dessa raiz (ex.: symlink). **`GET /logs`** executa auto-varredura de PII em segunda camada (`core/log_self_scan.py`, #877) antes de servir anexos: formas sensíveis em claro bloqueiam a exportação (HTTP **422**, só contagem por categoria) e registram finding de **Audit Trail** — complementa **`sanitize_log_text`** na escrita ([ADR-0036](adr/ADR-0036-exception-and-log-pii-redaction-pipeline.md)). O SQLite/Excel do relatório continua sendo a evidência canônica; veja **docs/USAGE.md** (formato do audit log). | `tests/test_audit_log_display.py`, `tests/test_database.py` (`test_normalize_config_sets_unique_audit_log_names`), `tests/test_log_self_scan.py` | | **Cabeçalho Host (`TrustedHostMiddleware`)** | Nomes Host aceitos vêm de `trusted_api_hosts()` na **importação**: sempre `127.0.0.1`, `localhost`, `testserver`; mais `api.host` quando definido; mais extras em `api.trusted_hosts`. Wildcards (`*`) são ignorados. Bind `0.0.0.0` não acrescenta DNS público. Salvar YAML no dashboard **não** atualiza a lista — reinicie `--web`. Host não confiável → **HTTP 400**. | `tests/test_host_resolution.py` (`test_api_rejects_untrusted_host_header`) | | **Injeção de fórmula no CSV de achados** | `GET /findings/csv` e `GET /findings/{session_id}/csv` passam cada célula string por `excel_sanitize_cell` (prefixo `'` se o valor começa com `=`, `+`, `-`, `@`, TAB ou CR — CWE-1236, mesmo helper do XLSX / #547 / #1723). O JSON `/findings` não muda. | `tests/test_api_findings_export.py` (`test_build_findings_csv_sanitizes_formula_prefixes`); `tests/test_report_formula_injection.py` | -| **Campo público `error` do snapshot de integridade** | O JSON fail-soft de `ensure_integrity_anchor` em **`/health`** e **`/status`** guarda **só `type(e).__name__`** (#1721). `str(e)` permanece no log do operador via `SanitizeLogFilter`. | `tests/test_integrity_anchor.py` (`test_ensure_exception_public_error_is_type_name_only`) | +| **Campo público `error` do snapshot de integridade** | O JSON fail-soft de `ensure_integrity_anchor` em **`/status`** guarda **só `type(e).__name__`** (#1721). `str(e)` permanece no log do operador via `SanitizeLogFilter`. | `tests/test_integrity_anchor.py` (`test_ensure_exception_public_error_is_type_name_only`) | A tabela acima é coberta por **`tests/test_security.py`** e pelos módulos nomeados. Executar `pytest tests/test_security.py tests/test_audit_log_display.py tests/test_host_resolution.py tests/test_api_findings_export.py tests/test_integrity_anchor.py -v` com frequência (ex.: no CI) ajuda a garantir que essas proteções não sejam removidas por engano. @@ -36,7 +36,7 @@ A tabela acima é coberta por **`tests/test_security.py`** e pelos módulos nome - **Senhas com caracteres especiais:** Você pode usar com segurança senhas de banco ou MongoDB que contenham `@`, `:`, `/` ou `#`. A aplicação as codifica ao montar as URLs de conexão. Nenhuma configuração extra é necessária. - **Nomes de env de credencial (#2086):** `pass_from_env`, `password_from_env`, `user_from_env`, `token_from_env` e `client_secret_from_env` precisam de um nome com prefixo `DATA_BOAR_` / `REST_API_` / `API_`, e o destino precisa estar em `allowed_hosts` daquele alvo (hosts fixos de fornecedor, como HubSpot e a URL de token padrão do Power BI, seguem a allowlist própria). `allowed_hosts` é lido do **mesmo YAML que o autor do config escreve**, o mesmo limite do #1977. Contra esse autor, só o **prefixo** é controle independente; a lista de host é um freio, não uma segunda autoridade. Não há allowlist controlada pelo operador nesta versão. - **Proteger o config na interface:** A página **Configuração** (GET `/config`) exibe e permite editar o arquivo de config principal. **GET `/config` sempre redige valores secretos** (senhas, chave de API, tokens, client_secret, etc.) antes de enviar o YAML ao navegador, para que a interface nunca exiba nem transmita segredos em texto claro; ao salvar, placeholders são mesclados com o arquivo atual para não sobrescrever os segredos reais. Se a API estiver exposta a usuários ou redes não confiáveis, defina **`api.require_api_key: true`** e forneça uma chave de API forte (ou `api.api_key_from_env`). -- **Chave de API vs /health:** **`GET /health`** permanece **sem autenticação** (liveness para orquestradores). Demais rotas exigem a chave quando habilitado. **401** = chave ausente ou inválida; **503** = `require_api_key` true porém chave não resolvida; **`main.py --web` encerra com código 2** antes de escutar nesse caso. Veja **[SECURITY.pt_BR.md](../SECURITY.pt_BR.md)** (chave opcional), **[USAGE.pt_BR.md](USAGE.pt_BR.md)** (Autenticação) e **[SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.pt_BR.md](ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.pt_BR.md)** ([EN](ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.md)). +- **Chave de API vs /health:** **`GET /health`** permanece **sem autenticação** (liveness para orquestradores) e devolve só `{"status":"ok"}` (#2077). Licença, confiança e transporte ficam em **`GET /status`**. Demais rotas exigem a chave quando habilitado. **401** = chave ausente ou inválida; **503** = `require_api_key` true porém chave não resolvida; **`main.py --web` encerra com código 2** antes de escutar nesse caso. Veja **[SECURITY.pt_BR.md](../SECURITY.pt_BR.md)** (chave opcional), **[USAGE.pt_BR.md](USAGE.pt_BR.md)** (Autenticação) e **[SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.pt_BR.md](ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.pt_BR.md)** ([EN](ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.md)). - **Host vs bind:** o cliente precisa enviar um `Host` que esteja em `trusted_api_hosts()`. Inclua o DNS público via **`api.host`** ou **`api.trusted_hosts`**. Bind `0.0.0.0` não basta. Host não confiável é **HTTP 400** (não 401). Reinicie `--web` após mudar o YAML. - **CSV de achados vs Excel:** o download CSV usa o mesmo sanitizador de prefixo de fórmula que a planilha. O JSON `/findings` não. - **Arquivo de config e segredos:** Restrinja as permissões do arquivo de config (ex.: `chmod 600` em `config.yaml`) para que apenas usuários confiáveis leiam. **Não faça commit** de `config.yaml` ou de qualquer arquivo com credenciais no controle de versão; use **`config.example.yaml`** (ou **`deploy/config.example.yaml`**) como modelo e mantenha o config local no **`.gitignore`** (o projeto ignora `config.yaml`, `config.local.yaml` e `*.vault`). Prefira armazenar segredos em variáveis de ambiente (ex.: `pass_from_env`, `api_key_from_env`). Veja **docs/USAGE.md** (Configuration) e **docs/plans/PLAN_SECRETS_VAULT.md** (Phase A). diff --git a/docs/USAGE.md b/docs/USAGE.md index 57058f209..53d9d5fd9 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -152,7 +152,7 @@ When **`DATA_BOAR_OTEL_ENABLED`** is `1` / `true` / `yes` / `on` and the optiona ## REST API server (`--web`) -**Transport:** you must either use **HTTPS** (PEM cert + key on the CLI or under `api` in config) or **explicitly** accept plaintext with **`--allow-insecure-http`** (or `api.allow_insecure_http: true`). Otherwise `main.py --web` exits with code **2** and an error on stderr. **`GET /status`** and **`GET /health`** include a `dashboard_transport` object (`mode`, `tls_active`, `summary`, etc.), an **`enterprise_surface`** object (transport + license trust + global API-key posture; optional per-route RBAC is summarized under **`enterprise_surface.access_surface.rbac`** — **`enabled`** when `api.rbac` is on and the tier allows **`dashboard_rbac`**, otherwise **`not_implemented`**), and canonical **`trust_state`** / **`trust_reasons`** / **`output_confidence`** (license + integrity + transport combined — plaintext opt-in is never a clean `trusted`); plaintext mode shows a **banner** on dashboard pages. **`GET /status`** also includes **`detection_prefilter`** (rust-regex-stage readiness: `active`, `name`, `backend`, `tier`, `reason`, `engine`) next to **`runtime_trust`** — observability only; it does not change findings (#1411 / #1412). +**Transport:** you must either use **HTTPS** (PEM cert + key on the CLI or under `api` in config) or **explicitly** accept plaintext with **`--allow-insecure-http`** (or `api.allow_insecure_http: true`). Otherwise `main.py --web` exits with code **2** and an error on stderr. **`GET /status`** includes a `dashboard_transport` object (`mode`, `tls_active`, `summary`, etc.), an **`enterprise_surface`** object (transport + license trust + global API-key posture; optional per-route RBAC is summarized under **`enterprise_surface.access_surface.rbac`** — **`enabled`** when `api.rbac` is on and the tier allows **`dashboard_rbac`**, otherwise **`not_implemented`**), and canonical **`trust_state`** / **`trust_reasons`** / **`output_confidence`** (license + integrity + transport combined — plaintext opt-in is never a clean `trusted`); plaintext mode shows a **banner** on dashboard pages. **`GET /status`** also includes **`detection_prefilter`** (rust-regex-stage readiness: `active`, `name`, `backend`, `tier`, `reason`, `engine`) next to **`runtime_trust`** — observability only; it does not change findings (#1411 / #1412). ```bash # TLS (example paths; keep keys out of git) @@ -273,9 +273,9 @@ Pre-built images are on Docker Hub: `fabioleitao/data_boar:latest` ([hub.docker. - **Authentication:** By default the API does not require authentication; secure it at the reverse proxy or network level if exposed. You can optionally enable a shared API key: set `api.require_api_key: true` and either `api.api_key` (avoid committing secrets) or `api.api_key_from_env: "VAR"` with the variable set **before** process start — see **[API_KEY_FROM_ENV_OPERATOR_STEPS.md](ops/API_KEY_FROM_ENV_OPERATOR_STEPS.md)**. **Step-by-step (key + HTTPS, Let’s Encrypt, lab certs):** **[SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.md](ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.md)** ([pt-BR](ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.pt_BR.md)). **For production we recommend `require_api_key: true` plus `api_key_from_env`** so the secret never lives in tracked YAML. - **WebAuthn (optional — Phase 1 JSON API + Phase 1b HTML session):** When **`api.webauthn.enabled: true`**, set the environment variable named by **`api.webauthn.token_secret_from_env`** (default **`DATA_BOAR_WEBAUTHN_TOKEN_SECRET`**) **before** starting the process; startup fails if the secret is missing. This enables **vendor-neutral** passkey registration/authentication JSON endpoints under **`/auth/webauthn/`** (open-source **`webauthn`** library on PyPI — not Bitwarden- or Microsoft-specific). **`api.require_api_key` does not apply** to these paths or to **`GET /{locale}/login`** (so you can open the sign-in page when a global API key is on). **First passkey bootstrap (#1553):** registering the **first** credential **always** requires a valid **`X-API-Key`** / Bearer (configure **`api.api_key`** or **`api.api_key_from_env`**). There is no loopback key-free exemption — same-host reverse proxies commonly present every client as **`127.0.0.1`**. Authentication ceremonies (sign-in after a passkey exists) stay key-free on these paths. Align **`api.webauthn.origin`** and **`api.webauthn.rp_id`** with the browser URL (HTTPS recommended). Credentials persist in SQLite (`webauthn_credentials`); **`--reset-data` / wipe** clears them. **After at least one passkey is registered**, locale-prefixed dashboard pages (**`/`, `/config`, `/reports`, `/assessment`, …**) require a valid WebAuthn session cookie unless you only visit **`help`**, **`about`**, or **`login`** (GET). Use **`/{locale}/login`** in the browser ( **`/static/webauthn-login.js`** ) to register or sign in; mutating HTML forms (**`POST …/config`**, **`POST …/assessment`**) **always** require a signed **CSRF** synchronizer token (**#1231** — independent of whether the WebAuthn gate is active; optional stable key via **`DATA_BOAR_HTML_CSRF_SECRET`**, else the WebAuthn token secret when set, else a process-ephemeral secret). Optional **per-route RBAC** (next bullet; **#86** Phase **2**) layers on top when enabled. See **[ADR 0033](adr/ADR-0033-webauthn-open-relying-party-json-endpoints.md)**. - **RBAC (optional — Phase 2, Pro+ `dashboard_rbac`):** When **`api.rbac.enabled: true`** and the effective tier allows **`dashboard_rbac`** (`licensing.effective_tier` in YAML, or JWT **`dbtier`** when enforcement is on — same as other tier-gated features), protected routes require **named roles**: `admin` (all), `dashboard`, `scanner`, `reports_reader`, `config_admin`, and **`audit_logs.read`** for **`GET /logs`** / **`GET /logs/{session_id}`**. **`GET`/`POST /{locale}/config`** requires **`config_admin`** or **`admin`** (#414). The **global API key** (when present and matching) receives **`api.rbac.api_key_roles`**; WebAuthn sessions use the **`roles_json`** column on the matching `webauthn_credentials` row (JSON array of role names), or **`api.rbac.default_roles`** when **`roles_json`** is unset. **401** if no principal; **403** if roles are insufficient. When RBAC is **not** active, `/logs` follows the same posture as `/findings` / `/report` (still gated by `api.audit_logs.enabled` / directory, and by **`api.require_api_key`** when that flag is on). **`GET /status`** and **`GET /health`** include **`enterprise_surface.access_surface.rbac`** (`enabled` vs `not_implemented`). **Community** tier cannot enable in-product RBAC; use **`require_api_key`**, reverse-proxy, or network controls. See the internal plan file **`PLAN_DASHBOARD_REPORTS_ACCESS_CONTROL.md`** (listed under **Internal and reference** in `docs/README.md`; GitHub **#86**); enterprise **SSO/OIDC** remains Phase **3**. -- **`GET /health` (always public):** Intended for load balancers and orchestrators. Returns JSON with at least `status`, a **public** `license` summary, `dashboard_transport`, `enterprise_surface`, canonical **`trust_state`** / **`trust_reasons`** / **`output_confidence`**, and an **`integrity`** snapshot from `ensure_integrity_anchor`. When the anchor cannot be read, **`integrity.error`** is the **exception class name only** (`type(e).__name__` — #1721); `str(e)` (paths, parse context) stays in the operator log via **`SanitizeLogFilter`**. **No** `X-API-Key` / Bearer header is required or checked. **`GET /status`** includes the same **`integrity`** object. +- **`GET /health` (always public):** Intended for load balancers and orchestrators. The body is exactly `{"status":"ok"}` (#2077). **No** `X-API-Key` / Bearer header is required or checked. License, `dashboard_transport`, `enterprise_surface`, canonical **`trust_state`** / **`trust_reasons`** / **`output_confidence`**, and **`integrity`** are on authenticated **`GET /status`**. When the anchor cannot be read, **`integrity.error`** is the **exception class name only** (`type(e).__name__` — #1721); `str(e)` stays in the operator log via **`SanitizeLogFilter`**. - **All other HTTP routes** (HTML pages, `GET /status`, `POST /scan`, locale-prefixed dashboard paths such as `GET /en/config`, OpenAPI `/docs`, etc.): when `require_api_key` is true and a key is configured, send **X-API-Key** or **Authorization: Bearer <key>**. **401** = missing or invalid key. **503** = `require_api_key` is true but no key could be resolved (fix config/env). **`main.py --web` refuses to start (exit code 2)** in that misconfiguration so you do not accidentally expose an open dashboard. -- **Rollout:** Inventory API clients (scripts, cron, CI, probes), enable **staging** first with TLS + API key where applicable, then production with a short compatibility window if needed — **[SECURE_BY_DEFAULT_BLOCKERS_AND_MIGRATION.md](ops/SECURE_BY_DEFAULT_BLOCKERS_AND_MIGRATION.md)**. After enabling HTTPS or plaintext HTTP explicitly, use **`GET /status` / `GET /health`** and **`--export-audit-trail`** (`dashboard_transport` in the JSON) to verify posture. +- **Rollout:** Inventory API clients (scripts, cron, CI, probes), enable **staging** first with TLS + API key where applicable, then production with a short compatibility window if needed — **[SECURE_BY_DEFAULT_BLOCKERS_AND_MIGRATION.md](ops/SECURE_BY_DEFAULT_BLOCKERS_AND_MIGRATION.md)**. After enabling HTTPS or plaintext HTTP explicitly, use **`GET /status`** and **`--export-audit-trail`** (`dashboard_transport` in the JSON) to verify posture. **`GET /health`** only reports liveness. - See also [SECURITY.md](../SECURITY.md#optional-api-key-enterprise) and the Configuration section below. ### Web dashboard {#web-dashboard} @@ -341,7 +341,7 @@ The **Start scan** button sends `POST /scan` and triggers a **full audit of all | `GET` | `/findings/csv` | Same session as UTF-8 CSV. String cells that start with `=`, `+`, `-`, `@`, TAB, or CR get a leading `'` (`excel_sanitize_cell`, CWE-1236 / #1723). | | `GET` | `/findings/{session_id}` | Same JSON schema for a specific session. | | `GET` | `/findings/{session_id}/csv` | CSV attachment for that session (same sanitization as `/findings/csv`). | -| `GET` | `/health` | Liveness/readiness for Docker and Kubernetes. Public JSON includes **`integrity`** (class-name-only `error` on fail-soft). | +| `GET` | `/health` | Liveness for Docker and Kubernetes. Public body is exactly `{"status":"ok"}` (#2077). Posture is on `GET /status`. | One process holds **one** `AuditEngine`. `POST /scan`, `/start`, and `/scan_database` claim that slot **when the request is accepted**, not when the background task later starts. A second overlapping start returns **HTTP 409** (`Audit already in progress.`). **HTTP 429** from `rate_limit` is a separate DB-backed cap. Parallel scans need multiple processes or instances. diff --git a/docs/USAGE.pt_BR.md b/docs/USAGE.pt_BR.md index 6d9a7b979..f49ed32c1 100644 --- a/docs/USAGE.pt_BR.md +++ b/docs/USAGE.pt_BR.md @@ -164,7 +164,7 @@ Quando **`DATA_BOAR_OTEL_ENABLED`** for `1` / `true` / `yes` / `on` e o extra op #### Servidor API (`--web`) -**Transporte:** é preciso usar **HTTPS** (certificado + chave PEM na CLI ou em `api` no config) ou **aceitar explicitamente** texto plano com **`--allow-insecure-http`** (ou `api.allow_insecure_http: true`). Caso contrário, `main.py --web` termina com código **2**. **`GET /status`** e **`GET /health`** incluem `dashboard_transport`, **`enterprise_surface`** (transporte + confiança da licença + postura global da chave de API; **RBAC** opcional por rota aparece em **`enterprise_surface.access_surface.rbac`** — **`enabled`** quando `api.rbac` está ligado e o tier permite **`dashboard_rbac`**, senão **`not_implemented`**) e os campos canônicos **`trust_state`** / **`trust_reasons`** / **`output_confidence`** (licença + integridade + transporte — opt-in HTTP nunca fica `trusted` limpo); em HTTP texto plano há **faixa de aviso** nas páginas do dashboard. **`GET /status`** também inclui **`detection_prefilter`** (prontidão do estágio de regex / acelerador: `active`, `name`, `backend`, `tier`, `reason`, `engine`) ao lado de **`runtime_trust`** — só observabilidade; não muda achados (#1411 / #1412). +**Transporte:** é preciso usar **HTTPS** (certificado + chave PEM na CLI ou em `api` no config) ou **aceitar explicitamente** texto plano com **`--allow-insecure-http`** (ou `api.allow_insecure_http: true`). Caso contrário, `main.py --web` termina com código **2**. **`GET /status`** inclui `dashboard_transport`, **`enterprise_surface`** (transporte + confiança da licença + postura global da chave de API; **RBAC** opcional por rota aparece em **`enterprise_surface.access_surface.rbac`** — **`enabled`** quando `api.rbac` está ligado e o tier permite **`dashboard_rbac`**, senão **`not_implemented`**) e os campos canônicos **`trust_state`** / **`trust_reasons`** / **`output_confidence`** (licença + integridade + transporte — opt-in HTTP nunca fica `trusted` limpo); em HTTP texto plano há **faixa de aviso** nas páginas do dashboard. **`GET /status`** também inclui **`detection_prefilter`** (prontidão do estágio de regex / acelerador: `active`, `name`, `backend`, `tier`, `reason`, `engine`) ao lado de **`runtime_trust`** — só observabilidade; não muda achados (#1411 / #1412). ```bash python main.py --config config.yaml --web --https-cert-file server.crt --https-key-file server.key --port 8088 @@ -288,9 +288,9 @@ O Excel é salvo na pasta de downloads do navegador; o mapa de calor PNG é gera - **Autenticação (API key opcional):** Por padrão a API não exige autenticação. Para exigir chave, use `api.require_api_key: true` e `api.api_key` **ou** `api.api_key_from_env: "VAR"` com a variável definida **antes** de subir o processo — passos em **[API_KEY_FROM_ENV_OPERATOR_STEPS.md](ops/API_KEY_FROM_ENV_OPERATOR_STEPS.md)** (EN, operador). **Passo a passo (chave + HTTPS, Let’s Encrypt, cert de lab):** **[SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.pt_BR.md](ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.pt_BR.md)** ([EN](ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.md)). Em produção, prefira **`api_key_from_env`** para não versionar segredo em YAML rastreado. - **WebAuthn (opcional — fase 1 API JSON + fase 1b sessão HTML):** Com **`api.webauthn.enabled: true`**, defina a variável de ambiente indicada por **`api.webauthn.token_secret_from_env`** (padrão **`DATA_BOAR_WEBAUTHN_TOKEN_SECRET`**) **antes** de subir o processo; a falta do segredo faz o processo **encerrar na inicialização**. Habilita endpoints JSON **sem lock-in de vendor** (biblioteca open-source **`webauthn`** no PyPI) em **`/auth/webauthn/`** para registro e autenticação por passkey. **`api.require_api_key` não se aplica** a esses caminhos nem a **`GET /{locale}/login`** (para abrir a página de entrada com chave global ligada). **Bootstrap da primeira passkey (#1553):** registrar a **primeira** credencial **sempre** exige **`X-API-Key`** / Bearer válida (configure **`api.api_key`** ou **`api.api_key_from_env`**). Não há isenção key-free em loopback — reverse proxies no mesmo host costumam apresentar todo cliente como **`127.0.0.1`**. Cerimônias de autenticação (entrar depois que já existe passkey) continuam sem chave nesses caminhos. Alinhe **`api.webauthn.origin`** e **`api.webauthn.rp_id`** com a URL do navegador (HTTPS recomendado). Credenciais ficam no SQLite (`webauthn_credentials`); **`--reset-data` / wipe** apaga essas linhas. **Depois que existir pelo menos uma passkey registrada**, as páginas do painel com prefixo de idioma (**`/`, `/config`, `/reports`, `/assessment`, …**) exigem cookie de sessão WebAuthn válido, exceto **`help`**, **`about`** e **`login`** (GET). Use **`/{locale}/login`** no navegador ( **`/static/webauthn-login.js`** ) para registrar ou entrar; formulários HTML mutáveis (**`POST …/config`**, **`POST …/assessment`**) **sempre** exigem token **CSRF** assinado (**#1231** — independente do gate WebAuthn; chave estável opcional via **`DATA_BOAR_HTML_CSRF_SECRET`**, senão o segredo WebAuthn quando configurado, senão segredo efêmero do processo). RBAC por rota (opcional; item seguinte) pode complementar quando habilitado. Ver **[ADR 0033](adr/ADR-0033-webauthn-open-relying-party-json-endpoints.md)** (EN). - **RBAC (opcional — fase 2, Pro+ `dashboard_rbac`):** Com **`api.rbac.enabled: true`** e tier efetivo que permita **`dashboard_rbac`** (`licensing.effective_tier` no YAML ou claim **`dbtier`** no JWT com enforcement — mesma regra das outras funções com tier), rotas protegidas exigem **papéis**: `admin` (tudo), `dashboard`, `scanner`, `reports_reader`, `config_admin`, e **`audit_logs.read`** para **`GET /logs`** / **`GET /logs/{session_id}`**. **`GET`/`POST /{locale}/config`** exige **`config_admin`** ou **`admin`** (#414). A **chave global de API** (quando presente e correta) recebe **`api.rbac.api_key_roles`**; sessões WebAuthn usam a coluna **`roles_json`** na linha correspondente de `webauthn_credentials` (array JSON de nomes de papel), ou **`api.rbac.default_roles`** quando **`roles_json`** estiver vazio. **401** sem principal; **403** com papéis insuficientes. Quando o RBAC **não** estiver ativo, `/logs` segue a mesma postura de `/findings` / `/report` (ainda sujeito a `api.audit_logs.enabled` / directory e a **`api.require_api_key`** quando essa flag estiver ligada). **`GET /status`** e **`GET /health`** incluem **`enterprise_surface.access_surface.rbac`** (`enabled` vs `not_implemented`). Tier **Community** não habilita RBAC no produto; use **`require_api_key`**, proxy reverso ou rede. Veja o plano interno **`PLAN_DASHBOARD_REPORTS_ACCESS_CONTROL.md`** (na lista *Internal and reference* em `docs/README.md`; GitHub **#86**); **SSO/OIDC** empresarial permanece na fase **3**. -- **`GET /health` (sempre público):** Sonda liveness/readiness; JSON com `status`, resumo **público** de `license`, `dashboard_transport`, `enterprise_surface`, **`trust_state`** / **`trust_reasons`** / **`output_confidence`**, e um snapshot **`integrity`** de `ensure_integrity_anchor`. Quando o âncora não puder ser lido, **`integrity.error`** é **só o nome da classe** da exceção (`type(e).__name__` — #1721); `str(e)` (caminhos, contexto de parse) fica no log do operador via **`SanitizeLogFilter`**. **Não** é necessário (nem checado) `X-API-Key` / Bearer. **`GET /status`** inclui o mesmo objeto **`integrity`**. +- **`GET /health` (sempre público):** Sonda de liveness para balanceadores e orquestradores. O corpo é exatamente `{"status":"ok"}` (#2077). **Não** é necessário (nem checado) `X-API-Key` / Bearer. Licença, `dashboard_transport`, `enterprise_surface`, **`trust_state`** / **`trust_reasons`** / **`output_confidence`** e **`integrity`** ficam no **`GET /status`** autenticado. Quando o âncora não puder ser lido, **`integrity.error`** é **só o nome da classe** da exceção (`type(e).__name__` — #1721); `str(e)` fica no log do operador via **`SanitizeLogFilter`**. - **Demais rotas** (HTML com prefixo de idioma, `GET /status`, `POST /scan`, `GET /en/config`, OpenAPI `/docs`, etc.): com `require_api_key` e chave resolvida, envie **X-API-Key** ou **Authorization: Bearer**. **401** = chave ausente ou inválida. **503** = `require_api_key` true porém nenhuma chave resolvida (corrija config/ambiente). O **`main.py --web` encerra com código 2** nesse caso para não subir dashboard aberto por engano. -- **Rollout:** Inventário de consumidores da API (scripts, cron, CI, probes), **staging** primeiro com TLS + chave quando couber, depois produção com janela curta de compatibilidade se necessário — **[SECURE_BY_DEFAULT_BLOCKERS_AND_MIGRATION.pt_BR.md](ops/SECURE_BY_DEFAULT_BLOCKERS_AND_MIGRATION.pt_BR.md)**. Com HTTPS ou HTTP explícito, use **`GET /status` / `GET /health`** e **`--export-audit-trail`** (campos `dashboard_transport`, e **`maturity_assessment_integrity`** quando aplicável) para auditar postura. +- **Rollout:** Inventário de consumidores da API (scripts, cron, CI, probes), **staging** primeiro com TLS + chave quando couber, depois produção com janela curta de compatibilidade se necessário — **[SECURE_BY_DEFAULT_BLOCKERS_AND_MIGRATION.pt_BR.md](ops/SECURE_BY_DEFAULT_BLOCKERS_AND_MIGRATION.pt_BR.md)**. Com HTTPS ou HTTP explícito, use **`GET /status`** e **`--export-audit-trail`** (campos `dashboard_transport`, e **`maturity_assessment_integrity`** quando aplicável) para auditar postura. - Veja [SECURITY.pt_BR.md](../SECURITY.pt_BR.md) (chave opcional). ### O que o dashboard mostra @@ -330,7 +330,7 @@ O Excel é salvo na pasta de downloads do navegador; o mapa de calor PNG é gera | `GET` | `/findings/csv` | Mesma sessão em CSV UTF-8. Células string que começam com `=`, `+`, `-`, `@`, TAB ou CR recebem `'` à frente (`excel_sanitize_cell`, CWE-1236 / #1723). | | `GET` | `/findings/{session_id}` | Mesmo esquema JSON para uma sessão específica. | | `GET` | `/findings/{session_id}/csv` | CSV da sessão (mesma sanitização que `/findings/csv`). | -| `GET` | `/health` | Sonda de liveness/readiness para Docker e Kubernetes. JSON público inclui **`integrity`** (`error` só com nome de classe no fail-soft). | +| `GET` | `/health` | Liveness para Docker e Kubernetes. Corpo público é exatamente `{"status":"ok"}` (#2077). A postura fica em `GET /status`. | Um processo mantém **um** `AuditEngine`. `POST /scan`, `/start` e `/scan_database` reservam esse slot **quando a requisição é aceita**, não quando a tarefa em background começa. Um segundo início sobreposto responde **HTTP 409** (`Audit already in progress.`). O **HTTP 429** de `rate_limit` é um teto separado (SQLite). Varreduras em paralelo exigem vários processos ou instâncias. diff --git a/docs/plans/PLAN_GRC_INSPIRED_ENTERPRISE_TRUST_ACCELERATORS.md b/docs/plans/PLAN_GRC_INSPIRED_ENTERPRISE_TRUST_ACCELERATORS.md index 58c688c2e..a4bc17af3 100644 --- a/docs/plans/PLAN_GRC_INSPIRED_ENTERPRISE_TRUST_ACCELERATORS.md +++ b/docs/plans/PLAN_GRC_INSPIRED_ENTERPRISE_TRUST_ACCELERATORS.md @@ -40,7 +40,7 @@ We still need a stronger "trust evidence loop" in runtime behavior: **Why now:** creates single-source-of-truth semantics for all later security work. -**Status (S2a wave-1):** ✅ Contract landed in `core/canonical_trust.py` — `trust_state` + `trust_reasons` on `GET /status`, `GET /health`, `--export-audit-trail`, plus stderr line on `--web`. Folds license trust, integrity tamper, and plaintext HTTP opt-in. Report-body metadata / DB audit rows for every scan remain follow-up. +**Status (S2a wave-1):** ✅ Contract landed in `core/canonical_trust.py` — `trust_state` + `trust_reasons` on `GET /status` and `--export-audit-trail` (`GET /health` is liveness only, #2077), plus stderr line on `--web`. Folds license trust, integrity tamper, and plaintext HTTP opt-in. Report-body metadata / DB audit rows for every scan remain follow-up. ### A2 — Output confidence policy diff --git a/tests/test_api_key.py b/tests/test_api_key.py index 4b1298e98..344b57a0d 100644 --- a/tests/test_api_key.py +++ b/tests/test_api_key.py @@ -200,9 +200,7 @@ def test_api_works_without_key_when_require_api_key_false(tmp_path): assert body["output_confidence"] in ("full", "reduced", "minimal") health = client.get("/health") assert health.status_code == 200 - hbody = health.json() - assert hbody["trust_state"] == body["trust_state"] - assert "dashboard_transport" in hbody + assert set(health.json()) == {"status"} finally: routes._config_path = orig_path routes._config = orig_cfg diff --git a/tests/test_licensing.py b/tests/test_licensing.py index 40962a7d5..8cc844d72 100644 --- a/tests/test_licensing.py +++ b/tests/test_licensing.py @@ -467,7 +467,9 @@ def test_health_includes_license(tmp_path, monkeypatch): client = TestClient(routes.app) r = client.get("/health") assert r.status_code == 200 - body = r.json() - assert body.get("status") == "ok" + assert set(r.json()) == {"status"} + detailed = client.get("/status") + assert detailed.status_code == 200 + body = detailed.json() assert "license" in body assert body["license"].get("license_state") == "OPEN" From 9c47f2a0c391571edcf7385591478b0140b0a51d Mon Sep 17 00:00:00 2001 From: Fabio Leitao Date: Mon, 19 Oct 2026 13:27:00 -0300 Subject: [PATCH 09/12] fix(security): stop consumers reading posture from /health (Refs #2077) Probes still use the status code. License and transport readers now use GET /status, including the nogil image check and the operator docs. --- SECURITY.md | 4 ++-- SECURITY.pt_BR.md | 4 ++-- api/locales/en.json | 2 +- api/locales/pt-BR.json | 2 +- docs/LICENSING_SPEC.md | 2 +- docs/LICENSING_SPEC.pt_BR.md | 2 +- docs/USAGE.pt_BR.md | 2 +- docs/ops/API_KEY_FROM_ENV_OPERATOR_STEPS.md | 2 +- .../ops/API_KEY_FROM_ENV_OPERATOR_STEPS.pt_BR.md | 2 +- .../ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.md | 8 ++++---- ...ECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.pt_BR.md | 8 ++++---- docs/plans/PLANS_TODO.md | 4 ++-- .../PLAN_DASHBOARD_REPORTS_ACCESS_CONTROL.md | 2 +- ...GRC_INSPIRED_ENTERPRISE_TRUST_ACCELERATORS.md | 2 +- scripts/build_locale_catalogs.py | 8 ++++---- scripts/docker/build-nogil-local.sh | 16 +++++++++++++--- 16 files changed, 40 insertions(+), 30 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index cef9aafca..316b36100 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -198,13 +198,13 @@ The application adds the following headers to all web and API responses by defau - **Permissions-Policy** – disables browser features not needed by the app (camera, microphone, geolocation, etc.). - **Strict-Transport-Security (HSTS)** – set only when the request is considered HTTPS (direct or via `X-Forwarded-Proto: https` from a trusted proxy), so HTTP-only deployments are not locked out. When present, it uses `max-age=31536000; includeSubDomains; preload`. -When the app is behind a reverse proxy (e.g. nginx, Caddy, load balancer), ensure the proxy sets **X-Forwarded-Proto: https** for TLS-terminated requests so HSTS is applied correctly. Configure **`api.trusted_proxy_cidrs`** to the CIDR(s) of the **direct** proxy peer; without a match, `X-Forwarded-*` headers are ignored (fail-safe). When the peer matches and the forwarded proto is trusted `https`, the dashboard suppresses the plaintext-risk banner for that request and **`GET /status`** / **`GET /health`** expose **`effective_external_transport`** with `tls_termination: trusted_proxy` — process-level **`dashboard_transport`** still reports upstream HTTP honestly. Do not enable HSTS at the app layer for plain HTTP; the proxy can add HSTS when serving over HTTPS. +When the app is behind a reverse proxy (e.g. nginx, Caddy, load balancer), ensure the proxy sets **X-Forwarded-Proto: https** for TLS-terminated requests so HSTS is applied correctly. Configure **`api.trusted_proxy_cidrs`** to the CIDR(s) of the **direct** proxy peer; without a match, `X-Forwarded-*` headers are ignored (fail-safe). When the peer matches and the forwarded proto is trusted `https`, the dashboard suppresses the plaintext-risk banner for that request and **`GET /status`** exposes **`effective_external_transport`** with `tls_termination: trusted_proxy` — process-level **`dashboard_transport`** still reports upstream HTTP honestly. Do not enable HSTS at the app layer for plain HTTP; the proxy can add HSTS when serving over HTTPS. ## Optional API key (enterprise) The API does not implement authentication by default; secure the app at the reverse proxy or network level when exposed. For enterprises that want a simple shared-secret gate without changing the “secure at proxy” model, the application supports an **optional API key**: -- In config, set `api.require_api_key: true` and either `api.api_key` (literal — avoid committing secrets) or `api.api_key_from_env: "VAR"` (read key from environment at startup). When enabled, **GET /health** stays **unauthenticated** on purpose: it returns liveness JSON (`status`, public `license` summary, `dashboard_transport`) for probes. **Every other route** must include **X-API-Key** or **Authorization: Bearer <key>** when a key is successfully resolved from config/env. **401** = missing or wrong key. **503** = `require_api_key` is true but no key could be resolved (misconfiguration). **`main.py --web` exits with code 2** before listening if the key is required but missing, so you do not accidentally run an open API. +- In config, set `api.require_api_key: true` and either `api.api_key` (literal — avoid committing secrets) or `api.api_key_from_env: "VAR"` (read key from environment at startup). When enabled, **GET /health** stays **unauthenticated** on purpose: it returns liveness JSON `{"status":"ok"}` for probes (#2077). License, trust, and `dashboard_transport` are on **GET /status**. **Every other route** must include **X-API-Key** or **Authorization: Bearer <key>** when a key is successfully resolved from config/env. **401** = missing or wrong key. **503** = `require_api_key` is true but no key could be resolved (misconfiguration). **`main.py --web` exits with code 2** before listening if the key is required but missing, so you do not accidentally run an open API. - **Good practice:** Use a strong, random key and store it in an environment variable (e.g. `api_key_from_env: "AUDIT_API_KEY"`). Do not log the key or commit it to version control. This is a simple gate only; for full authentication and authorization, continue to use the reverse proxy or an identity provider. - **Concrete operator steps** (shell, systemd, Docker/K8s patterns, `curl` checks, synthetic example key): [API_KEY_FROM_ENV_OPERATOR_STEPS.md](docs/ops/API_KEY_FROM_ENV_OPERATOR_STEPS.md). For ordering (inventory clients, staging first, monitor `dashboard_transport` and audit export), see [SECURE_BY_DEFAULT_BLOCKERS_AND_MIGRATION.md](docs/ops/SECURE_BY_DEFAULT_BLOCKERS_AND_MIGRATION.md). - **End-to-end technician guide (API key + TLS paths, Let’s Encrypt, lab self-signed, Docker):** [SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.md](docs/ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.md) ([pt-BR](docs/ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.pt_BR.md)). diff --git a/SECURITY.pt_BR.md b/SECURITY.pt_BR.md index 84c9ad973..460f45a48 100644 --- a/SECURITY.pt_BR.md +++ b/SECURITY.pt_BR.md @@ -197,13 +197,13 @@ A aplicação adiciona os seguintes cabeçalhos a todas as respostas web e da AP - **Permissions-Policy** — desabilita recursos do navegador não usados pelo app (câmera, microfone, geolocalização, etc.). - **Strict-Transport-Security (HSTS)** — definido apenas quando a requisição é considerada HTTPS (direta ou via `X-Forwarded-Proto: https` de um proxy confiável), para que implantações somente HTTP não fiquem bloqueadas. Quando presente, usa `max-age=31536000; includeSubDomains; preload`. -Quando o app estiver atrás de um proxy reverso (ex.: nginx, Caddy, load balancer), garanta que o proxy defina **X-Forwarded-Proto: https** para requisições com TLS terminado para que o HSTS seja aplicado corretamente. Configure **`api.trusted_proxy_cidrs`** com o(s) CIDR(s) do peer **direto** do proxy; sem correspondência, cabeçalhos `X-Forwarded-*` são ignorados (fail-safe). Quando o peer corresponde e o proto forwarded confiável é `https`, o painel suprime o banner de risco de texto claro nessa requisição e **`GET /status`** / **`GET /health`** expõem **`effective_external_transport`** com `tls_termination: trusted_proxy` — o **`dashboard_transport`** a nível de processo continua reportando HTTP upstream com honestidade. Não habilite HSTS na camada do app para HTTP puro; o proxy pode adicionar HSTS ao servir via HTTPS. +Quando o app estiver atrás de um proxy reverso (ex.: nginx, Caddy, load balancer), garanta que o proxy defina **X-Forwarded-Proto: https** para requisições com TLS terminado para que o HSTS seja aplicado corretamente. Configure **`api.trusted_proxy_cidrs`** com o(s) CIDR(s) do peer **direto** do proxy; sem correspondência, cabeçalhos `X-Forwarded-*` são ignorados (fail-safe). Quando o peer corresponde e o proto forwarded confiável é `https`, o painel suprime o banner de risco de texto claro nessa requisição e **`GET /status`** expõe **`effective_external_transport`** com `tls_termination: trusted_proxy` — o **`dashboard_transport`** a nível de processo continua reportando HTTP upstream com honestidade. Não habilite HSTS na camada do app para HTTP puro; o proxy pode adicionar HSTS ao servir via HTTPS. ## Chave de API opcional (empresarial) A API não implementa autenticação por padrão; proteja o app no proxy reverso ou no nível de rede quando exposto. Para empresas que desejam um gate de segredo compartilhado simples sem alterar o modelo “proteger no proxy”, a aplicação suporta uma **chave de API opcional**: -- No config, defina `api.require_api_key: true` e `api.api_key` (literal — evite commit de segredo) ou `api.api_key_from_env: "VAR"` (lê a chave do ambiente na subida do processo). Quando habilitado, **GET /health** permanece **sem autenticação** de propósito: devolve JSON de liveness (`status`, resumo **público** de `license`, `dashboard_transport`) para probes. **Todas as outras rotas** precisam de **X-API-Key** ou **Authorization: Bearer** quando a chave foi resolvida a partir do config/ambiente. **401** = chave ausente ou inválida. **503** = `require_api_key` true porém nenhuma chave resolvida (misconfig). O **`main.py --web` encerra com código 2** antes de escutar se a chave é obrigatória e está ausente, evitando API aberta por engano. +- No config, defina `api.require_api_key: true` e `api.api_key` (literal — evite commit de segredo) ou `api.api_key_from_env: "VAR"` (lê a chave do ambiente na subida do processo). Quando habilitado, **GET /health** permanece **sem autenticação** de propósito: devolve JSON de liveness `{"status":"ok"}` para probes (#2077). Licença, confiança e `dashboard_transport` ficam em **GET /status**. **Todas as outras rotas** precisam de **X-API-Key** ou **Authorization: Bearer** quando a chave foi resolvida a partir do config/ambiente. **401** = chave ausente ou inválida. **503** = `require_api_key` true porém nenhuma chave resolvida (misconfig). O **`main.py --web` encerra com código 2** antes de escutar se a chave é obrigatória e está ausente, evitando API aberta por engano. - **Boa prática:** Use uma chave forte e aleatória e armazene-a em uma variável de ambiente (ex.: `api_key_from_env: "AUDIT_API_KEY"`). Não registre em log nem faça commit da chave no controle de versão. Isso é apenas um gate simples; para autenticação e autorização completas, use o proxy reverso ou um provedor de identidade. - **Passos concretos (EN, para evitar ambiguidade):** nome da variável no YAML vs segredo no SO, precedência, exemplos `curl` — [API_KEY_FROM_ENV_OPERATOR_STEPS.md](docs/ops/API_KEY_FROM_ENV_OPERATOR_STEPS.md). Ordem sugerida (inventário, staging, monitorar `dashboard_transport` / audit export): [SECURE_BY_DEFAULT_BLOCKERS_AND_MIGRATION.pt_BR.md](docs/ops/SECURE_BY_DEFAULT_BLOCKERS_AND_MIGRATION.pt_BR.md). - **Guia ponta a ponta para técnico (chave de API + caminhos TLS, Let’s Encrypt, autoassinado em lab, Docker):** [SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.pt_BR.md](docs/ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.pt_BR.md) ([EN](docs/ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.md)). diff --git a/api/locales/en.json b/api/locales/en.json index e73176327..c41673ff2 100644 --- a/api/locales/en.json +++ b/api/locales/en.json @@ -181,7 +181,7 @@ "run_cli_demo": "Zero-config demo (synthetic config + corpus under /tmp/data_boar_demo/; ignores config.yaml in the CWD; loopback dashboard; GET /logs/{session_id} when audit_logs enabled):", "run_cli_api": "Start the API / dashBOARd (no scan until you use the browser or HTTP):", "run_cli_tls": "You must either terminate TLS at the process (--https-cert-file + --https-key-file, or the same under api: in config) or explicitly accept plaintext with --allow-insecure-http (or api.allow_insecure_http: true). The official Docker image passes --allow-insecure-http by default; mount certs and remove it for HTTPS.", - "run_cli_bind": "Bind order when using --web: --host overrides api.host in config and API_HOST; if none are set, the default is 127.0.0.1 (Docker image often sets API_HOST=0.0.0.0). Port: api.port in config vs --port. Transport: GET /status and GET /health include dashboard_transport — see docs/USAGE.md.", + "run_cli_bind": "Bind order when using --web: --host overrides api.host in config and API_HOST; if none are set, the default is 127.0.0.1 (Docker image often sets API_HOST=0.0.0.0). Port: api.port in config vs --port. Transport: GET /status includes dashboard_transport; GET /health is liveness only — see docs/USAGE.md.", "run_cli_danger": "Dangerous maintenance (lab/demo only): --reset-data wipes SQLite sessions/findings and deletes generated reports under report.output_dir. See SECURITY.md before using in production.", "run_cli_integrity": "After an official pip/pipx upgrade, re-baseline integrity hashes with --reconcile-integrity-anchor --confirm-upgrade-to= (must match the running package; does not auto-reconcile on semver).", "run_cli_audit": "Audit trail export (CLI only): --export-audit-trail or --export-audit-trail path.json writes a JSON snapshot (wipe log, session summary; future: integrity rows). Does not modify the database. Omit the path or use - for stdout.", diff --git a/api/locales/pt-BR.json b/api/locales/pt-BR.json index 4fe2a151f..243632455 100644 --- a/api/locales/pt-BR.json +++ b/api/locales/pt-BR.json @@ -181,7 +181,7 @@ "run_cli_demo": "Demonstração zero-config (config + corpus sintéticos em /tmp/data_boar_demo/; ignora o config.yaml do diretório atual; dashboard em loopback; GET /logs/{session_id} com audit_logs habilitado):", "run_cli_api": "Subir a API / dashBOARd (sem varredura até usar o navegador ou HTTP):", "run_cli_tls": "É preciso terminar TLS no processo (--https-cert-file + --https-key-file, ou o mesmo em api: na config) ou aceitar explicitamente texto claro com --allow-insecure-http (ou api.allow_insecure_http: true). A imagem Docker oficial passa --allow-insecure-http por padrão; monte certificados e remova para HTTPS.", - "run_cli_bind": "Ordem de bind com --web: --host sobrescreve api.host na config e API_HOST; se nada estiver definido, o padrão é 127.0.0.1 (a imagem Docker costuma definir API_HOST=0.0.0.0). Porta: api.port na config vs --port. Transporte: GET /status e GET /health incluem dashboard_transport — veja docs/USAGE.md.", + "run_cli_bind": "Ordem de bind com --web: --host sobrescreve api.host na config e API_HOST; se nada estiver definido, o padrão é 127.0.0.1 (a imagem Docker costuma definir API_HOST=0.0.0.0). Porta: api.port na config vs --port. Transporte: GET /status inclui dashboard_transport; GET /health é só liveness — veja docs/USAGE.md.", "run_cli_danger": "Manutenção perigosa (só lab/demo): --reset-data apaga sessões/achados no SQLite e remove relatórios gerados em report.output_dir. Veja SECURITY.md antes de usar em produção.", "run_cli_integrity": "Depois de um upgrade oficial pip/pipx, re-baseline os hashes de integridade com --reconcile-integrity-anchor --confirm-upgrade-to= (tem que casar com o pacote em execução; não reconcilia sozinho por semver).", "run_cli_audit": "Exportação de trilha de auditoria (só CLI): --export-audit-trail ou --export-audit-trail caminho.json grava um snapshot JSON (log de wipe, resumo de sessão; futuro: linhas de integridade). Não modifica o banco. Omita o caminho ou use - para stdout.", diff --git a/docs/LICENSING_SPEC.md b/docs/LICENSING_SPEC.md index 61774cde4..5b67873d2 100644 --- a/docs/LICENSING_SPEC.md +++ b/docs/LICENSING_SPEC.md @@ -208,7 +208,7 @@ which optional capabilities are available at runtime and which installer extras | Claim / control idea | Purpose | | --- | --- | -| `dbtier` (`standard`, `pro`, `partner`, `enterprise`) | Single source of truth for entitlement tier in runtime, report info, and audit trail. **Implementation (2026-04):** when `licensing.mode` is **enforced** and the token is **VALID** or **GRACE**, the verify path copies this claim into `LicenseContext.dbtier` and `GET /health` → `license.dbtier`; dashboard code maps it to internal tier for **optional** feature gates (e.g. maturity POC) alongside lab-only `licensing.effective_tier` in YAML. **Maturity POC gate:** `docs/plans/completed/PLAN_MATURITY_SELF_ASSESSMENT_GRC_QUESTIONNAIRE.md` § *Next slice*; regression: `tests/test_api_assessment_poc.py` (`test_assessment_enforced_jwt_dbtier_*`). | +| `dbtier` (`standard`, `pro`, `partner`, `enterprise`) | Single source of truth for entitlement tier in runtime, report info, and audit trail. **Implementation (2026-04):** when `licensing.mode` is **enforced** and the token is **VALID** or **GRACE**, the verify path copies this claim into `LicenseContext.dbtier` and `GET /status` → `license.dbtier` (`GET /health` is liveness only, #2077); dashboard code maps it to internal tier for **optional** feature gates (e.g. maturity POC) alongside lab-only `licensing.effective_tier` in YAML. **Maturity POC gate:** `docs/plans/completed/PLAN_MATURITY_SELF_ASSESSMENT_GRC_QUESTIONNAIRE.md` § *Next slice*; regression: `tests/test_api_assessment_poc.py` (`test_assessment_enforced_jwt_dbtier_*`). | | `dbmax_workers` (int) | **Implemented (#551/#853)** — cap on **parallel scan workers** (≈ concurrent targets). Band defaults: Community **2** · Std **2** · Pro **4** · Pro+ **8** (`Tier.PRO_PLUS`; a `dbmax_workers` claim still wins over the tier default) · Enterprise **unlimited** · Partner **contract-driven** — canonical ladder in [SUBSCRIPTION_TIERS.md](SUBSCRIPTION_TIERS.md) (not a price list). | | `dbmax_targets` (int) | Optional cap on **configured targets** per session or deployment envelope; **0** may mean unlimited when contract allows. | | `dbfeatures` (string list) | Explicit feature flags independent from tier defaults (e.g. `compressed_scan`, `content_type_cloaking`, future `ai_heuristics_plus`). | diff --git a/docs/LICENSING_SPEC.pt_BR.md b/docs/LICENSING_SPEC.pt_BR.md index 90d5bdf52..98d20fe22 100644 --- a/docs/LICENSING_SPEC.pt_BR.md +++ b/docs/LICENSING_SPEC.pt_BR.md @@ -212,7 +212,7 @@ quais capacidades opcionais ficam disponíveis em runtime e quais extras de inst | Ideia de claim / controle | Propósito | | --- | --- | -| `dbtier` (`standard`, `pro`, `partner`, `enterprise`) | Fonte única de verdade para o tier de direito de uso em runtime, info do relatório e trilha de auditoria. **Implementação (2026-04):** quando `licensing.mode` é **enforced** e o token é **VALID** ou **GRACE**, o caminho de verificação copia este claim para `LicenseContext.dbtier` e `GET /health` → `license.dbtier`; o código do dashboard o mapeia para o tier interno em gates de recurso **opcionais** (ex.: POC de maturidade) ao lado de `licensing.effective_tier` (só de lab) no YAML. **Gate do POC de maturidade:** `docs/plans/completed/PLAN_MATURITY_SELF_ASSESSMENT_GRC_QUESTIONNAIRE.md` § *Next slice*; regressão: `tests/test_api_assessment_poc.py` (`test_assessment_enforced_jwt_dbtier_*`). | +| `dbtier` (`standard`, `pro`, `partner`, `enterprise`) | Fonte única de verdade para o tier de direito de uso em runtime, info do relatório e trilha de auditoria. **Implementação (2026-04):** quando `licensing.mode` é **enforced** e o token é **VALID** ou **GRACE**, o caminho de verificação copia este claim para `LicenseContext.dbtier` e `GET /status` → `license.dbtier` (`GET /health` é só liveness, #2077); o código do dashboard o mapeia para o tier interno em gates de recurso **opcionais** (ex.: POC de maturidade) ao lado de `licensing.effective_tier` (só de lab) no YAML. **Gate do POC de maturidade:** `docs/plans/completed/PLAN_MATURITY_SELF_ASSESSMENT_GRC_QUESTIONNAIRE.md` § *Next slice*; regressão: `tests/test_api_assessment_poc.py` (`test_assessment_enforced_jwt_dbtier_*`). | | `dbmax_workers` (int) | **Implementado (#551/#853)** — limite de **workers de scan em paralelo** (≈ alvos concorrentes). Padrões por faixa: Community **2** · Std **2** · Pro **4** · Pro+ **8** (`Tier.PRO_PLUS`; um claim `dbmax_workers` ainda vence o padrão do tier) · Enterprise **ilimitado** · Partner **por contrato** — escada canônica em [SUBSCRIPTION_TIERS.pt_BR.md](SUBSCRIPTION_TIERS.pt_BR.md) (não é lista de preços). | | `dbmax_targets` (int) | Limite opcional de **alvos configurados** por sessão ou envelope de deployment; **0** pode significar ilimitado quando o contrato permite. | | `dbfeatures` (lista de strings) | Flags de recurso explícitas, independentes dos padrões do tier (ex.: `compressed_scan`, `content_type_cloaking`, futuro `ai_heuristics_plus`). | diff --git a/docs/USAGE.pt_BR.md b/docs/USAGE.pt_BR.md index f49ed32c1..3b0e2e853 100644 --- a/docs/USAGE.pt_BR.md +++ b/docs/USAGE.pt_BR.md @@ -284,7 +284,7 @@ O Excel é salvo na pasta de downloads do navegador; o mapa de calor PNG é gera - **About:** `http://:/en/about` — aplicação, versão, autor e licença (conforme LICENSE) - **Autoavaliação (POC):** `http://:/en/assessment` — placeholder quando **`api.maturity_self_assessment_poc_enabled`** está ligado; **404** se desligado. **Tier:** `licensing.effective_tier` no YAML para lab, ou claim **`dbtier`** no JWT com `licensing.mode: enforced` (**Pro+** para esta funcionalidade). Com enforcement ligado e JWT de licença válido, **`dbtier` prevalece sobre `effective_tier` no YAML** neste gate (mesma regra das outras funcionalidades do painel com tier). Opcional **`api.maturity_assessment_pack_path`** para YAML do questionário (`core/maturity_assessment/pack.py`); **`scores`** opcionais por resposta no YAML alimentam o **total da rubrica** no resumo pós-envio. Opcional **`DATA_BOAR_MATURITY_INTEGRITY_SECRET`** (ou **`api.maturity_integrity_secret_from_env`**) para **HMAC-SHA256 por linha** (integridade / anti-adulteração casual; **não** é criptografia). **`GET /status`** expõe **`maturity_assessment_integrity`** para demonstrar o resumo. Após **POST**, o redirect traz **`saved=1`** e **`batch=`**; o HTML mostra **resumo do envio** (linhas gravadas, pontuação da rubrica quando configurada, totais HMAC daquele batch) e, com respostas no SQLite, uma tabela de **envios recentes** por **batch** (mais recente primeiro). **Exportação:** `GET /{locale}/assessment/export?batch=&format=csv` ou `format=md` devolve **anexo** (download); não há exportação automática em disco no produto — salve pelo navegador ou redirecione `curl` para arquivo. Com **`api.require_api_key`** ativo, envie **`X-API-Key`** ou **`Authorization: Bearer`** como nas outras rotas do painel. Sem texto proprietário do questionário no repositório público. - **Swagger UI:** `http://:/docs` — documentação interativa da API (OpenAPI) -- **Health:** `http://:/health` — sonda de liveness/readiness para Docker/Kubernetes (sempre público mesmo com API key exigida) +- **Health:** `http://:/health` — sonda de liveness para Docker/Kubernetes (sempre público mesmo com API key exigida). O corpo é só `{"status":"ok"}` (#2077). - **Autenticação (API key opcional):** Por padrão a API não exige autenticação. Para exigir chave, use `api.require_api_key: true` e `api.api_key` **ou** `api.api_key_from_env: "VAR"` com a variável definida **antes** de subir o processo — passos em **[API_KEY_FROM_ENV_OPERATOR_STEPS.md](ops/API_KEY_FROM_ENV_OPERATOR_STEPS.md)** (EN, operador). **Passo a passo (chave + HTTPS, Let’s Encrypt, cert de lab):** **[SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.pt_BR.md](ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.pt_BR.md)** ([EN](ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.md)). Em produção, prefira **`api_key_from_env`** para não versionar segredo em YAML rastreado. - **WebAuthn (opcional — fase 1 API JSON + fase 1b sessão HTML):** Com **`api.webauthn.enabled: true`**, defina a variável de ambiente indicada por **`api.webauthn.token_secret_from_env`** (padrão **`DATA_BOAR_WEBAUTHN_TOKEN_SECRET`**) **antes** de subir o processo; a falta do segredo faz o processo **encerrar na inicialização**. Habilita endpoints JSON **sem lock-in de vendor** (biblioteca open-source **`webauthn`** no PyPI) em **`/auth/webauthn/`** para registro e autenticação por passkey. **`api.require_api_key` não se aplica** a esses caminhos nem a **`GET /{locale}/login`** (para abrir a página de entrada com chave global ligada). **Bootstrap da primeira passkey (#1553):** registrar a **primeira** credencial **sempre** exige **`X-API-Key`** / Bearer válida (configure **`api.api_key`** ou **`api.api_key_from_env`**). Não há isenção key-free em loopback — reverse proxies no mesmo host costumam apresentar todo cliente como **`127.0.0.1`**. Cerimônias de autenticação (entrar depois que já existe passkey) continuam sem chave nesses caminhos. Alinhe **`api.webauthn.origin`** e **`api.webauthn.rp_id`** com a URL do navegador (HTTPS recomendado). Credenciais ficam no SQLite (`webauthn_credentials`); **`--reset-data` / wipe** apaga essas linhas. **Depois que existir pelo menos uma passkey registrada**, as páginas do painel com prefixo de idioma (**`/`, `/config`, `/reports`, `/assessment`, …**) exigem cookie de sessão WebAuthn válido, exceto **`help`**, **`about`** e **`login`** (GET). Use **`/{locale}/login`** no navegador ( **`/static/webauthn-login.js`** ) para registrar ou entrar; formulários HTML mutáveis (**`POST …/config`**, **`POST …/assessment`**) **sempre** exigem token **CSRF** assinado (**#1231** — independente do gate WebAuthn; chave estável opcional via **`DATA_BOAR_HTML_CSRF_SECRET`**, senão o segredo WebAuthn quando configurado, senão segredo efêmero do processo). RBAC por rota (opcional; item seguinte) pode complementar quando habilitado. Ver **[ADR 0033](adr/ADR-0033-webauthn-open-relying-party-json-endpoints.md)** (EN). - **RBAC (opcional — fase 2, Pro+ `dashboard_rbac`):** Com **`api.rbac.enabled: true`** e tier efetivo que permita **`dashboard_rbac`** (`licensing.effective_tier` no YAML ou claim **`dbtier`** no JWT com enforcement — mesma regra das outras funções com tier), rotas protegidas exigem **papéis**: `admin` (tudo), `dashboard`, `scanner`, `reports_reader`, `config_admin`, e **`audit_logs.read`** para **`GET /logs`** / **`GET /logs/{session_id}`**. **`GET`/`POST /{locale}/config`** exige **`config_admin`** ou **`admin`** (#414). A **chave global de API** (quando presente e correta) recebe **`api.rbac.api_key_roles`**; sessões WebAuthn usam a coluna **`roles_json`** na linha correspondente de `webauthn_credentials` (array JSON de nomes de papel), ou **`api.rbac.default_roles`** quando **`roles_json`** estiver vazio. **401** sem principal; **403** com papéis insuficientes. Quando o RBAC **não** estiver ativo, `/logs` segue a mesma postura de `/findings` / `/report` (ainda sujeito a `api.audit_logs.enabled` / directory e a **`api.require_api_key`** quando essa flag estiver ligada). **`GET /status`** e **`GET /health`** incluem **`enterprise_surface.access_surface.rbac`** (`enabled` vs `not_implemented`). Tier **Community** não habilita RBAC no produto; use **`require_api_key`**, proxy reverso ou rede. Veja o plano interno **`PLAN_DASHBOARD_REPORTS_ACCESS_CONTROL.md`** (na lista *Internal and reference* em `docs/README.md`; GitHub **#86**); **SSO/OIDC** empresarial permanece na fase **3**. diff --git a/docs/ops/API_KEY_FROM_ENV_OPERATOR_STEPS.md b/docs/ops/API_KEY_FROM_ENV_OPERATOR_STEPS.md index 3299c1701..7fb84be3f 100644 --- a/docs/ops/API_KEY_FROM_ENV_OPERATOR_STEPS.md +++ b/docs/ops/API_KEY_FROM_ENV_OPERATOR_STEPS.md @@ -110,7 +110,7 @@ Expect **200** (or another success), not 401. 1. **Inventory** callers of the API (scripts, cron, CI, probes, internal dashboards). If the list is empty, you still run the checks so the **first** production user is not discovering 401/503 surprises cold. 1. **Staging:** set `api.require_api_key: true`, `api.api_key_from_env: "AUDIT_API_KEY"` (or your name), export the variable, and use **HTTPS** (`--https-cert-file` / `--https-key-file` or TLS at the reverse proxy) where applicable. Confirm **`GET /health`** without a key (200) vs **`GET /status`** without a key (**401**). 1. **Clients:** add **`X-API-Key`** or **`Authorization: Bearer`** to every non-health request; pin **TLS trust** (enterprise CA, `mkcert` chain in lab only, etc.). -1. **Production:** roll out with a **short** compatibility window only if needed (e.g. plaintext **`--allow-insecure-http`** or HTTP behind TLS-terminating proxy). **Phase it out** using **`GET /status` / `GET /health`** (`dashboard_transport`) and **`python main.py --export-audit-trail`** (same field in JSON) so insecure posture stays traceable. +1. **Production:** roll out with a **short** compatibility window only if needed (e.g. plaintext **`--allow-insecure-http`** or HTTP behind TLS-terminating proxy). **Phase it out** using **`GET /status`** (`dashboard_transport`; **`GET /health`** is liveness only) and **`python main.py --export-audit-trail`** (same field in JSON) so insecure posture stays traceable. 1. Full narrative: **[SECURE_BY_DEFAULT_BLOCKERS_AND_MIGRATION.md](SECURE_BY_DEFAULT_BLOCKERS_AND_MIGRATION.md)**. --- diff --git a/docs/ops/API_KEY_FROM_ENV_OPERATOR_STEPS.pt_BR.md b/docs/ops/API_KEY_FROM_ENV_OPERATOR_STEPS.pt_BR.md index 522c91e8c..4420fe223 100644 --- a/docs/ops/API_KEY_FROM_ENV_OPERATOR_STEPS.pt_BR.md +++ b/docs/ops/API_KEY_FROM_ENV_OPERATOR_STEPS.pt_BR.md @@ -112,7 +112,7 @@ Esperado: **200** (ou outro sucesso), não 401. 1. **Inventariar** os consumidores da API (scripts, cron, CI, probes, dashboards internos). Se a lista estiver vazia, execute os checks assim mesmo para que o primeiro usuário em produção não encontre surpresas de 401/503. 2. **Homologação:** defina `api.require_api_key: true`, `api.api_key_from_env: "AUDIT_API_KEY"` (ou seu nome), exporte a variável e use **HTTPS** (`--https-cert-file` / `--https-key-file` ou TLS no proxy reverso) onde aplicável. Confirme **`GET /health`** sem chave (200) vs **`GET /status`** sem chave (**401**). 3. **Clientes:** adicione **`X-API-Key`** ou **`Authorization: Bearer`** a todas as requisições que não sejam de health; fixe a **confiança TLS** (CA empresarial, `mkcert` somente em lab, etc.). -4. **Produção:** lance com uma **curta** janela de compatibilidade apenas se necessário (ex.: `--allow-insecure-http` em texto claro ou HTTP atrás de proxy TLS-terminador). **Encerre-a** usando **`GET /status` / `GET /health`** (`dashboard_transport`) e **`python main.py --export-audit-trail`** para que a postura insegura fique rastreável. +4. **Produção:** lance com uma **curta** janela de compatibilidade apenas se necessário (ex.: `--allow-insecure-http` em texto claro ou HTTP atrás de proxy TLS-terminador). **Encerre-a** usando **`GET /status`** (`dashboard_transport`; **`GET /health`** é só liveness) e **`python main.py --export-audit-trail`** para que a postura insegura fique rastreável. 5. Narrativa completa: **[SECURE_BY_DEFAULT_BLOCKERS_AND_MIGRATION.md](SECURE_BY_DEFAULT_BLOCKERS_AND_MIGRATION.md)**. --- diff --git a/docs/ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.md b/docs/ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.md index 734783e66..7b462e3d8 100644 --- a/docs/ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.md +++ b/docs/ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.md @@ -17,7 +17,7 @@ 1. When **`api.require_api_key: true`**, the process must have a **resolved** secret: either non-empty **`api.api_key`** in config, or **`api.api_key_from_env: "VAR"`** with **`VAR` set in the environment before startup**. 1. **`main.py --web`** **exits with code 2** if `require_api_key` is true but **no key** can be resolved (prevents an accidentally open dashboard). 1. **Non-loopback bind** (`0.0.0.0`, LAN IP, `::`): **`main.py --web` exits 2** unless **`api.require_api_key: true`** and a key is **resolved**. A configured **`api.api_key`** / env key or a WebAuthn token secret **without** `require_api_key` does **not** count — JSON routes (`GET /findings`, `POST /scan`, config) stay unauthenticated (#1714). Bind **`127.0.0.1`** for lab without a key, or set **`require_api_key: true`** before publishing a container port. -1. **`GET /health`** is **never** authenticated (probes and load balancers stay simple). It returns JSON including `status`, a public `license` summary, and `dashboard_transport`. +1. **`GET /health`** is **never** authenticated (probes and load balancers stay simple). The body is exactly `{"status":"ok"}`. License, trust, and `dashboard_transport` are on authenticated **`GET /status`** (#2077). 1. **All other routes** (HTML pages, `GET /status`, `POST /scan`, `GET /config`, OpenAPI `/docs`, …) require a valid key when enforcement is active: send **`X-API-Key: `** or **`Authorization: Bearer `**. 1. **401** = missing or wrong key. **503** = `require_api_key` is true but the key could not be resolved at runtime (misconfiguration); fix env/YAML and restart. @@ -120,7 +120,7 @@ api: - "dashboard.example.com" ``` -With a matching peer **and** trusted `X-Forwarded-Proto: https`, the dashboard suppresses the red plaintext-risk banner and reports request-scoped **`effective_external_transport`** (`tls_termination: trusted_proxy`) on **`GET /status`** / **`GET /health`**. Process-level **`dashboard_transport`** stays `mode: http` / `tls_active: false` — that is intentional honesty about the Uvicorn listener, not a claim of native app TLS. See [#1515](https://github.com/DataBoar/data-boar/issues/1515) / [PLAN_DASHBOARD_TRUSTED_PROXY_TLS.md](../plans/PLAN_DASHBOARD_TRUSTED_PROXY_TLS.md). +With a matching peer **and** trusted `X-Forwarded-Proto: https`, the dashboard suppresses the red plaintext-risk banner and reports request-scoped **`effective_external_transport`** (`tls_termination: trusted_proxy`) on **`GET /status`**. Process-level **`dashboard_transport`** stays `mode: http` / `tls_active: false` — that is intentional honesty about the Uvicorn listener, not a claim of native app TLS. See [#1515](https://github.com/DataBoar/data-boar/issues/1515) / [PLAN_DASHBOARD_TRUSTED_PROXY_TLS.md](../plans/PLAN_DASHBOARD_TRUSTED_PROXY_TLS.md). See [deploy/DEPLOY.md](../deploy/DEPLOY.md) ([pt-BR](../deploy/DEPLOY.pt_BR.md)) and root **SECURITY.md** for hardening context. @@ -151,7 +151,7 @@ api: # - "old-cert-still-valid-during-rotation…" ``` -**3.** Open **`https://:/`**. Use **`GET /status`** or **`GET /health`** and check **`dashboard_transport`** in JSON (`mode`, `tls_active`, `summary`, optional nested **`tls_posture`**) **and** canonical **`trust_state`** / **`trust_reasons`** / **`output_confidence`**. +**3.** Open **`https://:/`**. Use **`GET /status`** and check **`dashboard_transport`** in JSON (`mode`, `tls_active`, `summary`, optional nested **`tls_posture`**) **and** canonical **`trust_state`** / **`trust_reasons`** / **`output_confidence`**. **4.** Optional evidence: **`python main.py --export-audit-trail -`** (without `--web`) includes **`dashboard_transport`**, **`trust_state`**, and **`enterprise_surface`** in the exported JSON for governance snapshots. @@ -197,7 +197,7 @@ Exact Compose/Kubernetes patterns live next to your orchestration files; see [de | Check | What to look for | | ----- | ---------------- | -| **Runtime** | `GET /status` and `GET /health` → **`dashboard_transport`** reflects **`https`** vs explicit **http** opt-in; **`trust_state`** is the canonical combined signal. On native HTTPS, **`dashboard_transport.tls_posture`** reports cipher/protocol + leaf cert SHA-256 (S2a wave-2a/2b; published via env so uvicorn **workers** see it). Weak ciphers/protocol add **`tls_cipher_baseline_weak`** / **`tls_protocol_below_baseline`**; configured fingerprint allow-list mismatch adds **`tls_cert_fingerprint_mismatch`**. Without **`api.https_cert_fingerprint_sha256`**, fingerprint is observe-only (no trust downgrade). | +| **Runtime** | `GET /status` → **`dashboard_transport`** reflects **`https`** vs explicit **http** opt-in; **`GET /health`** is liveness only (`{"status":"ok"}`). **`trust_state`** is the canonical combined signal. On native HTTPS, **`dashboard_transport.tls_posture`** reports cipher/protocol + leaf cert SHA-256 (S2a wave-2a/2b; published via env so uvicorn **workers** see it). Weak ciphers/protocol add **`tls_cipher_baseline_weak`** / **`tls_protocol_below_baseline`**; configured fingerprint allow-list mismatch adds **`tls_cert_fingerprint_mismatch`**. Without **`api.https_cert_fingerprint_sha256`**, fingerprint is observe-only (no trust downgrade). | | **Audit export** | `python main.py --export-audit-trail -` → JSON fields **`dashboard_transport`**, **`trust_state`**, **`enterprise_surface`** (export runs without `--web`; transport may be `not_configured` unless env is set). | | **Auth** | With **`require_api_key`**, **`/health`** = no key; **`/status`** = key required. | diff --git a/docs/ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.pt_BR.md b/docs/ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.pt_BR.md index f0de95897..9a1a7cf69 100644 --- a/docs/ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.pt_BR.md +++ b/docs/ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.pt_BR.md @@ -17,7 +17,7 @@ 1. Com **`api.require_api_key: true`**, o processo precisa de um segredo **resolvido**: ou **`api.api_key`** não vazio no config, ou **`api.api_key_from_env: "VAR"`** com **`VAR` definida no ambiente antes da subida**. 1. **`main.py --web`** **encerra com código 2** se `require_api_key` estiver true e **nenhuma chave** puder ser resolvida (evita dashboard aberto por engano). 1. **Bind fora do loopback** (`0.0.0.0`, IP da LAN, `::`): **`main.py --web` encerra com código 2** a menos que **`api.require_api_key: true`** e a chave esteja **resolvida**. Ter **`api.api_key`** / chave no ambiente ou um segredo WebAuthn **sem** `require_api_key` **não** conta — as rotas JSON (`GET /findings`, `POST /scan`, config) continuam sem autenticação (#1714). Use bind **`127.0.0.1`** no lab sem chave, ou ligue **`require_api_key: true`** antes de publicar a porta do container. -1. **`GET /health`** **nunca** exige autenticação (probes e load balancers continuam simples). Devolve JSON com `status`, resumo público de `license` e `dashboard_transport`. +1. **`GET /health`** **nunca** exige autenticação (probes e load balancers continuam simples). O corpo é exatamente `{"status":"ok"}`. Licença, confiança e `dashboard_transport` ficam no **`GET /status`** autenticado (#2077). 1. **Todas as outras rotas** (páginas HTML, `GET /status`, `POST /scan`, `GET /config`, OpenAPI `/docs`, …) exigem chave válida quando a exigência está ativa: envie **`X-API-Key: `** ou **`Authorization: Bearer `**. 1. **401** = chave ausente ou incorreta. **503** = `require_api_key` true mas a chave não foi resolvida em runtime (misconfig); corrija YAML/ambiente e reinicie. @@ -120,7 +120,7 @@ api: - "dashboard.example.com" ``` -Com peer correspondente **e** `X-Forwarded-Proto: https` confiável, o painel suprime o banner vermelho de risco de texto claro e expõe **`effective_external_transport`** por requisição (`tls_termination: trusted_proxy`) em **`GET /status`** / **`GET /health`**. O **`dashboard_transport`** a nível de processo permanece `mode: http` / `tls_active: false` — honestidade sobre o listener Uvicorn, não alegação de TLS nativo no app. Veja [#1515](https://github.com/DataBoar/data-boar/issues/1515) / [PLAN_DASHBOARD_TRUSTED_PROXY_TLS.md](../plans/PLAN_DASHBOARD_TRUSTED_PROXY_TLS.md). +Com peer correspondente **e** `X-Forwarded-Proto: https` confiável, o painel suprime o banner vermelho de risco de texto claro e expõe **`effective_external_transport`** por requisição (`tls_termination: trusted_proxy`) em **`GET /status`**. O **`dashboard_transport`** a nível de processo permanece `mode: http` / `tls_active: false` — honestidade sobre o listener Uvicorn, não alegação de TLS nativo no app. Veja [#1515](https://github.com/DataBoar/data-boar/issues/1515) / [PLAN_DASHBOARD_TRUSTED_PROXY_TLS.md](../plans/PLAN_DASHBOARD_TRUSTED_PROXY_TLS.md). Veja [deploy/DEPLOY.md](../deploy/DEPLOY.pt_BR.md) e **SECURITY.md** na raiz. @@ -151,7 +151,7 @@ api: # - "cert-antigo-ainda-valido-na-transicao…" ``` -**3.** Abra **`https://:/`**. Use **`GET /status`** ou **`GET /health`** e confira **`dashboard_transport`** no JSON (`mode`, `tls_active`, `summary`, **`tls_posture`** opcional aninhado) **e** os campos canônicos **`trust_state`** / **`trust_reasons`** / **`output_confidence`**. +**3.** Abra **`https://:/`**. Use **`GET /status`** e confira **`dashboard_transport`** no JSON (`mode`, `tls_active`, `summary`, **`tls_posture`** opcional aninhado) **e** os campos canônicos **`trust_state`** / **`trust_reasons`** / **`output_confidence`**. **4.** Evidência opcional: **`python main.py --export-audit-trail -`** (sem `--web`) inclui **`dashboard_transport`**, **`trust_state`** e **`enterprise_surface`** no JSON exportado. @@ -194,7 +194,7 @@ Padrões em [deploy/DEPLOY.md](../deploy/DEPLOY.pt_BR.md) e [deploy/kubernetes/R | Verificação | O que observar | | ----------- | -------------- | -| **Runtime** | `GET /status` e `GET /health` → **`dashboard_transport`** com **`https`** vs **http** com opt-in; **`trust_state`** é o sinal canônico combinado. Em HTTPS nativo, **`dashboard_transport.tls_posture`** traz cipher/protocolo + SHA-256 do cert leaf (S2a wave-2a/2b; publicado via env para os **workers** do uvicorn). Cipher/protocolo fracos adicionam **`tls_cipher_baseline_weak`** / **`tls_protocol_below_baseline`**; mismatch da allow-list de fingerprint adiciona **`tls_cert_fingerprint_mismatch`**. Sem **`api.https_cert_fingerprint_sha256`**, o fingerprint é só observação (sem downgrade de trust). | +| **Runtime** | `GET /status` → **`dashboard_transport`** com **`https`** vs **http** com opt-in; **`GET /health`** é só liveness (`{"status":"ok"}`). **`trust_state`** é o sinal canônico combinado. Em HTTPS nativo, **`dashboard_transport.tls_posture`** traz cipher/protocolo + SHA-256 do cert leaf (S2a wave-2a/2b; publicado via env para os **workers** do uvicorn). Cipher/protocolo fracos adicionam **`tls_cipher_baseline_weak`** / **`tls_protocol_below_baseline`**; mismatch da allow-list de fingerprint adiciona **`tls_cert_fingerprint_mismatch`**. Sem **`api.https_cert_fingerprint_sha256`**, o fingerprint é só observação (sem downgrade de trust). | | **Export de auditoria** | `python main.py --export-audit-trail -` → JSON com **`dashboard_transport`**, **`trust_state`** e **`enterprise_surface`** (export sem `--web`; transporte pode ficar `not_configured` sem env). | | **Auth** | Com **`require_api_key`**, **`/health`** sem chave; **`/status`** com chave. | diff --git a/docs/plans/PLANS_TODO.md b/docs/plans/PLANS_TODO.md index 9ca1c9b93..ca0d1ca1b 100644 --- a/docs/plans/PLANS_TODO.md +++ b/docs/plans/PLANS_TODO.md @@ -82,7 +82,7 @@ When **partners** or **buyers** anchor on a vertical (MSP, insurance, RPO, real - **Database compat matrix — sparse A×B (`#1237` / `#1302`, plan 2026-07-22):** Axis A = platform hosts (lab nodes — FS/CSV/SQLite); axis B = engines on a Podman farm host (pg/mysql/mariadb/mssql/oracle/sqlite, mongo, redis) + separate cloud-lane; sparse intersections only for native drivers (pymssql/psycopg2/oracledb thick). A light lab client → remote farm validates [#1237](https://github.com/DataBoar/data-boar/issues/1237) latency. Synthetic fixtures only. Tracking plan: [PLAN_DATABASE_COMPAT_MATRIX.md](PLAN_DATABASE_COMPAT_MATRIX.md). Cross-refs [#1171](https://github.com/DataBoar/data-boar/issues/1171), [#1182](https://github.com/DataBoar/data-boar/issues/1182), [#782](https://github.com/DataBoar/data-boar/issues/782)/GAP-001. - **S0 – Trust burst:** **–1** merged in **[#195](https://github.com/DataBoar/data-boar/pull/195)** (Dependabot-equivalent bumps: FastAPI/pydantic/rich/mypy/Ruff + dev transitives; **github-actions**: `astral-sh/setup-uv` **v8.1.0**, `github/codeql-action` **4.35.2** SHAs); **`uv lock`** + **`requirements.txt`** on **`main`**. Pre-merge: **`uvx pip-audit`** clean; **–1b** snapshot on Hub **`fabioleitao/data_boar:latest`**: **`docker scout quickview`** **0C / 0H / 1M / 31L**; **`docker-scout-critical-gate.ps1`** pass. **Next release / prod image:** rebuild + push so Hub digest matches this lockfile; then Scout again. **A3** Hub tag hygiene remains **operator** (UI). Dependabot **#191–#194** closed as superseded by **#195**. - **S0b / M-OBS (partial):** **Backup and restore** for persistent data documented in [DEPLOY.md](../deploy/DEPLOY.md) **§9** and [DEPLOY.pt_BR.md](../deploy/DEPLOY.pt_BR.md) **§9** (config + SQLite + reports under `/data`). Kanban **Selected** now **S1 – Lab proof (–1L)** — see [SPRINTS_AND_MILESTONES.md](SPRINTS_AND_MILESTONES.md) §3. -- **S2a – Transport + trust (open-core POC, demo-ready):** **Wave-1 in flight / this PR** — canonical **`trust_state`** / **`trust_reasons`** / **`output_confidence`** (`core/canonical_trust.py`) on `/status`, `/health`, `--export-audit-trail` (+ `enterprise_surface` on audit); Phase 7 partial; #86 doc-first matrix refresh + HOWTO §B.7 demo. Fingerprint/cipher/SSO out of scope. Prior pairing text: first slice of [PLAN_DASHBOARD_HTTPS_BY_DEFAULT_AND_HTTP_EXPLICIT_RISK.md](completed/PLAN_DASHBOARD_HTTPS_BY_DEFAULT_AND_HTTP_EXPLICIT_RISK.md) **Phase 7** **together with** **one** access/governance step from [PLAN_DASHBOARD_REPORTS_ACCESS_CONTROL.md](PLAN_DASHBOARD_REPORTS_ACCESS_CONTROL.md) / **[#86](https://github.com/DataBoar/data-boar/issues/86)**. Vocabulary: [PLAN_GRC_INSPIRED_ENTERPRISE_TRUST_ACCELERATORS.md](PLAN_GRC_INSPIRED_ENTERPRISE_TRUST_ACCELERATORS.md) §A1–A2. **Demo bar:** operator can show TLS (native or documented reverse-proxy) plus **`GET /status`** / **`--export-audit-trail`** carrying **both** transport and trust signals; cross-check [SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.md](../ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.md). Kanban + sprint row: [SPRINTS_AND_MILESTONES.md](SPRINTS_AND_MILESTONES.md) §3 / §4 (**S2a**). +- **S2a – Transport + trust (open-core POC, demo-ready):** **Wave-1 in flight / this PR** — canonical **`trust_state`** / **`trust_reasons`** / **`output_confidence`** (`core/canonical_trust.py`) on `/status` and `--export-audit-trail` (`/health` is liveness only, #2077) (+ `enterprise_surface` on audit); Phase 7 partial; #86 doc-first matrix refresh + HOWTO §B.7 demo. Fingerprint/cipher/SSO out of scope. Prior pairing text: first slice of [PLAN_DASHBOARD_HTTPS_BY_DEFAULT_AND_HTTP_EXPLICIT_RISK.md](completed/PLAN_DASHBOARD_HTTPS_BY_DEFAULT_AND_HTTP_EXPLICIT_RISK.md) **Phase 7** **together with** **one** access/governance step from [PLAN_DASHBOARD_REPORTS_ACCESS_CONTROL.md](PLAN_DASHBOARD_REPORTS_ACCESS_CONTROL.md) / **[#86](https://github.com/DataBoar/data-boar/issues/86)**. Vocabulary: [PLAN_GRC_INSPIRED_ENTERPRISE_TRUST_ACCELERATORS.md](PLAN_GRC_INSPIRED_ENTERPRISE_TRUST_ACCELERATORS.md) §A1–A2. **Demo bar:** operator can show TLS (native or documented reverse-proxy) plus **`GET /status`** / **`--export-audit-trail`** carrying **both** transport and trust signals; cross-check [SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.md](../ops/SECURE_DASHBOARD_AUTH_AND_HTTPS_HOWTO.md). Kanban + sprint row: [SPRINTS_AND_MILESTONES.md](SPRINTS_AND_MILESTONES.md) §3 / §4 (**S2a**). - **Sprint focus:** **S2a – Transport + trust (POC)** is **Selected** for the **next agent-heavy window** (interleave **S1 – Lab proof** / –1L when hardware calendar demands it). After **S2a** reaches demo-ready checkpoint, return to Tier-2 table orders **4–7** and maturity **M-MATURITY-POC** / **#86** Phase 1 sequencing per §4.2 below. **S0** checklist closure: [SPRINTS_AND_MILESTONES.md](SPRINTS_AND_MILESTONES.md) **§4.0**. - **Branches:** Work ships via **PRs to `main`**; avoid tracking a long-lived “integration” branch name here (it goes stale). - **v1.8.0 competitive survey (docs-first):** **[#1056](https://github.com/DataBoar/data-boar/issues/1056)** ✅ closed · **[#1057](https://github.com/DataBoar/data-boar/issues/1057)** ✅ closed · **[#1058](https://github.com/DataBoar/data-boar/issues/1058)** ✅ closed · **[#1059](https://github.com/DataBoar/data-boar/issues/1059)** ✅ closed · **[#1060](https://github.com/DataBoar/data-boar/issues/1060)** ✅ this docs PR (`PLAN_SYNTHETIC_DATA_AND_CONFIDENCE_VALIDATION`). Remaining open on GitHub milestone **v1.8.0:** **[#1061](https://github.com/DataBoar/data-boar/issues/1061)** (enrich existing `PLAN_*.md`; no greenfield features). **Order remaining:** #1061 DSAR tombstone. **One PR per issue** (`Closes #N`); each PR runs `plans_hub_sync.py --write` + updates this file. Methodology: `compliance-sample-*.yaml`, `norm_tag`, `report.recommendation_overrides`, disclaimers per [COMPLIANCE_AND_LEGAL.md](../COMPLIANCE_AND_LEGAL.md). **Revisit (completed):** content-type/cloaking (#884), sensitive-categories ML/DL — addendum only unless operator reopens. @@ -101,7 +101,7 @@ When **partners** or **buyers** anchor on a vertical (MSP, insurance, RPO, real - **Dependabot / pip-audit / PyPI supply chain:** Committed **`uv.lock`**, CI **`pip-audit`**, and **Dependabot** (pip + github-actions) address **known CVEs** and **dependency drift** on the **PyPI** install path—they do **not** replace human review of PRs or detect every **malicious** / typosquat scenario. **SBOM** artifacts: [ADR 0003](../adr/ADR-0003-sbom-roadmap-cyclonedx-then-syft.md). CI workflows (**`ci.yml`**, **CodeQL**, **Semgrep**, **slack-ops-digest**) pin third-party Actions to **commit SHAs**; **`setup-uv`** pins **uv** CLI semver — bump via **Dependabot** (github-actions) or deliberately; details: [WORKFLOW_DEFERRED_FOLLOWUPS.md](../ops/WORKFLOW_DEFERRED_FOLLOWUPS.md). Framing (trust, marketplaces): [SUPPLY_CHAIN_AND_TRUST_SIGNALS.md](../ops/inspirations/SUPPLY_CHAIN_AND_TRUST_SIGNALS.md). **Bump workflow:** edit **`pyproject.toml`**, then **`uv lock`**, **`uv export --no-emit-package pyproject.toml -o requirements.txt`**, **`.\scripts\check-all.ps1`**. **Upstream-blocked alerts:** [DEPENDABOT_PYOPENSSL_SNOWFLAKE.md](../ops/DEPENDABOT_PYOPENSSL_SNOWFLAKE.md); [DEPENDABOT_PYGMENTS_CVE.md](../ops/DEPENDABOT_PYGMENTS_CVE.md). - **Docker Hub / Scout (–1b):** After **deps or Dockerfile** changes, rebuild, **push**, **`docker scout quickview`**. Debian **base** packages may still show **CVEs with no fix** in the current slim image (e.g. **ncurses**) — acceptable until **`python:*-slim`** refreshes; document operator exceptions if needed. - **Published vs working:** **published** customer channels remain **`1.7.4.post12`** / **`v1.7.4.post12`** (PyPI + Hub; GA **`1.7.4`** in history). **`main` working tree:** **`1.8.0-rc`** (`maturity_build=127`, ADR-0073 rc band; cut 2026-09-22) — git-only; notes [docs/releases/1.8.0-rc.md](../releases/1.8.0-rc.md) (beta history: [1.8.0-beta.md](../releases/1.8.0-beta.md)). **Next publishable horizon:** stable **`1.8.0`**. **Pick one primary slice per bundle** (avoid mixing maturity POC closure, vendor adapters, and unrelated H3 fronts): default **v1.8.0** candidate is maturity assessment **POC ready** (`scripts/smoke-maturity-assessment-poc.ps1` + [SMOKE_MATURITY_ASSESSMENT_POC.md](../ops/SMOKE_MATURITY_ASSESSMENT_POC.md) checklist). Further **scope-import** vendor adapters are GitHub milestone **v1.8.1** (not v1.8.0). -- **Dashboard transport (HTTPS / explicit HTTP):** Shipped on `main` — `main.py --web` requires **`--https-cert-file`+`--https-key-file`** or **`--allow-insecure-http`** (or `api.*` equivalents); Docker **`CMD`** includes `--allow-insecure-http`; **`GET /status`** and **`GET /health`** expose **`dashboard_transport`**, **`enterprise_surface`**, and canonical **`trust_state`** / **`trust_reasons`** / **`output_confidence`** (S2a wave-1). See [PLAN_DASHBOARD_HTTPS_BY_DEFAULT_AND_HTTP_EXPLICIT_RISK.md](completed/PLAN_DASHBOARD_HTTPS_BY_DEFAULT_AND_HTTP_EXPLICIT_RISK.md) (Phase 4 audit-export ✅; Phase 7 🔄 partial wave-1). **Trusted-proxy edge TLS (#1515):** [PLAN_DASHBOARD_TRUSTED_PROXY_TLS.md](PLAN_DASHBOARD_TRUSTED_PROXY_TLS.md) — suppress false plaintext banner only when peer ∈ `api.trusted_proxy_cidrs` and forwarded proto is trusted `https`. +- **Dashboard transport (HTTPS / explicit HTTP):** Shipped on `main` — `main.py --web` requires **`--https-cert-file`+`--https-key-file`** or **`--allow-insecure-http`** (or `api.*` equivalents); Docker **`CMD`** includes `--allow-insecure-http`; **`GET /status`** exposes **`dashboard_transport`**, **`enterprise_surface`**, and canonical **`trust_state`** / **`trust_reasons`** / **`output_confidence`** (S2a wave-1). **`GET /health`** is liveness only (#2077). See [PLAN_DASHBOARD_HTTPS_BY_DEFAULT_AND_HTTP_EXPLICIT_RISK.md](completed/PLAN_DASHBOARD_HTTPS_BY_DEFAULT_AND_HTTP_EXPLICIT_RISK.md) (Phase 4 audit-export ✅; Phase 7 🔄 partial wave-1). **Trusted-proxy edge TLS (#1515):** [PLAN_DASHBOARD_TRUSTED_PROXY_TLS.md](PLAN_DASHBOARD_TRUSTED_PROXY_TLS.md) — suppress false plaintext banner only when peer ∈ `api.trusted_proxy_cidrs` and forwarded proto is trusted `https`. - **Opt-in port / service hints (roadmap):** Proposal only — allowlisted hosts/ports, zero-trust friendly, optional enterprise feature; [PLAN_OPT_IN_NETWORK_PORT_SERVICE_HINTS.md](PLAN_OPT_IN_NETWORK_PORT_SERVICE_HINTS.md). - **Scope import / inventory bootstrap (roadmap):** **v1 shipped on `main`:** `scripts/scope_import_csv.py` + `config/scope_import_csv.py` + **GLPI adapter** (`scope_import_glpi`) + tests + USAGE/TECH_GUIDE + **EN/pt-BR** [SCOPE_IMPORT_QUICKSTART.md](../ops/SCOPE_IMPORT_QUICKSTART.md) — canonical **CSV → YAML `targets` fragment** for operator merge; see [PLAN_SCOPE_IMPORT_FROM_EXPORTS.md](completed/PLAN_SCOPE_IMPORT_FROM_EXPORTS.md). **Further vendor adapters** (Live Optics, Nagios/Cacti-class, observability exports) **stay backlog until promoted**; complements opt-in port hints (**passive seed** vs **active** probes). **Medium-term [H2]:** [PLAN_DATABRICKS_UNITY_LAKEHOUSE_SCOPE_AND_SCAN.md](PLAN_DATABRICKS_UNITY_LAKEHOUSE_SCOPE_AND_SCAN.md) — **lakehouse as soup ingredient** (SQL scan) + **UC / curated tables as recipe** (inventory → targets); **Pro/Enterprise–shaped**, **customer-pull** before deep design. **[H2] findings sink:** [PLAN_FINDINGS_CORPORATE_REPOSITORY_EXPORT.md](PLAN_FINDINGS_CORPORATE_REPOSITORY_EXPORT.md) — optional **export/sync of scan findings** from local SQLite to **customer-chosen** corporate stores (SQL, MongoDB, lake/object staging); **v1.8.0 [#1058](https://github.com/DataBoar/data-boar/issues/1058)** docs-first: catalog tag formats + opt-in PII-as-quality-check sidecar; **discovery stays read-only** (no source write-back). Same **customer-pull** rule; **reorder PLANS_TODO** if a contract requires it sooner. - **Maturity self-assessment / LGPD questionnaire (operator artefacts → product backlog):** [PLAN_MATURITY_SELF_ASSESSMENT_GRC_QUESTIONNAIRE.md](completed/PLAN_MATURITY_SELF_ASSESSMENT_GRC_QUESTIONNAIRE.md) — **POC A on `main`:** gated `GET`/`POST /{locale}/assessment` (flag + tier), optional YAML pack, **SQLite** answers, optional **HMAC** row sealing + **`GET /status`** / **`--export-audit-trail`** integrity summary — **no** bundled proprietary questionnaire text in public Git. **Scoring + export ✅** — YAML **`scores`** (rubric), **%** on post-submit summary, **`GET /{locale}/assessment/export`** (CSV/Markdown **attachment**; USAGE describes download vs disk). **Per-batch history table on `GET /{locale}/assessment` ✅** (see [ADR 0032](../adr/ADR-0032-maturity-assessment-batch-history-sqlite.md)). **POC ready** closure: **`scripts/smoke-maturity-assessment-poc.ps1`** (levels **A+B**) + [SMOKE_MATURITY_ASSESSMENT_POC.md](../ops/SMOKE_MATURITY_ASSESSMENT_POC.md) manual **§D** (level **C**; §E optional); checklist + **A/B/C evidence** in plan § *POC ready — evidence levels*; **autonomous vs operator** boundary + pytest↔§D mapping in plan + runbook; milestone **M-MATURITY-POC** in [SPRINTS_AND_MILESTONES.md](SPRINTS_AND_MILESTONES.md) §4.2 / §5. **Tier/JWT slice** sketch (orthogonal to **#86**): same plan § *Next slice*; [LICENSING_SPEC.md](../LICENSING_SPEC.md) `dbtier` row. **Next:** after closure, **[GitHub #86](https://github.com/DataBoar/data-boar/issues/86) Phase 1** (session + passwordless on `/{locale}/…`) on a **dedicated branch/PR** — then revisit maturity (DOCX→YAML per **tenant**, annex/bundle, consultant UX). Align **tier JWT enforcement** with [PLAN_PRODUCT_TIERS_AND_OPEN_CORE.md](PLAN_PRODUCT_TIERS_AND_OPEN_CORE.md) (Phases **1+**). **Related:** technical **PDF GRC** report for scan findings — [PLAN_PDF_GRC_REPORT.md](PLAN_PDF_GRC_REPORT.md) (planned, not the org form). Private: `docs/private/raw_pastes/general/LGPD_DOCS/`. **Commercial / subscription** feature—not open core. diff --git a/docs/plans/PLAN_DASHBOARD_REPORTS_ACCESS_CONTROL.md b/docs/plans/PLAN_DASHBOARD_REPORTS_ACCESS_CONTROL.md index ef2d01f08..ccfaa6097 100644 --- a/docs/plans/PLAN_DASHBOARD_REPORTS_ACCESS_CONTROL.md +++ b/docs/plans/PLAN_DASHBOARD_REPORTS_ACCESS_CONTROL.md @@ -54,7 +54,7 @@ See [SECURITY.md](../SECURITY.md), [USAGE.md](../USAGE.md), [TECH_GUIDE.md](../T | Phase | Scope | Outcome | | ----- | ----- | ------- | -| **0** | Docs + **D-WEB** | Route matrix (what is public vs protected); proxy recipes; **middleware order** diagram with [PLAN_DASHBOARD_I18N.md](completed/PLAN_DASHBOARD_I18N.md) (`API key` → `locale` for HTML → `session` → `RBAC`). **2026-04:** `GET /status` and `GET /health` expose **`enterprise_surface`** (transport + license trust + global API-key surface). **S2a wave-1:** also expose canonical **`trust_state`** / **`trust_reasons`** / **`output_confidence`**; `access_surface.rbac` is **`enabled`** when Phase 2 RBAC is active, else **`not_implemented`**. | +| **0** | Docs + **D-WEB** | Route matrix (what is public vs protected); proxy recipes; **middleware order** diagram with [PLAN_DASHBOARD_I18N.md](completed/PLAN_DASHBOARD_I18N.md) (`API key` → `locale` for HTML → `session` → `RBAC`). **2026-04:** `GET /status` exposes **`enterprise_surface`** (`GET /health` is liveness only since #2077) (transport + license trust + global API-key surface). **S2a wave-1:** also expose canonical **`trust_state`** / **`trust_reasons`** / **`output_confidence`**; `access_surface.rbac` is **`enabled`** when Phase 2 RBAC is active, else **`not_implemented`**. | | **1** | **Session + passwordless (minimum)** | **HTTPS required** for WebAuthn. After successful WebAuthn (via Passwordless.dev or equivalent), issue **opaque server session** (**httpOnly cookie** + CSRF strategy) or short-lived internal JWT **separate** from commercial license JWT. **Global `api.require_api_key`** can remain for automation / break-glass; **browser** flows use session. **Schedule after [M-LOCALE-V1](completed/PLAN_DASHBOARD_I18N.md)** so HTML routes are already under `/{locale}/…`. | | **2** | **RBAC** | Named roles (`scanner`, `reports_reader`, `config_admin`, …) bound to **authenticated subject**; route/resource gates on prefixed HTML paths; optional machine keys for API with role claims (design TBD). | | **3** | **Enterprise SSO (optional)** | **OIDC** (SAML later if needed): map IdP groups → product roles; **coexist** with passwordless (e.g. local passkeys for break-glass, SSO for staff). | diff --git a/docs/plans/PLAN_GRC_INSPIRED_ENTERPRISE_TRUST_ACCELERATORS.md b/docs/plans/PLAN_GRC_INSPIRED_ENTERPRISE_TRUST_ACCELERATORS.md index a4bc17af3..fb5606976 100644 --- a/docs/plans/PLAN_GRC_INSPIRED_ENTERPRISE_TRUST_ACCELERATORS.md +++ b/docs/plans/PLAN_GRC_INSPIRED_ENTERPRISE_TRUST_ACCELERATORS.md @@ -51,7 +51,7 @@ We still need a stronger "trust evidence loop" in runtime behavior: **Why now:** aligns with enterprise expectation that risky states do not emit normal-looking evidence. -**Status (S2a wave-2c / M-TRUST-02 thin):** 🔄 Partial — JSON marker `output_confidence` on status/health/audit; Excel Report info **TRUST WATERMARK** when `degraded`/`untrusted`; `untrusted` detail sheets present as stubs (0 finding rows + retention notice) via `report/trust_tint.py`. PDF/DOCX/dashboard download gating deferred. +**Status (S2a wave-2c / M-TRUST-02 thin):** 🔄 Partial — JSON marker `output_confidence` on status/audit (`GET /health` is liveness only, #2077); Excel Report info **TRUST WATERMARK** when `degraded`/`untrusted`; `untrusted` detail sheets present as stubs (0 finding rows + retention notice) via `report/trust_tint.py`. PDF/DOCX/dashboard download gating deferred. ### A3 — Crypto/runtime baseline self-check diff --git a/scripts/build_locale_catalogs.py b/scripts/build_locale_catalogs.py index c07bb0b5a..37d0a66d0 100644 --- a/scripts/build_locale_catalogs.py +++ b/scripts/build_locale_catalogs.py @@ -222,8 +222,8 @@ def _en() -> dict: "run_cli_bind": ( "Bind order when using --web: --host overrides api.host in config and API_HOST; " "if none are set, the default is 127.0.0.1 (Docker image often sets API_HOST=0.0.0.0). " - "Port: api.port in config vs --port. Transport: GET /status and GET /health include " - "dashboard_transport — see docs/USAGE.md." + "Port: api.port in config vs --port. Transport: GET /status includes " + "dashboard_transport; GET /health is liveness only — see docs/USAGE.md." ), "run_cli_danger": ( "Dangerous maintenance (lab/demo only): --reset-data wipes SQLite sessions/findings " @@ -607,8 +607,8 @@ def tr(d: dict) -> dict: base["help"]["run_cli_bind"] = ( "Ordem de bind com --web: --host sobrescreve api.host na config e API_HOST; se nada estiver " "definido, o padrão é 127.0.0.1 (a imagem Docker costuma definir API_HOST=0.0.0.0). " - "Porta: api.port na config vs --port. Transporte: GET /status e GET /health incluem " - "dashboard_transport — veja docs/USAGE.md." + "Porta: api.port na config vs --port. Transporte: GET /status inclui " + "dashboard_transport; GET /health é só liveness — veja docs/USAGE.md." ) base["help"]["run_cli_danger"] = ( "Manutenção perigosa (só lab/demo): --reset-data apaga sessões/achados no SQLite e remove " diff --git a/scripts/docker/build-nogil-local.sh b/scripts/docker/build-nogil-local.sh index 41f56a7a7..1f2173d98 100755 --- a/scripts/docker/build-nogil-local.sh +++ b/scripts/docker/build-nogil-local.sh @@ -184,14 +184,24 @@ if [[ -z "${HEALTH_JSON}" ]]; then exit 1 fi echo "${HEALTH_JSON}" +# License left /health (#2077). Read it from /status (demo does not require an API key). +STATUS_JSON="$( + podman exec "${DEMO_WEB_NAME}" /usr/local/bin/python3.14t -c ' +import json, sys, urllib.request +r = urllib.request.urlopen("http://127.0.0.1:8088/status", timeout=2) +body = json.loads(r.read().decode()) +sys.stdout.write(json.dumps(body, separators=(",", ":"))) +raise SystemExit(0 if r.status == 200 else 1) +' 2>/dev/null +)" || STATUS_JSON="" python3 -c ' import json, sys -body = json.load(sys.stdin) +body = json.loads(sys.argv[1]) lic = (body.get("license") or {}).get("license_state") if lic != "OPEN": raise SystemExit(f"FATAL: expected license_state OPEN, got {lic!r}") -' <<<"${HEALTH_JSON}" || { - echo "FATAL: /health JSON license_state is not OPEN" >&2 +' "${STATUS_JSON}" || { + echo "FATAL: /status JSON license_state is not OPEN" >&2 podman logs "${DEMO_WEB_NAME}" >&2 || true exit 1 } From 946cecb95ffb362cfb7387d153370c32ca5716ff Mon Sep 17 00:00:00 2001 From: Fabio Leitao Date: Mon, 19 Oct 2026 13:38:41 -0300 Subject: [PATCH 10/12] fix(security): require fastapi>=0.143.0 so OTel auto-config is opt-in (Closes #2089) --- CHANGELOG.md | 1 + api/routes.py | 5 +++-- pyproject.toml | 2 +- tests/test_otel_setup_1500.py | 29 +++++++++++++++++++++++++++-- uv.lock | 2 +- 5 files changed, 33 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f3491b3e..6397b105a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Human-readable summary of user-facing changes. **Detailed release notes:** [docs Migration notes on the **`1.8.0-rc`** working line (no new semver cut): +- **FastAPI floor (#2089):** `fastapi>=0.143.0`. Native OpenTelemetry auto-config is opt-in (`FASTAPI_OTEL_AUTO_CONFIGURE=true` or `telemetry={"auto_configure": True}`). The app passes `telemetry={"auto_configure": False}` and still exports only when `DATA_BOAR_OTEL_ENABLED` is set. - **Credential env names (#2086):** `pass_from_env`, `password_from_env`, `user_from_env`, `token_from_env`, and `client_secret_from_env` fail at load when the name is outside the `DATA_BOAR_` / `REST_API_` / `API_` prefixes, or when the destination is missing from that target’s `allowed_hosts`. Shipped examples now use prefixed names (`DATA_BOAR_DEMO_DB_PASSWORD`, `DATA_BOAR_LAB_*`). - **`GET /health` (#2077):** the unauthenticated body is only `{"status":"ok"}`. License, trust, transport, and integrity moved to authenticated `GET /status`. - **Notification `public_base_url` (#2087):** `public_base_url: ${VAR}` raises `ValueError` and is not expanded. Other notification `${VAR}` values still expand only for the same prefixes. diff --git a/api/routes.py b/api/routes.py index f16b118eb..ff921482c 100644 --- a/api/routes.py +++ b/api/routes.py @@ -801,8 +801,9 @@ async def _lifespan(app: FastAPI): title="LGPD/GDPR/CCPA Audit API", version=get_about_info()["version"], lifespan=_lifespan, - # FastAPI 0.142.x auto-exports OTLP when the env is set. Product export - # stays behind DATA_BOAR_OTEL_ENABLED (core.otel_setup, #1500 / #2089). + # Native OTel is opt-in from FastAPI 0.143 (floor in pyproject). Keep the + # kwarg explicit so a later default cannot export ahead of + # DATA_BOAR_OTEL_ENABLED (core.otel_setup, #1500 / #2089). telemetry={"auto_configure": False}, ) diff --git a/pyproject.toml b/pyproject.toml index 751b2e098..696cc5a3c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ dependencies = [ "ebcdic>=1.1.1,<2", "et-xmlfile>=2.0.0", "extract-msg>=0.55.0", - "fastapi>=0.141.1", + "fastapi>=0.143.0", "flask>=3.1.3", "flask-sqlalchemy>=3.1.1", "fonttools>=4.63.0", diff --git a/tests/test_otel_setup_1500.py b/tests/test_otel_setup_1500.py index 89978f717..6b0736948 100644 --- a/tests/test_otel_setup_1500.py +++ b/tests/test_otel_setup_1500.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +from pathlib import Path import pytest @@ -41,14 +42,38 @@ def test_otel_disabled_by_default(monkeypatch: pytest.MonkeyPatch) -> None: assert maybe_setup_otel(app=None) is False +def test_fastapi_floor_requires_otel_opt_in_release() -> None: + """0.142 auto-exports; 0.143+ is opt-in. The floor must exclude 0.142 (#2089).""" + text = (Path(__file__).resolve().parents[1] / "pyproject.toml").read_text( + encoding="utf-8" + ) + assert "fastapi>=0.143.0" in text + + def test_fastapi_app_does_not_auto_configure_otel( monkeypatch: pytest.MonkeyPatch, ) -> None: - """Native FastAPI export stays off unless DATA_BOAR_OTEL_ENABLED (#2089).""" + """No OTLP exporter when DATA_BOAR_OTEL_ENABLED is unset (#2089). + + Observable check: the process-global tracer provider is unchanged after the + ASGI lifespan runs with OTEL_EXPORTER_OTLP_ENDPOINT set. FastAPI only + installs a SDK TracerProvider when native auto-config is on. + """ + from opentelemetry import trace + from starlette.testclient import TestClient + monkeypatch.delenv("DATA_BOAR_OTEL_ENABLED", raising=False) + monkeypatch.delenv("OTEL_SDK_DISABLED", raising=False) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://127.0.0.1:9") + before = trace.get_tracer_provider() import api.routes as routes - assert routes.app._telemetry["auto_configure"] is False + with TestClient(routes.app) as client: + response = client.get("/health") + assert response.status_code == 200 + after = trace.get_tracer_provider() + assert after is before + assert type(after).__name__ != "TracerProvider" @pytest.mark.parametrize("truthy", ["1", "true", "YES", "on"]) diff --git a/uv.lock b/uv.lock index 064d77517..5d264fa8b 100644 --- a/uv.lock +++ b/uv.lock @@ -1266,7 +1266,7 @@ requires-dist = [ { name = "ebcdic", specifier = ">=1.1.1,<2" }, { name = "et-xmlfile", specifier = ">=2.0.0" }, { name = "extract-msg", specifier = ">=0.55.0" }, - { name = "fastapi", specifier = ">=0.141.1" }, + { name = "fastapi", specifier = ">=0.143.0" }, { name = "fastavro", marker = "extra == 'dataformats'", specifier = ">=1.9.0" }, { name = "flask", specifier = ">=3.1.3" }, { name = "flask-sqlalchemy", specifier = ">=3.1.1" }, From 44cbcb7188bc19770b23524ab99b274cb7810fd6 Mon Sep 17 00:00:00 2001 From: Fabio Leitao Date: Mon, 19 Oct 2026 14:05:54 -0300 Subject: [PATCH 11/12] docs: prefix the remaining commented *_from_env examples (Refs #2086) --- deploy/config.example.yaml | 7 ++++--- docs/plans/PLAN_SECRETS_VAULT.md | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/deploy/config.example.yaml b/deploy/config.example.yaml index 70c67cf3c..c5b17f091 100644 --- a/deploy/config.example.yaml +++ b/deploy/config.example.yaml @@ -31,7 +31,7 @@ api: # maturity_assessment_pack_path: /data/maturity_pack.yaml # Optional: HMAC-SHA256 per answer row (tamper-evident; not encryption). Set env before start: # DATA_BOAR_MATURITY_INTEGRITY_SECRET= - # Or: maturity_integrity_secret_from_env: "VAR_NAME" to read the secret from that env var. + # Or: maturity_integrity_secret_from_env: "DATA_BOAR_MATURITY_INTEGRITY_SECRET" to read the secret from that env var. # Optional: dashboard HTML locale (path prefix /en/, /pt-br/; cookie + Accept-Language; see docs/USAGE.md). # locale: @@ -94,9 +94,10 @@ remediation: # enabled: false # type: postgresql # host: db.example.com +# allowed_hosts: ["db.example.com"] # port: 5432 # database: data_governance_db -# user_from_env: SINK_DB_USER -# pass_from_env: SINK_DB_PASS +# user_from_env: DATA_BOAR_SINK_DB_USER +# pass_from_env: DATA_BOAR_SINK_DB_PASS # on_conflict: upsert # include_sample_content: false diff --git a/docs/plans/PLAN_SECRETS_VAULT.md b/docs/plans/PLAN_SECRETS_VAULT.md index 54186e37d..c57858602 100644 --- a/docs/plans/PLAN_SECRETS_VAULT.md +++ b/docs/plans/PLAN_SECRETS_VAULT.md @@ -64,7 +64,7 @@ So: a **local vault with import/re-import is feasible**; an **external vault wit These can be implemented **instead of** or **in addition to** a vault. 1. **Expand env-only for all credentials** - - For every secret field (pass, api_key, client_secret, token, etc.), support a `*_from_env` (or existing `*_from_env` / `${VAR}`) so the config file holds **no** secrets, only references like `pass_from_env: "DB_PASS"`. Document as the recommended pattern; keep inline secrets working for backward compatibility. + - For every secret field (pass, api_key, client_secret, token, etc.), support a `*_from_env` (or existing `*_from_env` / `${VAR}`) so the config file holds **no** secrets, only references like `pass_from_env: "DATA_BOAR_DB_PASS"`. Document as the recommended pattern; keep inline secrets working for backward compatibility. - **Pro:** No new storage; fits 12-factor and containers. **Con:** Many env vars for many targets; no “import from file” story. 1. **Redact secrets in the web UI** From f40c697bfc9be917f4920e470bdfcbdac04df0f7 Mon Sep 17 00:00:00 2001 From: Fabio Leitao Date: Mon, 19 Oct 2026 14:06:42 -0300 Subject: [PATCH 12/12] fix(security): require every credential destination in allowed_hosts (Refs #2086) --- config/loader.py | 59 ++++++++++++++------------ tests/test_credential_env_allowlist.py | 40 +++++++++++++++++ 2 files changed, 73 insertions(+), 26 deletions(-) diff --git a/config/loader.py b/config/loader.py index 4c9edd64c..1bc66a31d 100644 --- a/config/loader.py +++ b/config/loader.py @@ -117,81 +117,88 @@ def _credential_target_kind(cfg: dict[str, Any]) -> str: "cifs", } ) -# Secret is posted only to a host the connector already allowlists. -_VENDOR_FIXED_SECRET_KINDS = frozenset({"hubspot", "powerbi"}) -def _credential_destination_identities(cfg: dict[str, Any]) -> set[str] | None: - """Identities that must intersect ``allowed_hosts``. +def _credential_destination_groups(cfg: dict[str, Any]) -> list[set[str]] | None: + """Places the secret is sent, each as a set of aliases. - ``None`` means a vendor allowlist already binds where the secret is sent - (#1607 HubSpot, #2007 Power BI default token host). An empty set fails closed. + ``None`` means a vendor allowlist already binds the destination + (#1607 HubSpot, #2007 Power BI default token host). + + Two semantics (#2086): + + * Aliases of the **same** destination (Snowflake account or + ``.snowflakecomputing.com``) share one set. One match is enough. + * **Distinct** destinations (Dataverse/Power Apps ``org_url`` and + ``token_url``; a custom Power BI ``token_url``) are separate sets. + Every set must intersect ``allowed_hosts``. + + An empty set fails closed. """ kind = _credential_target_kind(cfg) auth = cfg.get("auth") if isinstance(cfg.get("auth"), dict) else {} token_url = str(auth.get("token_url") or "").strip() + def _one(host: str) -> list[set[str]]: + return [{host}] if host else [set()] + if kind == "hubspot": return None if kind == "powerbi": if not token_url: return None - host = _hostname_of(token_url) - return {host} if host else set() + return _one(_hostname_of(token_url)) if kind in {"dataverse", "powerapps"}: org = str(cfg.get("org_url") or cfg.get("environment_url") or "") - idents = {host} if (host := _hostname_of(org)) else set() + groups = _one(_hostname_of(org)) if token_url: - token_host = _hostname_of(token_url) - if not token_host: - return set() - idents.add(token_host) - return idents + groups.extend(_one(_hostname_of(token_url))) + return groups if kind == "snowflake": account = str(cfg.get("account") or "").strip().lower() if not account: - return set() + return [set()] derived = ( account if account.endswith(".snowflakecomputing.com") else f"{account}.snowflakecomputing.com" ) - return {account, derived} + return [{account, derived}] if kind == "sharepoint": host = _hostname_of(str(cfg.get("site_url") or cfg.get("base_url") or "")) - return {host} if host else set() + return _one(host) if kind == "webdav": host = _hostname_of( str(cfg.get("base_url") or cfg.get("url") or cfg.get("host") or "") ) - return {host} if host else set() + return _one(host) if kind == "nfs": host = str(cfg.get("host") or cfg.get("server") or "").strip().lower() - return {host} if host else set() + return _one(host) if kind in _HOST_DESTINATION_KINDS: host = str(cfg.get("host") or "").strip().lower() if not host: host = _hostname_of(str(cfg.get("url") or cfg.get("base_url") or "")) - return {host} if host else set() - return set() + return _one(host) + return [set()] def assert_explicit_destination_allowlisted(cfg: dict[str, Any], label: str) -> None: - """Fail closed unless the credential destination is explicitly allowlisted. + """Fail closed unless every credential destination is explicitly allowlisted. ``allowed_hosts`` comes from the same YAML the config author writes (same limit as #1977). Against that author only the env-name prefix is an independent control; the host list is a brake, not a second authority. """ - identities = _credential_destination_identities(cfg) - if identities is None: + groups = _credential_destination_groups(cfg) + if groups is None: return allowed = _explicit_allowed_hosts(cfg) - if identities and identities & allowed: + if groups and all(group & allowed for group in groups): return raise ValueError( f"{label} requires an explicit allowed_hosts entry " - f"for the destination ({_CREDENTIAL_ENV_TAG})." + f"for every destination ({_CREDENTIAL_ENV_TAG})." ) diff --git a/tests/test_credential_env_allowlist.py b/tests/test_credential_env_allowlist.py index 2259b486b..6a99047b6 100644 --- a/tests/test_credential_env_allowlist.py +++ b/tests/test_credential_env_allowlist.py @@ -275,3 +275,43 @@ def test_powerbi_client_secret_from_env_uses_vendor_token_host( } out = normalize_config(data) assert out["targets"][0]["client_secret"] == "pbi-secret" + + +@pytest.mark.parametrize("kind", ["dataverse", "powerapps"]) +def test_org_and_token_url_must_both_be_allowlisted( + kind: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("API_DV_SECRET", "dv-secret") + target = { + "name": kind, + "type": kind, + "org_url": "https://org.crm.dynamics.com", + "client_secret_from_env": "API_DV_SECRET", + "auth": {"token_url": "https://evil.example/oauth/token"}, + "allowed_hosts": ["org.crm.dynamics.com"], + } + with pytest.raises(ValueError, match="allowed_hosts.*#1977"): + normalize_config( + {"targets": [target], "api": {}, "report": {"output_dir": "."}} + ) + + +@pytest.mark.parametrize("kind", ["dataverse", "powerapps"]) +def test_org_and_token_url_copy_when_both_allowlisted( + kind: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("API_DV_SECRET", "dv-secret") + target = { + "name": kind, + "type": kind, + "org_url": "https://org.crm.dynamics.com", + "client_secret_from_env": "API_DV_SECRET", + "auth": { + "token_url": "https://login.microsoftonline.com/tenant/oauth2/v2.0/token" + }, + "allowed_hosts": ["org.crm.dynamics.com", "login.microsoftonline.com"], + } + out = normalize_config( + {"targets": [target], "api": {}, "report": {"output_dir": "."}} + ) + assert out["targets"][0]["client_secret"] == "dv-secret"