From 7a3f5f595c27dfd27511be5456dfc34e0263c8fb Mon Sep 17 00:00:00 2001 From: Dev Punia Date: Wed, 29 Jul 2026 16:47:29 +0530 Subject: [PATCH] fix: accept dbt 2.0 capitalized source freshness statuses in sources v3 parser dbt 2.0.0-preview.202 emits 'Pass'/'Error' where earlier dbt emitted lowercase. Every freshness result then failed validation and the whole sources.json was silently dropped (live: harvestgroup, 50 runs/day). Same class + same fix pattern as run_results_v6 'reused' (AI-7435/#106): str Enum + _missing_. Known statuses canonicalize by case to their lowercase members; unknown future statuses become dynamic members. The runtime-error union arm canonicalizes case only and stays narrow so it cannot swallow freshness rows. Version 0.3.5 -> 0.3.6. Co-Authored-By: Claude Fable 5 --- .bumpversion.cfg | 2 +- docs/conf.py | 2 +- setup.py | 2 +- src/datapilot/__init__.py | 2 +- .../parsers/sources/sources_v3.py | 37 +++++++- tests/test_vendor/test_sources_v3.py | 92 +++++++++++++++++++ 6 files changed, 131 insertions(+), 6 deletions(-) create mode 100644 tests/test_vendor/test_sources_v3.py diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 61d68fd..79783be 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.5 +current_version = 0.3.6 commit = True tag = True diff --git a/docs/conf.py b/docs/conf.py index cf02f3d..bf1eca2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,7 +15,7 @@ year = "2024" author = "Altimate Inc." copyright = f"{year}, {author}" -version = release = "0.3.5" +version = release = "0.3.6" pygments_style = "trac" templates_path = ["."] diff --git a/setup.py b/setup.py index 571d1a6..298f964 100755 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ def read(*names, **kwargs): setup( name="altimate-datapilot-cli", - version="0.3.5", + version="0.3.6", license="MIT", description="Assistant for Data Teams", long_description="{}\n{}".format( diff --git a/src/datapilot/__init__.py b/src/datapilot/__init__.py index a8d4557..d7b30e1 100644 --- a/src/datapilot/__init__.py +++ b/src/datapilot/__init__.py @@ -1 +1 @@ -__version__ = "0.3.5" +__version__ = "0.3.6" diff --git a/src/vendor/dbt_artifacts_parser/parsers/sources/sources_v3.py b/src/vendor/dbt_artifacts_parser/parsers/sources/sources_v3.py index b4465c0..58ab182 100644 --- a/src/vendor/dbt_artifacts_parser/parsers/sources/sources_v3.py +++ b/src/vendor/dbt_artifacts_parser/parsers/sources/sources_v3.py @@ -25,9 +25,23 @@ class Metadata(BaseParserModel): env: Optional[dict[str, str]] = None -class Status(Enum): +class Status(str, Enum): runtime_error = "runtime error" + @classmethod + def _missing_(cls, value): + # dbt 2.0 capitalizes freshness statuses (e.g. "Runtime Error"). + # Canonicalize case here, but stay NARROW otherwise: this enum is the + # runtime-error union arm and must not swallow freshness rows (see + # Status1 below for the permissive arm). Same class of fix as + # run_results_v6.Status (AI-7435 / PR #106). + if isinstance(value, str): + low = value.lower() + for member in cls: + if member.value == low: + return member + return None + class Results(BaseParserModel): model_config = ConfigDict( @@ -38,12 +52,31 @@ class Results(BaseParserModel): status: Status -class Status1(Enum): +class Status1(str, Enum): pass_ = "pass" warn = "warn" error = "error" runtime_error = "runtime error" + @classmethod + def _missing_(cls, value): + # dbt 2.0 (>= 2.0.0-preview.202) emits capitalized freshness statuses + # ("Pass", "Error") where every earlier dbt emitted lowercase. That made + # ALL source freshness results fail validation and silently dropped the + # whole sources.json during ingestion. Canonicalize known values by + # case; surface truly unknown future statuses as real members so + # downstream `.value` access keeps working (same forward-compat pattern + # as run_results_v6.Status, AI-7435 / PR #106). + if isinstance(value, str): + low = value.lower() + for member in cls: + if member.value == low: + return member + member = str.__new__(cls, value) + member._name_ = str(value) + member._value_ = value + return member + class Period(Enum): minute = "minute" diff --git a/tests/test_vendor/test_sources_v3.py b/tests/test_vendor/test_sources_v3.py new file mode 100644 index 0000000..57b167d --- /dev/null +++ b/tests/test_vendor/test_sources_v3.py @@ -0,0 +1,92 @@ +"""Tests for sources v3 parser: resilient freshness Status enums. + +Regression coverage for dbt 2.0 (>= 2.0.0-preview.202) emitting CAPITALIZED +freshness statuses ("Pass", "Error") where earlier dbt emitted lowercase. +Previously every result failed validation and the entire sources.json was +silently dropped during ingestion (observed live: harvestgroup, 50 runs/day). +Same defect class as run_results_v6 ``reused`` (AI-7435 / PR #106). +""" +from vendor.dbt_artifacts_parser.parser import parse_sources +from vendor.dbt_artifacts_parser.parsers.sources.sources_v3 import Status +from vendor.dbt_artifacts_parser.parsers.sources.sources_v3 import Status1 + +V3_SCHEMA = "https://schemas.getdbt.com/dbt/sources/v3.json" + + +def _freshness_result(status: str, unique_id: str) -> dict: + return { + "unique_id": unique_id, + "max_loaded_at": "2026-07-28T00:00:00Z", + "snapshotted_at": "2026-07-28T01:00:00Z", + "max_loaded_at_time_ago_in_s": 3600.0, + "status": status, + "criteria": { + "warn_after": {"count": 12, "period": "hour"}, + "error_after": {"count": 24, "period": "hour"}, + }, + "adapter_response": {}, + "timing": [], + "thread_id": "Thread-1", + "execution_time": 1.0, + } + + +def _sources(results: list) -> dict: + return { + "metadata": { + "dbt_schema_version": V3_SCHEMA, + "dbt_version": "2.0.0-preview.202", + "generated_at": "2026-07-28T01:00:00Z", + "invocation_id": "11111111-1111-1111-1111-111111111111", + "env": {}, + }, + "results": results, + "elapsed_time": 2.0, + } + + +def test_capitalized_pass_and_error_parse_and_canonicalize(): + """The exact live failure: dbt 2.0 'Pass'/'Error' must parse, and known + statuses canonicalize to their lowercase members.""" + parsed = parse_sources( + _sources( + [ + _freshness_result("Pass", "source.p.s.a"), + _freshness_result("Error", "source.p.s.b"), + ] + ) + ) + statuses = [r.status for r in parsed.results] + assert statuses[0] is Status1.pass_ + assert statuses[0].value == "pass" + assert statuses[1] is Status1.error + assert statuses[1].value == "error" + + +def test_lowercase_statuses_still_parse_unchanged(): + parsed = parse_sources( + _sources( + [ + _freshness_result("pass", "source.p.s.a"), + _freshness_result("warn", "source.p.s.b"), + _freshness_result("error", "source.p.s.c"), + ] + ) + ) + assert [r.status.value for r in parsed.results] == ["pass", "warn", "error"] + + +def test_unknown_future_status_surfaces_as_member(): + """Forward-compat: a status dbt invents later must not drop the file.""" + parsed = parse_sources(_sources([_freshness_result("Stale", "source.p.s.a")])) + assert parsed.results[0].status.value == "Stale" + + +def test_runtime_error_arm_canonicalizes_case_but_stays_narrow(): + """The runtime-error union arm accepts case variants of its one value and + nothing else — it must not swallow freshness rows.""" + assert Status("Runtime Error") is Status.runtime_error + assert Status1("Runtime Error") is Status1.runtime_error + runtime_row = {"unique_id": "source.p.s.x", "status": "runtime error", "error": "boom"} + parsed = parse_sources(_sources([runtime_row])) + assert parsed.results[0].status.value == "runtime error"