From f4762e3a95ee41eb4c8d5af3422f245b7a16947a Mon Sep 17 00:00:00 2001 From: Vibe Mapper Date: Mon, 24 Aug 2026 10:56:05 -0700 Subject: [PATCH 1/2] feat: field-coverage regression gate in overture_canary (#416) Extends the weekly canary (#219) with a coverage probe over 5 metro bboxes, comparing pinned vs newest release on places/addresses row counts and brand/confidence/taxonomy non-null rates. Flags a >20% relative drop (OvertureMaps/data#546's median -21% brand collapse) via the existing markdown-report/exit-1 channel, catching a release that keeps every required column but silently guts the values behind them. Script-only; comparison/report logic is pure and offline-tested. Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 20 +++ scripts/overture_canary.py | 239 +++++++++++++++++++++++++++++++++- tests/test_overture_canary.py | 99 ++++++++++++++ 3 files changed, 354 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e31b6f9..4d85d6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,26 @@ fixing behavior is patch. ## [Unreleased] ### Added +- `scripts/overture_canary.py` (#219) gained a field-coverage regression + gate (#416): alongside pin-staleness and per-theme column presence, the + weekly canary now scans 5 dense metro bboxes (Paris, Manhattan, Tokyo, + Sao Paulo, Lagos) against both the pinned and the newest Overture + release and compares places row count, addresses row count, and the + non-null rate of `brand`, `confidence`, and `taxonomy.primary` (the + category field `find_places` actually reads). A relative drop past + `REGRESSION_TOLERANCE` (0.20, chosen just under OvertureMaps/data#546's + documented median -21% brand-match collapse) on any metric fails the + canary through the existing markdown-report/exit-1 channel, so a release + that keeps every required column but silently guts the values behind + them — exactly what #546 documented for Brazil, with no schema change at + all — gets caught before the next pin bump adopts it. Skips the gate + (with a note, not a failure) when the pin already is newest, or a probe + fails to read; row-count-backed metrics skip below `MIN_ROWS` (50) on + the pinned side rather than reporting a meaningless percentage off a + tiny denominator. Script-only — no workflow/CI changes; the comparison + and report-rendering logic is pure and covered offline in + `tests/test_overture_canary.py` with synthetic numbers, same as the + existing column-drift checks. - Declared MCP `outputSchema` on all 42 tools (docs/ROADMAP.md §4 feature 3 / §5.3) — the one MCP-conformance gap docs/benchmarks-vs.md conceded to Mapbox is closed. Hand-authored, not derived: every tool returns a bare, diff --git a/scripts/overture_canary.py b/scripts/overture_canary.py index 9d14750..8b0702c 100644 --- a/scripts/overture_canary.py +++ b/scripts/overture_canary.py @@ -11,10 +11,19 @@ is a safety net, not a notification — this probe is the notification, fired before the TTL rollover (#219) adopts the new release in production. +3. Does the *newest* release still carry the same field COVERAGE as the + pinned one, over a small fixed set of dense metro bboxes (#416)? + OvertureMaps/data#546 documents a release where every required column + stayed present but the values behind `brand`/`confidence`/`taxonomy` + collapsed for a whole country (median -21% brand-matched POIs, some + chains to zero) with no schema change at all — (2) above would have + sailed straight through that release. This probe re-runs the same + column-presence-blind bbox scan against both the pinned and the newest + release and flags a large relative drop. -Exit code 0 when both hold; 1 with a markdown report on stdout otherwise — -the workflow turns that report into a GitHub issue. Network-dependent by -design; never run from pytest. +Exit code 0 when all three hold; 1 with a markdown report on stdout +otherwise — the workflow turns that report into a GitHub issue. +Network-dependent by design; never run from pytest. Usage: uv run python scripts/overture_canary.py @@ -32,6 +41,7 @@ buildings, db, divisions, + geo, infrastructure, land_use, overture, @@ -73,6 +83,200 @@ def probe_columns(con: duckdb.DuckDBPyConnection, glob: str) -> set[str] | None: return {r[0] for r in rows} +# --- #416 field-coverage gate ----------------------------------------------- +# +# (name, lat, lon) for five dense metros spread across continents/hemispheres +# — a values-collapsed release (OvertureMaps/data#546) hit one country's +# brand matches, so no single bbox would reliably catch the next one. Picked +# for density (a small bbox still returns a useful row count) and stability +# (city-center coordinates, not anything likely to move or empty out). +PROBE_METROS: list[tuple[str, float, float]] = [ + ("Central Paris", 48.8566, 2.3522), # Europe — Ile de la Cite / Louvre area + ("Manhattan", 40.7549, -73.9840), # North America — Midtown + ("Tokyo (Shibuya)", 35.6595, 139.7005), # Asia — Shibuya crossing area + ("Sao Paulo (Se)", -23.5505, -46.6333), # South America — historic center + ("Lagos Island", 6.4550, 3.3841), # Africa — the metric this gate exists for (#546) +] + +# geo.bbox_around's half-width scales with radius/111_320m; 5_500m keeps +# every probe box in the "~0.1 degree class" the issue asked for (about +# 0.1 deg tall, narrower in longitude away from the equator) while still +# covering enough of a dense downtown to make row counts meaningful. +PROBE_RADIUS_M = 5_500.0 + +# Relative drop (newest vs pinned) on any one probe metric that flags a +# regression. OvertureMaps/data#546's Brazil brand-collapse measured a +# *median* -21% drop (worse for some chains, to zero) with no schema +# change — 0.20 sits just under that median so the canary catches an +# incident of that shape without chasing ordinary release-to-release noise. +REGRESSION_TOLERANCE = 0.20 + +# Floor on the pinned-side row count a metric's denominator must clear +# before its percentage is trusted. A bbox with a handful of pinned rows +# turns "one row disappeared" into a 100% "drop" — noise, not signal. +MIN_ROWS = 50 + +# Metric keys, in report order. "*_rows" are raw row counts (their own +# denominator); the "*_non_null_rate" metrics are non-null fractions among +# a bbox's places rows, so they share places_rows as their denominator. +COVERAGE_METRICS: tuple[str, ...] = ( + "places_rows", + "brand_non_null_rate", + "confidence_non_null_rate", + "category_non_null_rate", + "addresses_rows", +) + +# Which metric in a bbox's own result dict gauges whether that metric's +# denominator is big enough to trust (see MIN_ROWS above). +_DENOMINATOR_METRIC: dict[str, str] = { + "places_rows": "places_rows", + "brand_non_null_rate": "places_rows", + "confidence_non_null_rate": "places_rows", + "category_non_null_rate": "places_rows", + "addresses_rows": "addresses_rows", +} + +# Human-readable label per metric, for the report table. +METRIC_LABELS: dict[str, str] = { + "places_rows": "places rows", + "brand_non_null_rate": "brand non-null rate", + "confidence_non_null_rate": "confidence non-null rate", + # taxonomy.primary is the column find_places actually reads for + # "category" (overture._place_select_exprs) — named for both so a + # reader who only knows the API field still recognizes it. + "category_non_null_rate": "category (taxonomy.primary) non-null rate", + "addresses_rows": "addresses rows", +} + + +def probe_bbox_metrics( + con: duckdb.DuckDBPyConnection, release_name: str, lat: float, lon: float +) -> dict[str, float] | None: + """Coverage metrics for one probe bbox at one release, or None on failure. + + Network-dependent (reads live/pinned S3 parquet through the runtime's own + glob resolution) — never exercised from pytest; compare_bbox_metrics() + below is where the offline tests live. + """ + xmin, ymin, xmax, ymax = geo.bbox_around(lat, lon, PROBE_RADIUS_M) + filter_sql, params = geo.bbox_filter_sql(xmin, ymin, xmax, ymax) + places_glob = overture.upstream_glob("places", "place", release=release_name) + addresses_glob = overture.upstream_glob("addresses", "address", release=release_name) + try: + n, brand_n, confidence_n, category_n = con.execute( + f""" + SELECT + count(*) AS n, + count(brand.names.primary) AS brand_n, + count(confidence) AS confidence_n, + count(taxonomy.primary) AS category_n + FROM read_parquet('{places_glob}', hive_partitioning=1) + WHERE {filter_sql} + """, + params, + ).fetchone() + (addresses_n,) = con.execute( + f""" + SELECT count(*) AS n + FROM read_parquet('{addresses_glob}', hive_partitioning=1) + WHERE {filter_sql} + """, + params, + ).fetchone() + except duckdb.Error as e: + print(f"coverage probe failed for {release_name} @ ({lat}, {lon}): {e}", file=sys.stderr) + return None + return { + "places_rows": float(n), + "brand_non_null_rate": (brand_n / n) if n else 0.0, + "confidence_non_null_rate": (confidence_n / n) if n else 0.0, + "category_non_null_rate": (category_n / n) if n else 0.0, + "addresses_rows": float(addresses_n), + } + + +def compare_bbox_metrics( + pinned_by_bbox: dict[str, dict[str, float]], + newest_by_bbox: dict[str, dict[str, float]], +) -> list[dict]: + """Pure comparison: pinned vs newest metrics -> one row per (bbox, metric). + + Each row is {"bbox", "metric", "pinned", "newest", "delta_pct", + "regression", "skip_reason"}. delta_pct is (newest - pinned) / pinned * + 100, None when skipped. A metric is skipped (skip_reason set, + regression False, delta_pct None) when its denominator (see + _DENOMINATOR_METRIC) is below MIN_ROWS on the pinned side, or when the + pinned value is exactly 0 (nothing to take a percentage of). Only bboxes + present in both dicts are compared — a bbox missing from one side (a + failed probe) is silently skipped by the caller, not reported as a + regression. + """ + rows: list[dict] = [] + for bbox, pinned in pinned_by_bbox.items(): + newest = newest_by_bbox.get(bbox) + if newest is None: + continue + for metric in COVERAGE_METRICS: + pinned_value = pinned.get(metric) + newest_value = newest.get(metric) + if pinned_value is None or newest_value is None: + continue + denom = pinned.get(_DENOMINATOR_METRIC[metric]) + row = { + "bbox": bbox, + "metric": metric, + "pinned": pinned_value, + "newest": newest_value, + "delta_pct": None, + "regression": False, + "skip_reason": None, + } + if denom is not None and denom < MIN_ROWS: + row["skip_reason"] = f"pinned denominator {denom:g} < MIN_ROWS ({MIN_ROWS})" + rows.append(row) + continue + if pinned_value == 0: + row["skip_reason"] = "pinned value is 0" + rows.append(row) + continue + delta_pct = (newest_value - pinned_value) / pinned_value * 100.0 + row["delta_pct"] = delta_pct + row["regression"] = delta_pct <= -REGRESSION_TOLERANCE * 100.0 + rows.append(row) + return rows + + +def render_coverage_report( + rows: list[dict], pinned_release: str, newest_release: str +) -> list[str]: + """Markdown finding lines for the flagged (regression=True) rows in + `rows`, or [] if none. Pure — takes compare_bbox_metrics()'s output, not + a connection, so it's fully covered by offline tests with synthetic + numbers. + """ + flagged = [r for r in rows if r["regression"]] + if not flagged: + return [] + lines = [ + f"- **Field-coverage regression** in `{newest_release}` vs pinned " + f"`{pinned_release}`: at least one probe bbox lost more than " + f"{REGRESSION_TOLERANCE:.0%} of a metric with no schema change " + f"(the shape OvertureMaps/data#546 documented — columns present, " + f"values collapsed). Do not bump the pin to this release until " + f"this is understood.\n" + f"\n" + f" | bbox | metric | pinned | newest | delta % |\n" + f" |---|---|---|---|---|" + ] + for r in flagged: + lines.append( + f" | {r['bbox']} | {METRIC_LABELS[r['metric']]} | {r['pinned']:g} | " + f"{r['newest']:g} | {r['delta_pct']:+.1f}% |" + ) + return lines + + def main() -> int: newest = release._discover(timeout_s=30.0) findings: list[str] = [] @@ -134,6 +338,29 @@ def main() -> int: f"degrades these to None fields — fix before the TTL rollover " f"adopts this release." ) + + coverage_note = None + if newest is None: + coverage_note = "release discovery failed" + elif newest == release.PINNED_RELEASE: + coverage_note = "pinned release is already newest" + else: + pinned_by_bbox: dict[str, dict[str, float]] = {} + newest_by_bbox: dict[str, dict[str, float]] = {} + for name, lat, lon in PROBE_METROS: + pinned_metrics = probe_bbox_metrics(con, release.PINNED_RELEASE, lat, lon) + newest_metrics = probe_bbox_metrics(con, newest, lat, lon) + if pinned_metrics is None or newest_metrics is None: + findings.append( + f"- **Could not run coverage probe** for `{name}` " + f"({'pinned' if pinned_metrics is None else 'newest'} " + f"release unreadable) — skipped, not counted as clean." + ) + continue + pinned_by_bbox[name] = pinned_metrics + newest_by_bbox[name] = newest_metrics + comparison = compare_bbox_metrics(pinned_by_bbox, newest_by_bbox) + findings.extend(render_coverage_report(comparison, release.PINNED_RELEASE, newest)) finally: con.close() @@ -141,7 +368,11 @@ def main() -> int: print(f"## Overture canary findings ({target})\n") print("\n".join(findings)) return 1 - print(f"canary clean: pin {release.PINNED_RELEASE} is newest ({target}); all schemas hold") + suffix = f" (coverage gate skipped: {coverage_note})" if coverage_note else "" + print( + f"canary clean: pin {release.PINNED_RELEASE} is newest ({target}); " + f"all schemas hold{suffix}" + ) return 0 diff --git a/tests/test_overture_canary.py b/tests/test_overture_canary.py index 8a5f96f..ef207ed 100644 --- a/tests/test_overture_canary.py +++ b/tests/test_overture_canary.py @@ -10,6 +10,8 @@ import importlib.util from pathlib import Path +import pytest + from placeroot import addresses, divisions, land_use, overture REPO_ROOT = Path(__file__).resolve().parent.parent @@ -67,3 +69,100 @@ def test_land_cover_is_not_watched_for_columns_it_never_had(): assert "class" not in land_use.LAND_COVER_REQUIRED_COLUMNS assert "names" not in land_use.LAND_COVER_REQUIRED_COLUMNS assert set(land_use.LAND_COVER_REQUIRED_COLUMNS) <= set(land_use.REQUIRED_COLUMNS) + + +# --- #416 field-coverage gate: compare_bbox_metrics / render_coverage_report, +# offline with synthetic numbers. probe_bbox_metrics() itself is network- +# dependent and, per this script's own rule, never exercised here. + + +def _metrics(places_rows, brand_rate, confidence_rate, category_rate, addresses_rows): + return { + "places_rows": float(places_rows), + "brand_non_null_rate": brand_rate, + "confidence_non_null_rate": confidence_rate, + "category_non_null_rate": category_rate, + "addresses_rows": float(addresses_rows), + } + + +def test_probe_set_is_three_to_five_bboxes_with_comments(): + assert 3 <= len(overture_canary.PROBE_METROS) <= 5 + names = [name for name, _, _ in overture_canary.PROBE_METROS] + assert len(names) == len(set(names)), "duplicate probe bbox name" + + +def test_healthy_release_flags_nothing(): + pinned = {"Paris": _metrics(1000, 0.5, 0.9, 0.95, 2000)} + # Small, ordinary jitter — well within REGRESSION_TOLERANCE. + newest = {"Paris": _metrics(1010, 0.49, 0.91, 0.94, 1990)} + rows = overture_canary.compare_bbox_metrics(pinned, newest) + assert not any(r["regression"] for r in rows) + assert overture_canary.render_coverage_report(rows, "2026-08-19.0", "2026-09-16.0") == [] + + +def test_regression_detected_with_correct_table_numbers(): + """A synthetic brand-collapse (#546's shape): places/confidence/category + hold, brand's non-null rate craters — the exact metric #546 was about.""" + pinned = {"Sao Paulo": _metrics(10_000, 0.60, 0.90, 0.95, 5_000)} + newest = {"Sao Paulo": _metrics(10_000, 0.40, 0.90, 0.95, 5_000)} # brand -33% + rows = overture_canary.compare_bbox_metrics(pinned, newest) + flagged = [r for r in rows if r["regression"]] + assert len(flagged) == 1 + row = flagged[0] + assert row["bbox"] == "Sao Paulo" + assert row["metric"] == "brand_non_null_rate" + assert row["pinned"] == 0.60 + assert row["newest"] == 0.40 + assert row["delta_pct"] == pytest.approx((0.40 - 0.60) / 0.60 * 100.0) + assert row["delta_pct"] < -overture_canary.REGRESSION_TOLERANCE * 100.0 + + report = overture_canary.render_coverage_report(rows, "2026-08-19.0", "2026-09-16.0") + assert report, "regression must produce a non-empty report" + joined = "\n".join(report) + assert "Sao Paulo" in joined + assert "brand non-null rate" in joined + assert "0.6" in joined and "0.4" in joined + assert "-33.3%" in joined + assert "OvertureMaps/data#546" in joined + + +def test_row_count_metric_regression_also_flagged(): + pinned = {"Tokyo": _metrics(5_000, 0.5, 0.9, 0.95, 3_000)} + newest = {"Tokyo": _metrics(3_000, 0.5, 0.9, 0.95, 3_000)} # places rows -40% + rows = overture_canary.compare_bbox_metrics(pinned, newest) + flagged = {r["metric"] for r in rows if r["regression"]} + assert "places_rows" in flagged + + +def test_small_denominator_is_skipped_not_flagged(): + """MIN_ROWS floors row-count-backed metrics: a tiny pinned count makes a + percentage meaningless (one missing row is a "100% drop").""" + tiny = overture_canary.MIN_ROWS - 1 + pinned = {"Tiny": _metrics(tiny, 0.5, 0.9, 0.95, 3_000)} + newest = {"Tiny": _metrics(0, 0.0, 0.0, 0.0, 3_000)} + rows = overture_canary.compare_bbox_metrics(pinned, newest) + by_metric = {r["metric"]: r for r in rows} + for metric in ("places_rows", "brand_non_null_rate", "confidence_non_null_rate"): + assert by_metric[metric]["regression"] is False + assert by_metric[metric]["skip_reason"] is not None + assert by_metric[metric]["delta_pct"] is None + # addresses_rows' own denominator (3_000) clears the floor, so it is + # still compared even though places_rows didn't. + assert by_metric["addresses_rows"]["skip_reason"] is None + assert overture_canary.render_coverage_report(rows, "2026-08-19.0", "2026-09-16.0") == [] + + +def test_pinned_zero_denominator_is_skipped(): + pinned = {"Empty": _metrics(0, 0.0, 0.0, 0.0, 0)} + newest = {"Empty": _metrics(0, 0.0, 0.0, 0.0, 0)} + rows = overture_canary.compare_bbox_metrics(pinned, newest) + assert all(r["skip_reason"] is not None for r in rows) + assert all(r["regression"] is False for r in rows) + + +def test_bbox_missing_from_one_side_is_not_compared(): + pinned = {"A": _metrics(1000, 0.5, 0.9, 0.95, 500), "B": _metrics(1000, 0.5, 0.9, 0.95, 500)} + newest = {"A": _metrics(1000, 0.5, 0.9, 0.95, 500)} # B's probe failed upstream + rows = overture_canary.compare_bbox_metrics(pinned, newest) + assert {r["bbox"] for r in rows} == {"A"} From 600c5aac1ee0cd72934d178dce4c7a3784e06129 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 25 Aug 2026 03:56:13 +0000 Subject: [PATCH 2/2] overture_canary: floor rate numerators, probe the live bucket directly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The MIN_ROWS floor now also applies to a rate metric's pinned non-null count: 800 places rows with 6 branded ones turned two rows moving into a -33% 'drop' — a recurring false alarm in exactly the sparse-brand metros the probe set targets. Offline test added. - Coverage-probe globs are built from DEFAULT_UPSTREAM_BASE directly, matching the column probe: a runner's PLACEROOT_DATA_PATH* pin or a registered override can no longer redirect the probes or raise UpstreamUnavailable outside the try block (which aborted the run and discarded findings the other checks had already collected). - Corrected the probe-box size comment (half-width semantics, and longitude degrees widen away from the equator). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01YCuTkstNj5KscUQ9u8cLMK --- scripts/overture_canary.py | 35 +++++++++++++++++++++++++++++------ tests/test_overture_canary.py | 17 +++++++++++++++++ 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/scripts/overture_canary.py b/scripts/overture_canary.py index 8b0702c..ba35a5e 100644 --- a/scripts/overture_canary.py +++ b/scripts/overture_canary.py @@ -98,10 +98,11 @@ def probe_columns(con: duckdb.DuckDBPyConnection, glob: str) -> set[str] | None: ("Lagos Island", 6.4550, 3.3841), # Africa — the metric this gate exists for (#546) ] -# geo.bbox_around's half-width scales with radius/111_320m; 5_500m keeps -# every probe box in the "~0.1 degree class" the issue asked for (about -# 0.1 deg tall, narrower in longitude away from the equator) while still -# covering enough of a dense downtown to make row counts meaningful. +# geo.bbox_around takes a half-width: 5_500m makes each box ~0.1 deg in +# half-height (~0.2 deg tall in total, wider in longitude *degrees* away +# from the equator, where a degree of longitude covers fewer meters) — +# the "~0.1 degree class" the issue asked for, while still covering +# enough of a dense downtown to make row counts meaningful. PROBE_RADIUS_M = 5_500.0 # Relative drop (newest vs pinned) on any one probe metric that flags a @@ -161,8 +162,18 @@ def probe_bbox_metrics( """ xmin, ymin, xmax, ymax = geo.bbox_around(lat, lon, PROBE_RADIUS_M) filter_sql, params = geo.bbox_filter_sql(xmin, ymin, xmax, ymax) - places_glob = overture.upstream_glob("places", "place", release=release_name) - addresses_glob = overture.upstream_glob("addresses", "address", release=release_name) + # Globs built directly from DEFAULT_UPSTREAM_BASE, exactly like the + # column probe above: the canary's job is to look at the *live* bucket, + # so a runner's PLACEROOT_DATA_PATH* pin or a registered override must + # neither redirect these probes nor raise UpstreamUnavailable out of + # them (which would abort the run and discard findings already + # collected by the other checks). + places_glob = ( + f"{overture.DEFAULT_UPSTREAM_BASE}/{release_name}/theme=places/type=place/*" + ) + addresses_glob = ( + f"{overture.DEFAULT_UPSTREAM_BASE}/{release_name}/theme=addresses/type=address/*" + ) try: n, brand_n, confidence_n, category_n = con.execute( f""" @@ -236,6 +247,18 @@ def compare_bbox_metrics( row["skip_reason"] = f"pinned denominator {denom:g} < MIN_ROWS ({MIN_ROWS})" rows.append(row) continue + if metric.endswith("_non_null_rate") and denom is not None: + # For the rate metrics the noisy quantity is the *numerator* + # (the pinned non-null count), not the row count: 800 places + # rows with 6 branded ones still make a -33% "drop" out of + # two rows moving. Floor it with the same MIN_ROWS. + non_null = pinned_value * denom + if non_null < MIN_ROWS: + row["skip_reason"] = ( + f"pinned non-null count {non_null:g} < MIN_ROWS ({MIN_ROWS})" + ) + rows.append(row) + continue if pinned_value == 0: row["skip_reason"] = "pinned value is 0" rows.append(row) diff --git a/tests/test_overture_canary.py b/tests/test_overture_canary.py index ef207ed..dbb23c7 100644 --- a/tests/test_overture_canary.py +++ b/tests/test_overture_canary.py @@ -153,6 +153,23 @@ def test_small_denominator_is_skipped_not_flagged(): assert overture_canary.render_coverage_report(rows, "2026-08-19.0", "2026-09-16.0") == [] +def test_small_numerator_is_skipped_for_rate_metrics(): + """The rate metrics' noisy quantity is the pinned non-null *count*: 800 + places rows with 6 branded ones would turn two rows moving into a -33% + "drop". The MIN_ROWS floor applies to that count too.""" + pinned = {"Lagos": _metrics(800, 6 / 800, 0.9, 0.95, 3_000)} + newest = {"Lagos": _metrics(800, 4 / 800, 0.9, 0.95, 3_000)} + rows = overture_canary.compare_bbox_metrics(pinned, newest) + by_metric = {r["metric"]: r for r in rows} + brand = by_metric["brand_non_null_rate"] + assert brand["regression"] is False + assert brand["skip_reason"] is not None + assert brand["delta_pct"] is None + # A rate whose pinned non-null count clears the floor is still compared. + assert by_metric["confidence_non_null_rate"]["skip_reason"] is None + assert by_metric["places_rows"]["skip_reason"] is None + + def test_pinned_zero_denominator_is_skipped(): pinned = {"Empty": _metrics(0, 0.0, 0.0, 0.0, 0)} newest = {"Empty": _metrics(0, 0.0, 0.0, 0.0, 0)}