From 92bd04a1d5345db9650835050d5a797aa5321f3c Mon Sep 17 00:00:00 2001 From: Michael Carlone Date: Sat, 18 Jul 2026 09:47:08 -0400 Subject: [PATCH 01/18] chore: gitignore specs/ working directory specs/ is a local-only living record for the consumption+observability v1 effort; keep it out of version control. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index b2b60b08..254ea6f3 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ CLAUDE.md # Python bytecode cache (e.g. from running scripts/release/*.py) __pycache__/ *.pyc +specs/ From 4f6872673893f2b351a762e95f9accd007b7b518 Mon Sep 17 00:00:00 2001 From: Michael Carlone Date: Sat, 18 Jul 2026 09:53:58 -0400 Subject: [PATCH 02/18] C-01: add dispatched cast_to_utc_date() helper Cross-adapter macro converting a stored run timestamp to its UTC calendar date. default__ and snowflake__ both plain-cast to date: run_started_at is stored TIMESTAMP_NTZ holding dbt's UTC value, so the cast is session-TZ invariant. Shared infra for consumption + observability daily-grain marts. --- .../datetime_helpers.sql | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 macros/database_specific_helpers/datetime_helpers.sql diff --git a/macros/database_specific_helpers/datetime_helpers.sql b/macros/database_specific_helpers/datetime_helpers.sql new file mode 100644 index 00000000..00732389 --- /dev/null +++ b/macros/database_specific_helpers/datetime_helpers.sql @@ -0,0 +1,31 @@ +{#- + cast_to_utc_date(column_expr) + + Converts a stored run timestamp to its UTC calendar date. Shared + infrastructure for every daily-grain mart in the consumption and + observability features. + + Storage assumption (Snowflake): dbt captures run_started_at (and the + execution-level *_started_at / *_completed_at columns) in UTC, and the + package stores them via type_timestamp(), which on Snowflake is + TIMESTAMP_NTZ -- a wall-clock value carrying no timezone. Casting a + TIMESTAMP_NTZ to date simply truncates to the date part, so the result is + invariant to the session TIMEZONE setting and is already the correct UTC + date. convert_timezone() is deliberately NOT used: on an NTZ input it + assumes the value is in the session timezone and would reintroduce session + dependence, corrupting the date near midnight. +-#} + +{% macro cast_to_utc_date(column_expr) %} + {{ return(adapter.dispatch('cast_to_utc_date', 'dbt_artifacts')(column_expr)) }} +{% endmacro %} + +{% macro default__cast_to_utc_date(column_expr) %} + cast({{ column_expr }} as date) +{% endmacro %} + +{% macro snowflake__cast_to_utc_date(column_expr) %} + {#- run_started_at is TIMESTAMP_NTZ holding dbt's UTC timestamp; a direct + date cast yields the UTC calendar date and is session-TZ invariant. -#} + cast({{ column_expr }} as date) +{% endmacro %} From 58572501893cebba4885978ff595dce8db56bf12 Mon Sep 17 00:00:00 2001 From: Michael Carlone Date: Sat, 18 Jul 2026 09:55:56 -0400 Subject: [PATCH 03/18] O-01: add dispatched median()/percentile()/p95() helpers ANSI percentile_cont(fraction) within group ordered-set aggregates, usable in GROUP BY queries; Snowflake supports them natively so no snowflake__ override needed. p95() is thin sugar over percentile(col, 0.95). For the observability performance-baseline math. --- .../statistical_helpers.sql | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 macros/database_specific_helpers/statistical_helpers.sql diff --git a/macros/database_specific_helpers/statistical_helpers.sql b/macros/database_specific_helpers/statistical_helpers.sql new file mode 100644 index 00000000..728edf9c --- /dev/null +++ b/macros/database_specific_helpers/statistical_helpers.sql @@ -0,0 +1,36 @@ +{#- + Statistical helpers for the performance-baseline math (observability marts). + + Both macros are aggregate expressions intended for use in GROUP BY queries + (e.g. per-day, per-node medians). The default__ implementations use the + ANSI ordered-set aggregate percentile_cont(fraction) within group + (order by ...), which Snowflake supports natively as an aggregate, so no + snowflake__ override is required. Non-Snowflake overrides (SQL Server + window-only syntax, Spark approx_percentile) are fast-follow O-12. +-#} + +{#- MEDIAN -#} + +{% macro median(column_expr) %} + {{ return(adapter.dispatch('median', 'dbt_artifacts')(column_expr)) }} +{% endmacro %} + +{% macro default__median(column_expr) %} + percentile_cont(0.5) within group (order by {{ column_expr }}) +{% endmacro %} + +{#- PERCENTILE (arbitrary fraction in [0, 1]) -#} + +{% macro percentile(column_expr, fraction) %} + {{ return(adapter.dispatch('percentile', 'dbt_artifacts')(column_expr, fraction)) }} +{% endmacro %} + +{% macro default__percentile(column_expr, fraction) %} + percentile_cont({{ fraction }}) within group (order by {{ column_expr }}) +{% endmacro %} + +{#- P95 -- thin sugar over percentile(col, 0.95); not dispatched. -#} + +{% macro p95(column_expr) %} + {{ dbt_artifacts.percentile(column_expr, 0.95) }} +{% endmacro %} From c454fc4a9c9bec06d2b70852726891e61feb448c Mon Sep 17 00:00:00 2001 From: Michael Carlone Date: Sat, 18 Jul 2026 10:01:35 -0400 Subject: [PATCH 04/18] C-02: add classify_invocation_billing() macro Dispatched case-expression classifying invocations as deployment/development per the design precedence table (cloud job -> deployment targets -> optional env-var rule -> fallback), with a count_all_invocations escape hatch. Rule 3 (env_vars JSON) lives only in snowflake__ (env_vars is OBJECT via type_json); default__ raises a clear compile error if rule 3 is requested off Snowflake. Additive: does not modify stg_dbt__invocations or any existing contract. --- .../classify_invocation_billing.sql | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 macros/consumption/classify_invocation_billing.sql diff --git a/macros/consumption/classify_invocation_billing.sql b/macros/consumption/classify_invocation_billing.sql new file mode 100644 index 00000000..2284d315 --- /dev/null +++ b/macros/consumption/classify_invocation_billing.sql @@ -0,0 +1,84 @@ +{#- + classify_invocation_billing() + + Returns a scalar-per-row SQL `case` expression that classifies each + invocation as 'deployment' or 'development'. dbt bills only orchestrated + deployment builds, so every consumption metric filters on this. + + Consumed inside the new consumption models' CTEs only -- it references the + columns dbt_cloud_job_id, target_name and env_vars, which must be in scope + (they are on stg_dbt__invocations). This macro does NOT modify any existing + staging/source contract. + + Precedence (first match wins), per specs/consumption/design.md: + 1. dbt_cloud_job_id is not null -> deployment + 2. lower(target_name) in deployment_targets (lowered) -> deployment + 3. optional: env var named by dbt_artifacts_deployment_env_var is + present and truthy in env_vars -> deployment + fallback -> development + + Escape hatch: dbt_artifacts_count_all_invocations = true classes every + invocation as deployment. + + Rule 3 needs adapter-specific JSON access, so it lives only in snowflake__ + (env_vars is stored via type_json() = OBJECT on Snowflake, accessed with + the variant path env_vars:"NAME"). It is emitted only when the var is set, + so the default path parses no JSON. default__ raises a clear compile error + if rule 3 is requested on an unsupported adapter (fast-follow C-11). +-#} + +{% macro classify_invocation_billing() %} + {{ return(adapter.dispatch('classify_invocation_billing', 'dbt_artifacts')()) }} +{% endmacro %} + + +{#- Shared predicate for rule 2: lower(target_name) in the (lowered) deployment + target list. Emits `false` for an empty list so `in ()` is never produced. -#} +{% macro _billing_deployment_targets_predicate() %} + {%- set targets = var('dbt_artifacts_deployment_targets', ['prod', 'production', 'ci']) -%} + {%- if targets | length == 0 -%} + false + {%- else -%} + lower(target_name) in ( + {%- for t in targets -%} + '{{ t | lower }}'{% if not loop.last %}, {% endif %} + {%- endfor -%} + ) + {%- endif -%} +{% endmacro %} + + +{% macro default__classify_invocation_billing() %} + {%- if var('dbt_artifacts_count_all_invocations', false) -%} + cast('deployment' as {{ dbt.type_string() }}) + {%- else -%} + {%- if var('dbt_artifacts_deployment_env_var', none) is not none -%} + {{ exceptions.raise_compiler_error( + "dbt_artifacts_deployment_env_var (billing classification rule 3) requires adapter-specific JSON support and is only implemented for Snowflake in v1. Unset the var or run on Snowflake." + ) }} + {%- endif -%} + case + when dbt_cloud_job_id is not null then 'deployment' + when {{ dbt_artifacts._billing_deployment_targets_predicate() }} then 'deployment' + else 'development' + end + {%- endif -%} +{% endmacro %} + + +{% macro snowflake__classify_invocation_billing() %} + {%- if var('dbt_artifacts_count_all_invocations', false) -%} + cast('deployment' as {{ dbt.type_string() }}) + {%- else -%} + {%- set env_var_name = var('dbt_artifacts_deployment_env_var', none) -%} + case + when dbt_cloud_job_id is not null then 'deployment' + when {{ dbt_artifacts._billing_deployment_targets_predicate() }} then 'deployment' + {%- if env_var_name is not none %} + when coalesce(env_vars:"{{ env_var_name }}"::string, '') not in ('', 'false', 'False', '0') + then 'deployment' + {%- endif %} + else 'development' + end + {%- endif -%} +{% endmacro %} From e5fd7f51e40e614428473edae2fcdc708d62dee4 Mon Sep 17 00:00:00 2001 From: Michael Carlone Date: Sat, 18 Jul 2026 10:10:00 -0400 Subject: [PATCH 05/18] O-02: add dim_dbt__lineage_edges + flatten_json_array() helper New view exploding depends_on_nodes (native ARRAY on Snowflake) into one row per parent x child edge from the latest graph state, across models/snapshots/ tests. Snowflake-gated (enabled = target.type == 'snowflake'); JSON-array explode via new dispatched flatten_json_array() helper whose default__ raises a clear compile error naming the adapter (fast-follow O-12/C-11). Adds an idiomatic lineage_edge_id surrogate key so grain is testable without a dbt_utils dependency. Additive; no existing model/contract touched. --- .../flatten_json_array.sql | 36 +++++++ models/dim_dbt__lineage_edges.sql | 93 +++++++++++++++++++ models/dim_dbt__lineage_edges.yml | 38 ++++++++ 3 files changed, 167 insertions(+) create mode 100644 macros/database_specific_helpers/flatten_json_array.sql create mode 100644 models/dim_dbt__lineage_edges.sql create mode 100644 models/dim_dbt__lineage_edges.yml diff --git a/macros/database_specific_helpers/flatten_json_array.sql b/macros/database_specific_helpers/flatten_json_array.sql new file mode 100644 index 00000000..86e1b34b --- /dev/null +++ b/macros/database_specific_helpers/flatten_json_array.sql @@ -0,0 +1,36 @@ +{#- + flatten_json_array(array_column, alias) + + Renders a lateral table-function FROM-clause fragment that explodes an + array-valued column into one row per element. Intended for use in the + FROM clause after a comma: + + from my_model + , {{ dbt_artifacts.flatten_json_array('my_model.depends_on_nodes', 'dep') }} + + The per-element scalar is then accessed as `{{ alias }}.value` (cast as + needed). On Snowflake the package's depends_on_nodes columns are native + ARRAY (type_array()), so the input is flattened directly -- no parse_json. + + Snowflake only in v1. Other adapters raise a clear compile error; the model + that uses this helper is itself gated to Snowflake. Cross-adapter overrides + (BigQuery unnest, Postgres jsonb_array_elements_text, Trino unnest, Spark + explode, SQL Server openjson) are fast-follow (O-12 / C-11) and coordinated + with specs/materialize-docs/design.md. +-#} + +{% macro flatten_json_array(array_column, alias) %} + {{ return(adapter.dispatch('flatten_json_array', 'dbt_artifacts')(array_column, alias)) }} +{% endmacro %} + +{% macro default__flatten_json_array(array_column, alias) %} + {{ exceptions.raise_compiler_error( + "dbt_artifacts.flatten_json_array() is only implemented for Snowflake in v1 (adapter '" + ~ target.type + ~ "' is unsupported for now). The models using it are Snowflake-gated; cross-adapter support is fast-follow O-12 / C-11." + ) }} +{% endmacro %} + +{% macro snowflake__flatten_json_array(array_column, alias) %} + lateral flatten(input => {{ array_column }}) as {{ alias }} +{% endmacro %} diff --git a/models/dim_dbt__lineage_edges.sql b/models/dim_dbt__lineage_edges.sql new file mode 100644 index 00000000..74c4b888 --- /dev/null +++ b/models/dim_dbt__lineage_edges.sql @@ -0,0 +1,93 @@ +{{ config(enabled = target.type == "snowflake") }} + +with + models as ( + select + command_invocation_id + , node_id + , run_started_at + , depends_on_nodes + , 'model' as resource_type + from {{ ref("stg_dbt__models") }} + ), + + snapshots as ( + select + command_invocation_id + , node_id + , run_started_at + , depends_on_nodes + , 'snapshot' as resource_type + from {{ ref("stg_dbt__snapshots") }} + ), + + tests as ( + select + command_invocation_id + , node_id + , run_started_at + , depends_on_nodes + , 'test' as resource_type + from {{ ref("stg_dbt__tests") }} + ), + + all_nodes as ( + select * from models + union all + select * from snapshots + union all + select * from tests + ), + + latest_per_node as ( + + {# Latest graph state: keep only each node's most-recent appearance, + mirroring the dedupe intent of dim_dbt__current_models. #} + select + command_invocation_id + , node_id + , depends_on_nodes + , resource_type + , row_number() over ( + partition by node_id order by run_started_at desc + ) as run_rank + from all_nodes + ), + + latest_graph as ( + select * from latest_per_node where run_rank = 1 + ), + + edges as ( + select + dep.value::string as parent_node_id + , latest_graph.node_id as child_node_id + , latest_graph.resource_type as child_resource_type + , latest_graph.command_invocation_id as edge_source_invocation_id + from latest_graph + , {{ dbt_artifacts.flatten_json_array("latest_graph.depends_on_nodes", "dep") }} + ), + + distinct_edges as ( + select distinct + parent_node_id + , child_node_id + , child_resource_type + , edge_source_invocation_id + from edges + ), + + final as ( + select + {{ dbt_artifacts.generate_surrogate_key([ + "parent_node_id", "child_node_id", "child_resource_type" + ]) }} as lineage_edge_id + , parent_node_id + , child_node_id + , child_resource_type + , edge_source_invocation_id + from distinct_edges + ) + +select * +from final diff --git a/models/dim_dbt__lineage_edges.yml b/models/dim_dbt__lineage_edges.yml new file mode 100644 index 00000000..d69fe563 --- /dev/null +++ b/models/dim_dbt__lineage_edges.yml @@ -0,0 +1,38 @@ +version: 2 + +models: +- name: dim_dbt__lineage_edges + description: > + One row per parent_node_id x child_node_id dependency edge from the latest + graph state, built by exploding the depends_on_nodes arrays the package + already uploads for models, snapshots and tests. Unblocks stall-time and + flaky-test parentage. **Snowflake only in v1** (enabled = target.type == + 'snowflake'); the JSON-array explode uses the dispatched flatten_json_array() + helper, whose non-Snowflake overrides are fast-follow (O-12 / C-11). Latest + graph state = each node's most-recent appearance (row_number by run_started_at). + columns: + - name: lineage_edge_id + description: > + Surrogate key of the edge (hash of parent_node_id, child_node_id, + child_resource_type). Added so the grain can be tested without a dbt_utils + dependency; also a stable BI join key. + tests: + - unique + - not_null + - name: parent_node_id + description: > + node_id (or source id) the child depends on, i.e. the upstream end of the + edge. Sourced from the child's depends_on_nodes array; may reference a + source (source.*) as well as a model/snapshot/seed. + tests: + - not_null + - name: child_node_id + description: The downstream node_id that declares the dependency. + tests: + - not_null + - name: child_resource_type + description: "Resource type of the child node: 'model', 'snapshot' or 'test'." + - name: edge_source_invocation_id + description: > + command_invocation_id of the latest-graph-state row the edge was derived + from (the invocation in which the child most recently appeared). From b07663eddbffdbd9510f96a697d74a1609bdcdd4 Mon Sep 17 00:00:00 2001 From: Michael Carlone Date: Sat, 18 Jul 2026 10:23:37 -0400 Subject: [PATCH 06/18] C-03: add fct_dbt__consumption_daily (+ _detail) Core consumption marts over existing staging. _detail is the base grain (day x meter x materialization x target_name); _daily is a strict sum roll-up of _detail so the two reconcile by construction. Meters: 'smb' (deployment model successes on run/build/retry) and 'active_target_tables' (distinct deployment-active nodes per target, DATT upper-bound proxy). Uses C-01 cast_to_utc_date + C-02 classify_invocation_billing. Additive views only. --- models/fct_dbt__consumption_daily.sql | 32 +++++ models/fct_dbt__consumption_daily.yml | 31 +++++ models/fct_dbt__consumption_daily_detail.sql | 135 +++++++++++++++++++ models/fct_dbt__consumption_daily_detail.yml | 39 ++++++ 4 files changed, 237 insertions(+) create mode 100644 models/fct_dbt__consumption_daily.sql create mode 100644 models/fct_dbt__consumption_daily.yml create mode 100644 models/fct_dbt__consumption_daily_detail.sql create mode 100644 models/fct_dbt__consumption_daily_detail.yml diff --git a/models/fct_dbt__consumption_daily.sql b/models/fct_dbt__consumption_daily.sql new file mode 100644 index 00000000..0474ccd4 --- /dev/null +++ b/models/fct_dbt__consumption_daily.sql @@ -0,0 +1,32 @@ +{#- + Core consumption mart: one row per UTC day x meter. + + A strict roll-up of fct_dbt__consumption_daily_detail (sum of quantity), + so daily totals always reconcile exactly to the detail grain. Meters and + their definitions live in the detail model's header. +-#} + +with + detail as (select * from {{ ref("fct_dbt__consumption_daily_detail") }}), + + aggregated as ( + select + date_day + , meter + , sum(quantity) as quantity + from detail + group by date_day, meter + ), + + final as ( + select + {{ dbt_artifacts.generate_surrogate_key(["date_day", "meter"]) }} + as consumption_daily_id + , date_day + , meter + , quantity + from aggregated + ) + +select * +from final diff --git a/models/fct_dbt__consumption_daily.yml b/models/fct_dbt__consumption_daily.yml new file mode 100644 index 00000000..311d6ac7 --- /dev/null +++ b/models/fct_dbt__consumption_daily.yml @@ -0,0 +1,31 @@ +version: 2 + +models: +- name: fct_dbt__consumption_daily + description: > + Core consumption mart: one row per UTC day x meter. A strict roll-up of + fct_dbt__consumption_daily_detail (sum of quantity), so daily totals always + reconcile exactly to the detail grain. v1 meters: 'smb' (Successful Models + Built, mirroring dbt's published rules) and 'active_target_tables' (distinct + deployment-active nodes, the DATT upper-bound proxy). Consumption metrics + only count deployment-classed invocations (see classify_invocation_billing). + columns: + - name: consumption_daily_id + description: Surrogate key of the grain (hash of date_day, meter). Stable BI join key. + tests: + - unique + - not_null + - name: date_day + description: UTC calendar day of the executions (via cast_to_utc_date()). + tests: + - not_null + - name: meter + description: "Consumption meter: 'smb' or 'active_target_tables'." + tests: + - not_null + - accepted_values: + values: ['smb', 'active_target_tables'] + - name: quantity + description: Metered quantity for the day x meter (sum of the detail rows). + tests: + - not_null diff --git a/models/fct_dbt__consumption_daily_detail.sql b/models/fct_dbt__consumption_daily_detail.sql new file mode 100644 index 00000000..0748604c --- /dev/null +++ b/models/fct_dbt__consumption_daily_detail.sql @@ -0,0 +1,135 @@ +{#- + Detail grain for consumption: UTC day x meter x materialization x target_name. + This is the BASE grain; fct_dbt__consumption_daily is a strict roll-up of + this model (sum of quantity), so the two always reconcile by construction. + + Meters (v1): + - 'smb' : Successful Models Built. One row per successful model execution + in a deployment-classed invocation whose dbt_command is + run/build/retry (mirrors dbt's published SMB rules). Additive. + - 'active_target_tables' : distinct node_ids (models u seeds u snapshots u + tests) with >=1 successful deployment execution that day, counted + per deployment target (materialization is null). On single- + deployment-target setups (the common case) this equals the + per-day distinct-node count in specs/consumption/design.md; when + a node runs under several deployment targets in one day it is + counted once per target, which keeps the roll-up additive. +-#} + +with + model_executions as ( + select + me.node_id + , me.status + , me.materialization + , {{ dbt_artifacts.cast_to_utc_date("me.run_started_at") }} as date_day + , i.target_name + , i.dbt_command + , {{ dbt_artifacts.classify_invocation_billing() }} as billing_class + from {{ ref("stg_dbt__model_executions") }} as me + inner join {{ ref("stg_dbt__invocations") }} as i + on me.command_invocation_id = i.command_invocation_id + ), + + smb_detail as ( + select + date_day + , 'smb' as meter + , materialization + , target_name + , count(*) as quantity + from model_executions + where status = 'success' + and dbt_command in ('run', 'build', 'retry') + and billing_class = 'deployment' + group by date_day, materialization, target_name + ), + + all_executions as ( + select + node_id + , status + , run_started_at + , command_invocation_id + from {{ ref("stg_dbt__model_executions") }} + union all + select + node_id + , status + , run_started_at + , command_invocation_id + from {{ ref("stg_dbt__seed_executions") }} + union all + select + node_id + , status + , run_started_at + , command_invocation_id + from {{ ref("stg_dbt__snapshot_executions") }} + union all + select + node_id + , status + , run_started_at + , command_invocation_id + from {{ ref("stg_dbt__test_executions") }} + ), + + active_executions as ( + select + ae.node_id + , ae.status + , {{ dbt_artifacts.cast_to_utc_date("ae.run_started_at") }} as date_day + , i.target_name + , {{ dbt_artifacts.classify_invocation_billing() }} as billing_class + from all_executions as ae + inner join {{ ref("stg_dbt__invocations") }} as i + on ae.command_invocation_id = i.command_invocation_id + ), + + active_target_tables_detail as ( + select + date_day + , 'active_target_tables' as meter + , cast(null as {{ dbt.type_string() }}) as materialization + , target_name + , count(distinct node_id) as quantity + from active_executions + where status = 'success' + and billing_class = 'deployment' + group by date_day, target_name + ), + + combined as ( + select + date_day + , meter + , materialization + , target_name + , quantity + from smb_detail + union all + select + date_day + , meter + , materialization + , target_name + , quantity + from active_target_tables_detail + ), + + final as ( + select + {{ dbt_artifacts.generate_surrogate_key([ + "date_day", "meter", "materialization", "target_name" + ]) }} as consumption_daily_detail_id + , date_day + , meter + , materialization + , target_name + , quantity + from combined + ) + +select * +from final diff --git a/models/fct_dbt__consumption_daily_detail.yml b/models/fct_dbt__consumption_daily_detail.yml new file mode 100644 index 00000000..6ffdba87 --- /dev/null +++ b/models/fct_dbt__consumption_daily_detail.yml @@ -0,0 +1,39 @@ +version: 2 + +models: +- name: fct_dbt__consumption_daily_detail + description: > + Detail (base) grain for consumption: UTC day x meter x materialization x + target_name. fct_dbt__consumption_daily is a strict roll-up of this model. + For 'smb' rows, materialization is the model's materialization; for + 'active_target_tables' rows materialization is null and the count is + distinct nodes per deployment target that day. Only deployment-classed + invocations are counted. + columns: + - name: consumption_daily_detail_id + description: > + Surrogate key of the grain (hash of date_day, meter, materialization, + target_name). Stable BI join key. + tests: + - unique + - not_null + - name: date_day + description: UTC calendar day of the executions (via cast_to_utc_date()). + tests: + - not_null + - name: meter + description: "Consumption meter: 'smb' or 'active_target_tables'." + tests: + - not_null + - accepted_values: + values: ['smb', 'active_target_tables'] + - name: materialization + description: > + Model materialization for 'smb' rows; null for 'active_target_tables' + (a cross-resource distinct count with no single materialization). + - name: target_name + description: dbt target name (target.name) of the invocations contributing the rows. + - name: quantity + description: Metered quantity for this detail cell. + tests: + - not_null From 97c40b14ac7d7b64a132e84324935525a72f5948 Mon Sep 17 00:00:00 2001 From: Michael Carlone Date: Sat, 18 Jul 2026 10:31:24 -0400 Subject: [PATCH 07/18] C-05: add fct_dbt__consumption_by_model + is_between test macro Per-model monthly SMB burn, cadence, runtime and dbt State ROI estimate (distinct_days_built x datt_price). Same SMB filter as consumption_daily so monthly totals reconcile across marts. Adds a reusable dependency-free generic range test dbt_artifacts.is_between (used here for pct_of_month_smb in [0,1]; reused by O-tier marts). Additive views only. --- macros/tests/is_between.sql | 23 ++++++++ models/fct_dbt__consumption_by_model.sql | 72 ++++++++++++++++++++++++ models/fct_dbt__consumption_by_model.yml | 46 +++++++++++++++ 3 files changed, 141 insertions(+) create mode 100644 macros/tests/is_between.sql create mode 100644 models/fct_dbt__consumption_by_model.sql create mode 100644 models/fct_dbt__consumption_by_model.yml diff --git a/macros/tests/is_between.sql b/macros/tests/is_between.sql new file mode 100644 index 00000000..abaecbb5 --- /dev/null +++ b/macros/tests/is_between.sql @@ -0,0 +1,23 @@ +{#- + Generic schema test: fails for any row whose column_name is outside the + inclusive range [min_value, max_value]. A dependency-free range guard + (dbt_utils.accepted_range equivalent) used by the consumption/observability + marts. Null column values are ignored (pair with not_null where required). + + Usage in a model .yml: + columns: + - name: pct_of_month_smb + tests: + - dbt_artifacts.is_between: + min_value: 0 + max_value: 1 +-#} + +{% test is_between(model, column_name, min_value, max_value) %} + +select {{ column_name }} as value_out_of_range +from {{ model }} +where {{ column_name }} < {{ min_value }} + or {{ column_name }} > {{ max_value }} + +{% endtest %} diff --git a/models/fct_dbt__consumption_by_model.sql b/models/fct_dbt__consumption_by_model.sql new file mode 100644 index 00000000..1d8d1c4e --- /dev/null +++ b/models/fct_dbt__consumption_by_model.sql @@ -0,0 +1,72 @@ +{#- + "Where is the consumption going" mart: one row per billing_month x node_id + (models only), with SMB burn, build cadence, runtime, and the dbt State ROI + estimate. + + Uses the same SMB filter as fct_dbt__consumption_daily (deployment-classed + invocation, status = 'success', dbt_command in run/build/retry) so the two + marts reconcile. billing_month is the calendar month (date_trunc) of the + UTC execution day. v1 excludes attached-test DATT counting (design's v2). +-#} + +with + model_executions as ( + select + me.node_id + , me.name + , me.status + , me.total_node_runtime + , {{ dbt_artifacts.cast_to_utc_date("me.run_started_at") }} as date_day + , i.dbt_command + , {{ dbt_artifacts.classify_invocation_billing() }} as billing_class + from {{ ref("stg_dbt__model_executions") }} as me + inner join {{ ref("stg_dbt__invocations") }} as i + on me.command_invocation_id = i.command_invocation_id + ), + + smb_executions as ( + select + node_id + , name + , total_node_runtime + , date_day + , date_trunc('month', date_day) as billing_month + from model_executions + where status = 'success' + and dbt_command in ('run', 'build', 'retry') + and billing_class = 'deployment' + ), + + by_model as ( + select + billing_month + , node_id + , max(name) as name + , count(*) as smb_quantity + , count(distinct date_day) as distinct_days_built + , sum(total_node_runtime) as total_runtime_seconds + from smb_executions + group by billing_month, node_id + ), + + final as ( + select + {{ dbt_artifacts.generate_surrogate_key(["billing_month", "node_id"]) }} + as consumption_by_model_id + , billing_month + , node_id + , name + , smb_quantity + , smb_quantity + / sum(smb_quantity) over (partition by billing_month) + as pct_of_month_smb + , smb_quantity / nullif(distinct_days_built, 0) as builds_per_day_avg + , distinct_days_built + , total_runtime_seconds + , distinct_days_built * {{ var("dbt_artifacts_datt_price", 0.094) }} + as estimated_monthly_datt_cost_if_reused + from by_model + ) + +select * +from final diff --git a/models/fct_dbt__consumption_by_model.yml b/models/fct_dbt__consumption_by_model.yml new file mode 100644 index 00000000..1e238ef0 --- /dev/null +++ b/models/fct_dbt__consumption_by_model.yml @@ -0,0 +1,46 @@ +version: 2 + +models: +- name: fct_dbt__consumption_by_model + description: > + Consumption attributed per model: one row per billing_month x node_id + (models only). Same SMB definition as fct_dbt__consumption_daily + (deployment-classed successful model builds on run/build/retry), so monthly + SMB totals reconcile across the two marts. Includes build cadence, total + runtime, and a dbt State ROI estimate. Primary use case: "top 20 models by + SMB burn." + columns: + - name: consumption_by_model_id + description: Surrogate key of the grain (hash of billing_month, node_id). BI join key. + tests: + - unique + - not_null + - name: billing_month + description: First day of the calendar month (UTC) the SMB were built in. + tests: + - not_null + - name: node_id + description: The model's unique node id. + tests: + - not_null + - name: name + description: The model's name. + - name: smb_quantity + description: Successful Models Built for this model in the month (deployment, run/build/retry). + - name: pct_of_month_smb + description: This model's share of the month's total SMB (0..1), a window over billing_month. + tests: + - dbt_artifacts.is_between: + min_value: 0 + max_value: 1 + - name: builds_per_day_avg + description: Average successful builds per active build-day (smb_quantity / distinct_days_built). + - name: distinct_days_built + description: Number of distinct UTC days on which the model was successfully built (deployment). + - name: total_runtime_seconds + description: Sum of total_node_runtime over the model's SMB executions in the month. + - name: estimated_monthly_datt_cost_if_reused + description: > + dbt State ROI estimate = distinct_days_built x dbt_artifacts_datt_price + (default 0.094). A directional upper-bound of what dbt State could meter + for this model; see specs/consumption/design.md. From fc148e0c26853afb1ea0379b4461567698bc29bb Mon Sep 17 00:00:00 2001 From: Michael Carlone Date: Sat, 18 Jul 2026 10:38:34 -0400 Subject: [PATCH 08/18] O-03: add fct_dbt__run_health_daily (+ _detail) Per-UTC-day run-health rollup across all invocations (no billing filter): invocation/command counts, node success/failure/error/skip counts across models+seeds+snapshots+tests, success_rate, runtime, first/last timestamps, failed-invocation count. _detail adds a target_name cut. success_rate guarded with dbt_artifacts.is_between. Additive views only. --- models/fct_dbt__run_health_daily.sql | 130 +++++++++++++++++++ models/fct_dbt__run_health_daily.yml | 45 +++++++ models/fct_dbt__run_health_daily_detail.sql | 134 ++++++++++++++++++++ models/fct_dbt__run_health_daily_detail.yml | 48 +++++++ 4 files changed, 357 insertions(+) create mode 100644 models/fct_dbt__run_health_daily.sql create mode 100644 models/fct_dbt__run_health_daily.yml create mode 100644 models/fct_dbt__run_health_daily_detail.sql create mode 100644 models/fct_dbt__run_health_daily_detail.yml diff --git a/models/fct_dbt__run_health_daily.sql b/models/fct_dbt__run_health_daily.sql new file mode 100644 index 00000000..e103fc87 --- /dev/null +++ b/models/fct_dbt__run_health_daily.sql @@ -0,0 +1,130 @@ +{#- + Run-health rollup: one row per UTC day across ALL invocations (no billing + classification -- dev failures are still failures). Node counts span + models + seeds + snapshots + tests. Companion fct_dbt__run_health_daily_detail + adds a target_name cut. + + Status mapping (defensive across adapters/resource types): + success = status in ('success','pass'); failure = ('fail','failure'); + error = ('error'); skip = ('skipped','skip'). + success_rate = successes / (successes + failures + errors) (skips excluded). + A "failed invocation" is any invocation with >=1 node in error or failure. +-#} + +with + invocations as ( + select + command_invocation_id + , dbt_command + , target_name + , run_started_at + , {{ dbt_artifacts.cast_to_utc_date("run_started_at") }} as date_day + from {{ ref("stg_dbt__invocations") }} + ), + + executions as ( + select + command_invocation_id + , node_id + , status + , total_node_runtime + , run_started_at + from {{ ref("stg_dbt__model_executions") }} + union all + select + command_invocation_id + , node_id + , status + , total_node_runtime + , run_started_at + from {{ ref("stg_dbt__seed_executions") }} + union all + select + command_invocation_id + , node_id + , status + , total_node_runtime + , run_started_at + from {{ ref("stg_dbt__snapshot_executions") }} + union all + select + command_invocation_id + , node_id + , status + , total_node_runtime + , run_started_at + from {{ ref("stg_dbt__test_executions") }} + ), + + exec_classified as ( + select + e.command_invocation_id + , i.date_day + , e.total_node_runtime + , e.run_started_at + , case when e.status in ('success', 'pass') then 1 else 0 end as is_success + , case when e.status in ('fail', 'failure') then 1 else 0 end as is_failure + , case when e.status = 'error' then 1 else 0 end as is_error + , case when e.status in ('skipped', 'skip') then 1 else 0 end as is_skip + from executions as e + inner join invocations as i + on e.command_invocation_id = i.command_invocation_id + ), + + node_daily as ( + select + date_day + , sum(is_success) as node_successes + , sum(is_failure) as node_failures + , sum(is_error) as node_errors + , sum(is_skip) as node_skips + , sum(total_node_runtime) as total_runtime_seconds + , max(total_node_runtime) as max_node_runtime_seconds + , min(run_started_at) as first_run_started_at + , max(run_started_at) as last_run_started_at + from exec_classified + group by date_day + ), + + invocation_daily as ( + select + date_day + , count(distinct command_invocation_id) as invocations + , count(distinct dbt_command) as distinct_commands + from invocations + group by date_day + ), + + failed_invocation_daily as ( + select + date_day + , count(distinct command_invocation_id) as failed_invocations + from exec_classified + where is_error = 1 or is_failure = 1 + group by date_day + ), + + final as ( + select + inv.date_day + , inv.invocations + , inv.distinct_commands + , coalesce(nd.node_successes, 0) as node_successes + , coalesce(nd.node_failures, 0) as node_failures + , coalesce(nd.node_errors, 0) as node_errors + , coalesce(nd.node_skips, 0) as node_skips + , coalesce(fid.failed_invocations, 0) as failed_invocations + , nd.node_successes + / nullif(nd.node_successes + nd.node_failures + nd.node_errors, 0) + as success_rate + , coalesce(nd.total_runtime_seconds, 0) as total_runtime_seconds + , nd.max_node_runtime_seconds + , nd.first_run_started_at + , nd.last_run_started_at + from invocation_daily as inv + left join node_daily as nd on inv.date_day = nd.date_day + left join failed_invocation_daily as fid on inv.date_day = fid.date_day + ) + +select * +from final diff --git a/models/fct_dbt__run_health_daily.yml b/models/fct_dbt__run_health_daily.yml new file mode 100644 index 00000000..cc26d535 --- /dev/null +++ b/models/fct_dbt__run_health_daily.yml @@ -0,0 +1,45 @@ +version: 2 + +models: +- name: fct_dbt__run_health_daily + description: > + Run-health rollup: one row per UTC day across all invocations (no billing + classification). Node counts span models + seeds + snapshots + tests. + success = status in (success, pass); failure = (fail, failure); + error = (error); skip = (skipped, skip). success_rate = + successes / (successes + failures + errors). A failed invocation has >=1 + node in error or failure. + columns: + - name: date_day + description: UTC calendar day (via cast_to_utc_date() on invocation run_started_at). + tests: + - not_null + - unique + - name: invocations + description: Distinct command_invocation_ids that day. + - name: distinct_commands + description: Distinct dbt_command values that day. + - name: node_successes + description: Successful node executions (models + seeds + snapshots + tests). + - name: node_failures + description: Failed node executions (test fails). + - name: node_errors + description: Errored node executions. + - name: node_skips + description: Skipped node executions. + - name: failed_invocations + description: Invocations with >=1 node in error or failure. + - name: success_rate + description: successes / (successes + failures + errors); null on days with no such executions. + tests: + - dbt_artifacts.is_between: + min_value: 0 + max_value: 1 + - name: total_runtime_seconds + description: Sum of total_node_runtime across all node executions that day. + - name: max_node_runtime_seconds + description: Longest single node execution that day. + - name: first_run_started_at + description: Earliest node run_started_at that day. + - name: last_run_started_at + description: Latest node run_started_at that day. diff --git a/models/fct_dbt__run_health_daily_detail.sql b/models/fct_dbt__run_health_daily_detail.sql new file mode 100644 index 00000000..92c6900c --- /dev/null +++ b/models/fct_dbt__run_health_daily_detail.sql @@ -0,0 +1,134 @@ +{#- + Run-health rollup at UTC day x target_name grain (companion to + fct_dbt__run_health_daily). Same status mapping and definitions as the + daily model; adds the target_name cut. Additive count columns sum to the + daily model across targets; distinct_commands does not (a command may run + under several targets) and is therefore per-target here. +-#} + +with + invocations as ( + select + command_invocation_id + , dbt_command + , target_name + , run_started_at + , {{ dbt_artifacts.cast_to_utc_date("run_started_at") }} as date_day + from {{ ref("stg_dbt__invocations") }} + ), + + executions as ( + select + command_invocation_id + , node_id + , status + , total_node_runtime + , run_started_at + from {{ ref("stg_dbt__model_executions") }} + union all + select + command_invocation_id + , node_id + , status + , total_node_runtime + , run_started_at + from {{ ref("stg_dbt__seed_executions") }} + union all + select + command_invocation_id + , node_id + , status + , total_node_runtime + , run_started_at + from {{ ref("stg_dbt__snapshot_executions") }} + union all + select + command_invocation_id + , node_id + , status + , total_node_runtime + , run_started_at + from {{ ref("stg_dbt__test_executions") }} + ), + + exec_classified as ( + select + e.command_invocation_id + , i.date_day + , i.target_name + , e.total_node_runtime + , e.run_started_at + , case when e.status in ('success', 'pass') then 1 else 0 end as is_success + , case when e.status in ('fail', 'failure') then 1 else 0 end as is_failure + , case when e.status = 'error' then 1 else 0 end as is_error + , case when e.status in ('skipped', 'skip') then 1 else 0 end as is_skip + from executions as e + inner join invocations as i + on e.command_invocation_id = i.command_invocation_id + ), + + node_daily as ( + select + date_day + , target_name + , sum(is_success) as node_successes + , sum(is_failure) as node_failures + , sum(is_error) as node_errors + , sum(is_skip) as node_skips + , sum(total_node_runtime) as total_runtime_seconds + , max(total_node_runtime) as max_node_runtime_seconds + , min(run_started_at) as first_run_started_at + , max(run_started_at) as last_run_started_at + from exec_classified + group by date_day, target_name + ), + + invocation_daily as ( + select + date_day + , target_name + , count(distinct command_invocation_id) as invocations + , count(distinct dbt_command) as distinct_commands + from invocations + group by date_day, target_name + ), + + failed_invocation_daily as ( + select + date_day + , target_name + , count(distinct command_invocation_id) as failed_invocations + from exec_classified + where is_error = 1 or is_failure = 1 + group by date_day, target_name + ), + + final as ( + select + {{ dbt_artifacts.generate_surrogate_key(["inv.date_day", "inv.target_name"]) }} + as run_health_daily_detail_id + , inv.date_day + , inv.target_name + , inv.invocations + , inv.distinct_commands + , coalesce(nd.node_successes, 0) as node_successes + , coalesce(nd.node_failures, 0) as node_failures + , coalesce(nd.node_errors, 0) as node_errors + , coalesce(nd.node_skips, 0) as node_skips + , coalesce(fid.failed_invocations, 0) as failed_invocations + , nd.node_successes + / nullif(nd.node_successes + nd.node_failures + nd.node_errors, 0) + as success_rate + , coalesce(nd.total_runtime_seconds, 0) as total_runtime_seconds + , nd.max_node_runtime_seconds + , nd.first_run_started_at + , nd.last_run_started_at + from invocation_daily as inv + left join node_daily as nd + on inv.date_day = nd.date_day and inv.target_name = nd.target_name + left join failed_invocation_daily as fid + on inv.date_day = fid.date_day and inv.target_name = fid.target_name + ) + +select * +from final diff --git a/models/fct_dbt__run_health_daily_detail.yml b/models/fct_dbt__run_health_daily_detail.yml new file mode 100644 index 00000000..a7aa0f3c --- /dev/null +++ b/models/fct_dbt__run_health_daily_detail.yml @@ -0,0 +1,48 @@ +version: 2 + +models: +- name: fct_dbt__run_health_daily_detail + description: > + Run-health rollup at UTC day x target_name grain (companion to + fct_dbt__run_health_daily). Same status mapping and definitions; additive + count columns sum to the daily model across targets. + columns: + - name: run_health_daily_detail_id + description: Surrogate key of the grain (hash of date_day, target_name). BI join key. + tests: + - unique + - not_null + - name: date_day + description: UTC calendar day (via cast_to_utc_date()). + tests: + - not_null + - name: target_name + description: dbt target name (target.name) of the invocations. + - name: invocations + description: Distinct command_invocation_ids that day for the target. + - name: distinct_commands + description: Distinct dbt_command values that day for the target. + - name: node_successes + description: Successful node executions for the target. + - name: node_failures + description: Failed node executions for the target. + - name: node_errors + description: Errored node executions for the target. + - name: node_skips + description: Skipped node executions for the target. + - name: failed_invocations + description: Invocations with >=1 node in error or failure for the target. + - name: success_rate + description: successes / (successes + failures + errors) for the target. + tests: + - dbt_artifacts.is_between: + min_value: 0 + max_value: 1 + - name: total_runtime_seconds + description: Sum of total_node_runtime for the target that day. + - name: max_node_runtime_seconds + description: Longest single node execution for the target that day. + - name: first_run_started_at + description: Earliest node run_started_at for the target that day. + - name: last_run_started_at + description: Latest node run_started_at for the target that day. From 0ca0386cbab8912fcd73a1ed5b3d9d17c7fb79cd Mon Sep 17 00:00:00 2001 From: Michael Carlone Date: Sat, 18 Jul 2026 10:46:43 -0400 Subject: [PATCH 09/18] O-04: add fct_dbt__model_performance (regression detection) Per day x node runtime stats vs a trailing 28d same-DOW baseline. median/p95/ rows over successful non-full-refresh executions; full_refresh_executions exposed separately. baseline_runtime = median of same-node same-ISO-DOW prior median_runtime within run_rate_days, excluding current day. is_regressed gated on regression_threshold (1.5) AND >= regression_min_samples (3) same-DOW days. Enhances is_between to allow open-ended bounds (ratio >= 0 test). Additive. --- macros/tests/is_between.sql | 19 ++--- models/fct_dbt__model_performance.sql | 100 ++++++++++++++++++++++++++ models/fct_dbt__model_performance.yml | 55 ++++++++++++++ 3 files changed, 166 insertions(+), 8 deletions(-) create mode 100644 models/fct_dbt__model_performance.sql create mode 100644 models/fct_dbt__model_performance.yml diff --git a/macros/tests/is_between.sql b/macros/tests/is_between.sql index abaecbb5..87e8bb49 100644 --- a/macros/tests/is_between.sql +++ b/macros/tests/is_between.sql @@ -1,6 +1,7 @@ {#- - Generic schema test: fails for any row whose column_name is outside the - inclusive range [min_value, max_value]. A dependency-free range guard + Generic schema test: fails for any row whose column_name falls outside the + inclusive range [min_value, max_value]. Either bound may be omitted (null) + for an open-ended check. A dependency-free range guard (dbt_utils.accepted_range equivalent) used by the consumption/observability marts. Null column values are ignored (pair with not_null where required). @@ -8,16 +9,18 @@ columns: - name: pct_of_month_smb tests: - - dbt_artifacts.is_between: - min_value: 0 - max_value: 1 + - dbt_artifacts.is_between: {min_value: 0, max_value: 1} + - name: runtime_regression_ratio + tests: + - dbt_artifacts.is_between: {min_value: 0} # >= 0, no upper bound -#} -{% test is_between(model, column_name, min_value, max_value) %} +{% test is_between(model, column_name, min_value=none, max_value=none) %} select {{ column_name }} as value_out_of_range from {{ model }} -where {{ column_name }} < {{ min_value }} - or {{ column_name }} > {{ max_value }} +where + {% if min_value is not none %}{{ column_name }} < {{ min_value }}{% else %}1 = 0{% endif %} + or {% if max_value is not none %}{{ column_name }} > {{ max_value }}{% else %}1 = 0{% endif %} {% endtest %} diff --git a/models/fct_dbt__model_performance.sql b/models/fct_dbt__model_performance.sql new file mode 100644 index 00000000..6b4ed7b1 --- /dev/null +++ b/models/fct_dbt__model_performance.sql @@ -0,0 +1,100 @@ +{#- + Runtime-regression detection: one row per UTC day x node_id (models). + + Runtime stats (median/p95/rows_affected) are computed over SUCCESSFUL, + NON-full-refresh executions only -- a full refresh is not a regression, and + failures are O-03's job. full_refresh_executions is exposed as a count so + the signal isn't lost. + + baseline_runtime = median of the SAME node's SAME-day-of-week median_runtime + over the trailing dbt_artifacts_run_rate_days (default 28) days, EXCLUDING + the current day (28 days = 4 balanced weekday samples). is_regressed fires + only when the ratio exceeds dbt_artifacts_regression_threshold (default 1.5) + AND the baseline was built from at least dbt_artifacts_regression_min_samples + (default 3) same-DOW days -- so a thin baseline never cries wolf. +-#} + +with + executions as ( + select + node_id + , {{ dbt_artifacts.cast_to_utc_date("run_started_at") }} as date_day + , status + , total_node_runtime + , rows_affected + , was_full_refresh + from {{ ref("stg_dbt__model_executions") }} + ), + + daily as ( + select + date_day + , node_id + , count(*) as executions + , sum(case when status = 'success' then 1 else 0 end) as success_count + , sum(case when was_full_refresh then 1 else 0 end) as full_refresh_executions + , {{ dbt_artifacts.median( + "case when status = 'success' and not was_full_refresh then total_node_runtime end" + ) }} as median_runtime + , {{ dbt_artifacts.p95( + "case when status = 'success' and not was_full_refresh then total_node_runtime end" + ) }} as p95_runtime + , sum( + case when status = 'success' and not was_full_refresh then rows_affected else 0 end + ) as rows_affected_sum + from executions + group by date_day, node_id + ), + + baseline as ( + select + cur.date_day + , cur.node_id + , {{ dbt_artifacts.median("hist.median_runtime") }} as baseline_runtime + , count(distinct hist.date_day) as baseline_sample_days + from daily as cur + left join daily as hist + on cur.node_id = hist.node_id + and dayofweekiso(hist.date_day) = dayofweekiso(cur.date_day) + and hist.date_day < cur.date_day + and hist.date_day + >= dateadd('day', -{{ var("dbt_artifacts_run_rate_days", 28) }}, cur.date_day) + and hist.median_runtime is not null + group by cur.date_day, cur.node_id + ), + + final as ( + select + {{ dbt_artifacts.generate_surrogate_key(["daily.date_day", "daily.node_id"]) }} + as model_performance_id + , daily.date_day + , daily.node_id + , daily.executions + , daily.success_count + , daily.full_refresh_executions + , daily.median_runtime + , daily.p95_runtime + , daily.rows_affected_sum + , baseline.baseline_runtime + , baseline.baseline_sample_days + , case + when baseline.baseline_runtime > 0 + then daily.median_runtime / baseline.baseline_runtime + end as runtime_regression_ratio + , case + when baseline.baseline_runtime > 0 + and daily.median_runtime / baseline.baseline_runtime + > {{ var("dbt_artifacts_regression_threshold", 1.5) }} + and baseline.baseline_sample_days + >= {{ var("dbt_artifacts_regression_min_samples", 3) }} + then true + else false + end as is_regressed + from daily + left join baseline + on daily.date_day = baseline.date_day + and daily.node_id = baseline.node_id + ) + +select * +from final diff --git a/models/fct_dbt__model_performance.yml b/models/fct_dbt__model_performance.yml new file mode 100644 index 00000000..5bd877f3 --- /dev/null +++ b/models/fct_dbt__model_performance.yml @@ -0,0 +1,55 @@ +version: 2 + +models: +- name: fct_dbt__model_performance + description: > + Per-model daily runtime stats vs a trailing weekday-aware baseline: the + "why is the 6am job slow since Tuesday" mart. One row per UTC day x node_id + (models). Runtime stats use successful, non-full-refresh executions only. + baseline_runtime is the median of the same node's same-day-of-week + median_runtime over the trailing dbt_artifacts_run_rate_days (default 28) + days, excluding the current day. is_regressed requires ratio > + dbt_artifacts_regression_threshold (default 1.5) and a baseline of at least + dbt_artifacts_regression_min_samples (default 3) same-DOW days. + columns: + - name: model_performance_id + description: Surrogate key of the grain (hash of date_day, node_id). BI join key. + tests: + - unique + - not_null + - name: date_day + description: UTC calendar day (via cast_to_utc_date()). + tests: + - not_null + - name: node_id + description: The model's unique node id. + tests: + - not_null + - name: executions + description: Total executions of the model that day (all statuses). + - name: success_count + description: Successful executions that day. + - name: full_refresh_executions + description: Executions that were full refreshes (excluded from runtime stats and baseline). + - name: median_runtime + description: Median total_node_runtime over successful non-full-refresh executions that day. + - name: p95_runtime + description: 95th-percentile total_node_runtime over successful non-full-refresh executions that day. + - name: rows_affected_sum + description: Sum of rows_affected over successful non-full-refresh executions that day. + - name: baseline_runtime + description: > + Trailing weekday-aware baseline: median of the same node's same-DOW + median_runtime over the prior dbt_artifacts_run_rate_days days (excludes + today). Null until enough same-DOW history exists. + - name: baseline_sample_days + description: Number of same-DOW prior days contributing to the baseline. + - name: runtime_regression_ratio + description: median_runtime / baseline_runtime; null when baseline is null or 0. + tests: + - dbt_artifacts.is_between: + min_value: 0 + - name: is_regressed + description: > + True when runtime_regression_ratio > regression_threshold and + baseline_sample_days >= regression_min_samples; else false. From 49a473b2cc69f23b47fe4bc7c651900534590952 Mon Sep 17 00:00:00 2001 From: Michael Carlone Date: Sat, 18 Jul 2026 10:51:55 -0400 Subject: [PATCH 10/18] O-06: add fct_dbt__dag_bottlenecks (+ _detail) measured stall time Per parent model x UTC day: measured wall-clock stall its children spent waiting (child.compile_started_at - binding parent.query_completed_at, clamped >=0). Binding parent = latest-completing model-parent per (invocation, child) via qualify. Roll-up counts only gating rows (stall>0). Detail view for drill-down. Depends on dim_dbt__lineage_edges (O-02); Snowflake-gated. Descendant counts/blocking scores parked (v1 scope). Additive views only. --- models/fct_dbt__dag_bottlenecks.sql | 35 +++++++++ models/fct_dbt__dag_bottlenecks.yml | 38 ++++++++++ models/fct_dbt__dag_bottlenecks_detail.sql | 88 ++++++++++++++++++++++ models/fct_dbt__dag_bottlenecks_detail.yml | 39 ++++++++++ 4 files changed, 200 insertions(+) create mode 100644 models/fct_dbt__dag_bottlenecks.sql create mode 100644 models/fct_dbt__dag_bottlenecks.yml create mode 100644 models/fct_dbt__dag_bottlenecks_detail.sql create mode 100644 models/fct_dbt__dag_bottlenecks_detail.yml diff --git a/models/fct_dbt__dag_bottlenecks.sql b/models/fct_dbt__dag_bottlenecks.sql new file mode 100644 index 00000000..3067bc8f --- /dev/null +++ b/models/fct_dbt__dag_bottlenecks.sql @@ -0,0 +1,35 @@ +{{ config(enabled = target.type == "snowflake") }} + +{#- + Per parent model x UTC day: how much wall-clock time its children spent + waiting on it (measured stall). The "pinch point" mart. Roll-up of + fct_dbt__dag_bottlenecks_detail over rows where the parent actually gated a + child (stall_seconds > 0). Descendant counts / blocking scores are out of + scope for v1 (parked). Snowflake-only (inherits O-02 enablement). +-#} + +with + detail as (select * from {{ ref("fct_dbt__dag_bottlenecks_detail") }}), + + gating as ( + select * + from detail + where stall_seconds > 0 + ), + + final as ( + select + {{ dbt_artifacts.generate_surrogate_key(["date_day", "binding_parent_node_id"]) }} + as dag_bottleneck_id + , date_day + , binding_parent_node_id as parent_node_id + , count(distinct child_node_id) as blocked_children + , sum(stall_seconds) as total_stall_seconds + , max(stall_seconds) as max_stall_seconds + , count(distinct command_invocation_id) as invocations_observed + from gating + group by date_day, binding_parent_node_id + ) + +select * +from final diff --git a/models/fct_dbt__dag_bottlenecks.yml b/models/fct_dbt__dag_bottlenecks.yml new file mode 100644 index 00000000..87b23279 --- /dev/null +++ b/models/fct_dbt__dag_bottlenecks.yml @@ -0,0 +1,38 @@ +version: 2 + +models: +- name: fct_dbt__dag_bottlenecks + description: > + Pinch-point mart: per parent model x UTC day, the measured wall-clock time + its children spent waiting on it. Roll-up of fct_dbt__dag_bottlenecks_detail + over rows where the parent was the binding constraint and actually gated a + child (stall_seconds > 0). Measured gating, not graph theory; descendant + counts / blocking scores are out of scope for v1. Snowflake-only. + columns: + - name: dag_bottleneck_id + description: Surrogate key of the grain (hash of date_day, parent_node_id). BI join key. + tests: + - unique + - not_null + - name: date_day + description: UTC calendar day (via cast_to_utc_date()). + tests: + - not_null + - name: parent_node_id + description: The parent model node id that gated its children. + tests: + - not_null + - name: blocked_children + description: Distinct child models for which this parent was the binding (latest-completing) parent that day. + - name: total_stall_seconds + description: Sum of measured stall seconds attributed to this parent that day. + tests: + - dbt_artifacts.is_between: + min_value: 0 + - name: max_stall_seconds + description: Largest single measured stall attributed to this parent that day. + tests: + - dbt_artifacts.is_between: + min_value: 0 + - name: invocations_observed + description: Distinct invocations in which this parent gated at least one child that day. diff --git a/models/fct_dbt__dag_bottlenecks_detail.sql b/models/fct_dbt__dag_bottlenecks_detail.sql new file mode 100644 index 00000000..2a4811d6 --- /dev/null +++ b/models/fct_dbt__dag_bottlenecks_detail.sql @@ -0,0 +1,88 @@ +{{ config(enabled = target.type == "snowflake") }} + +{#- + Measured stall time, per (invocation, child, binding parent). For each + successful child model execution, its stall against the BINDING parent is + child.compile_started_at - max(parent.query_completed_at) among that child's + model-parents in the same invocation (the latest-completing parent is the + binding constraint). Negative stalls (child started before parent finished -- + different threads / non-blocking) clamp to 0. + + This is *measured* wall-clock gating, not graph theory. Model parents only; + seed/source parents are excluded (documented) -- seeds rarely gate and keep + the join simple. Depends on dim_dbt__lineage_edges (O-02); Snowflake-only. +-#} + +with + edges as ( + select + parent_node_id + , child_node_id + from {{ ref("dim_dbt__lineage_edges") }} + where child_resource_type = 'model' + and parent_node_id like 'model.%' + ), + + model_executions as ( + select + command_invocation_id + , node_id + , run_started_at + , compile_started_at + , query_completed_at + from {{ ref("stg_dbt__model_executions") }} + where status = 'success' + ), + + child_parent as ( + select + child.command_invocation_id + , child.node_id as child_node_id + , child.run_started_at + , child.compile_started_at as child_compile_started_at + , parent.node_id as parent_node_id + , parent.query_completed_at as parent_query_completed_at + from model_executions as child + inner join edges on child.node_id = edges.child_node_id + inner join model_executions as parent + on parent.node_id = edges.parent_node_id + and parent.command_invocation_id = child.command_invocation_id + ), + + binding_parent as ( + select + command_invocation_id + , child_node_id + , run_started_at + , child_compile_started_at + , parent_node_id + , parent_query_completed_at + from child_parent + qualify + row_number() over ( + partition by command_invocation_id, child_node_id + order by parent_query_completed_at desc, parent_node_id asc + ) + = 1 + ), + + final as ( + select + {{ dbt_artifacts.generate_surrogate_key(["command_invocation_id", "child_node_id"]) }} + as dag_bottleneck_detail_id + , command_invocation_id + , {{ dbt_artifacts.cast_to_utc_date("run_started_at") }} as date_day + , child_node_id + , parent_node_id as binding_parent_node_id + , parent_query_completed_at + , child_compile_started_at + , greatest( + timestampdiff('millisecond', parent_query_completed_at, child_compile_started_at) + / 1000.0, + 0 + ) as stall_seconds + from binding_parent + ) + +select * +from final diff --git a/models/fct_dbt__dag_bottlenecks_detail.yml b/models/fct_dbt__dag_bottlenecks_detail.yml new file mode 100644 index 00000000..e545ae22 --- /dev/null +++ b/models/fct_dbt__dag_bottlenecks_detail.yml @@ -0,0 +1,39 @@ +version: 2 + +models: +- name: fct_dbt__dag_bottlenecks_detail + description: > + One row per (invocation, child model), with the binding parent (the + latest-completing model-parent in that invocation) and the measured stall = + child.compile_started_at - binding parent.query_completed_at, clamped to >= 0. + Drill-down behind fct_dbt__dag_bottlenecks. Model parents only (seed/source + parents excluded). Snowflake-only. + columns: + - name: dag_bottleneck_detail_id + description: Surrogate key of the grain (hash of command_invocation_id, child_node_id). BI join key. + tests: + - unique + - not_null + - name: command_invocation_id + description: The invocation the child executed in. + tests: + - not_null + - name: date_day + description: UTC calendar day of the invocation (via cast_to_utc_date()). + tests: + - not_null + - name: child_node_id + description: The child model whose start was (potentially) gated. + tests: + - not_null + - name: binding_parent_node_id + description: The model-parent with the latest query_completed_at among the child's parents in the invocation. + - name: parent_query_completed_at + description: query_completed_at of the binding parent. + - name: child_compile_started_at + description: compile_started_at of the child. + - name: stall_seconds + description: max(child_compile_started_at - parent_query_completed_at, 0) in seconds. + tests: + - dbt_artifacts.is_between: + min_value: 0 From 0122a9e04da2b05e5d6f47a9fd590649d84a8d1c Mon Sep 17 00:00:00 2001 From: Michael Carlone Date: Sat, 18 Jul 2026 10:57:26 -0400 Subject: [PATCH 11/18] O-05: add fct_dbt__flaky_tests (+ _detail) Detects tests that flip fail/error -> pass on the same UTC day with no parent model rebuilt between (retry-until-green signal), via lag() over test executions + dim_dbt__lineage_edges. parent_rebuilt_between is false (parents exist, none rebuilt) or null (no model parents); candidates with a parent rebuild between are legit fixes and excluded. Rollup test x month with flips, flake_rate, is_flaky. Snowflake-gated. Additive views only. --- models/fct_dbt__flaky_tests.sql | 60 ++++++++++++++ models/fct_dbt__flaky_tests.yml | 38 +++++++++ models/fct_dbt__flaky_tests_detail.sql | 104 +++++++++++++++++++++++++ models/fct_dbt__flaky_tests_detail.yml | 34 ++++++++ 4 files changed, 236 insertions(+) create mode 100644 models/fct_dbt__flaky_tests.sql create mode 100644 models/fct_dbt__flaky_tests.yml create mode 100644 models/fct_dbt__flaky_tests_detail.sql create mode 100644 models/fct_dbt__flaky_tests_detail.yml diff --git a/models/fct_dbt__flaky_tests.sql b/models/fct_dbt__flaky_tests.sql new file mode 100644 index 00000000..a1ebcd31 --- /dev/null +++ b/models/fct_dbt__flaky_tests.sql @@ -0,0 +1,60 @@ +{{ config(enabled = target.type == "snowflake") }} + +{#- + Flaky-test rollup: one row per test_node_id x month. flips comes from + fct_dbt__flaky_tests_detail (genuine fail->pass-without-parent-rebuild + events); executions counts all of the test's executions that month. + is_flaky = flips >= dbt_artifacts_flaky_min_flips (default 2). Snowflake-only. +-#} + +with + test_executions as ( + select + node_id as test_node_id + , date_trunc('month', {{ dbt_artifacts.cast_to_utc_date("run_started_at") }}) + as month + from {{ ref("stg_dbt__test_executions") }} + ), + + executions_monthly as ( + select + test_node_id + , month + , count(*) as executions + from test_executions + group by test_node_id, month + ), + + flips_monthly as ( + select + test_node_id + , date_trunc('month', date_day) as month + , count(*) as flips + , max(passed_at) as last_flip_at + from {{ ref("fct_dbt__flaky_tests_detail") }} + group by test_node_id, date_trunc('month', date_day) + ), + + final as ( + select + {{ dbt_artifacts.generate_surrogate_key(["em.test_node_id", "em.month"]) }} + as flaky_test_id + , em.test_node_id + , em.month + , coalesce(fm.flips, 0) as flips + , em.executions + , coalesce(fm.flips, 0) / nullif(em.executions, 0) as flake_rate + , fm.last_flip_at + , case + when coalesce(fm.flips, 0) >= {{ var("dbt_artifacts_flaky_min_flips", 2) }} + then true + else false + end as is_flaky + from executions_monthly as em + left join flips_monthly as fm + on em.test_node_id = fm.test_node_id + and em.month = fm.month + ) + +select * +from final diff --git a/models/fct_dbt__flaky_tests.yml b/models/fct_dbt__flaky_tests.yml new file mode 100644 index 00000000..b85e3bbf --- /dev/null +++ b/models/fct_dbt__flaky_tests.yml @@ -0,0 +1,38 @@ +version: 2 + +models: +- name: fct_dbt__flaky_tests + description: > + Flaky-test rollup: one row per test_node_id x month. flips are genuine + fail->pass transitions with no parent rebuild in between (from + fct_dbt__flaky_tests_detail); executions is all of the test's executions + that month. is_flaky = flips >= dbt_artifacts_flaky_min_flips (default 2). + Snowflake-only. + columns: + - name: flaky_test_id + description: Surrogate key of the grain (hash of test_node_id, month). BI join key. + tests: + - unique + - not_null + - name: test_node_id + description: The test's unique node id. + tests: + - not_null + - name: month + description: First day of the calendar month (UTC). + tests: + - not_null + - name: flips + description: Number of genuine flip events for the test that month. + - name: executions + description: Total executions of the test that month. + - name: flake_rate + description: flips / executions (0..1). + tests: + - dbt_artifacts.is_between: + min_value: 0 + max_value: 1 + - name: last_flip_at + description: run_started_at of the most recent flip's pass that month (null if no flips). + - name: is_flaky + description: True when flips >= dbt_artifacts_flaky_min_flips (default 2). diff --git a/models/fct_dbt__flaky_tests_detail.sql b/models/fct_dbt__flaky_tests_detail.sql new file mode 100644 index 00000000..2fc7dab8 --- /dev/null +++ b/models/fct_dbt__flaky_tests_detail.sql @@ -0,0 +1,104 @@ +{{ config(enabled = target.type == "snowflake") }} + +{#- + One row per flip event: a test that went fail/error -> pass on the SAME UTC + day with NO parent model rebuilt in between (the "retry until green" signal). + + Flip (v1): order a test's executions by run_started_at; a pass whose + immediately preceding execution (same test) was a fail or error on the same + UTC day is a flip candidate. warn does NOT count as a fail. A candidate is a + genuine flip unless a parent model (via dim_dbt__lineage_edges) had a + successful build with run_started_at strictly between the fail and the pass + (that's a legitimate fix, excluded). If the test has no resolvable model + parents, we still count the flip but set parent_rebuilt_between = null -- + we don't hide the uncertainty. Depends on dim_dbt__lineage_edges (O-02); + Snowflake-only. +-#} + +with + test_executions as ( + select + node_id as test_node_id + , run_started_at + , status + , {{ dbt_artifacts.cast_to_utc_date("run_started_at") }} as date_day + from {{ ref("stg_dbt__test_executions") }} + ), + + sequenced as ( + select + test_node_id + , run_started_at + , status + , date_day + , lag(status) over ( + partition by test_node_id order by run_started_at + ) as prev_status + , lag(run_started_at) over ( + partition by test_node_id order by run_started_at + ) as prev_run_started_at + , lag(date_day) over ( + partition by test_node_id order by run_started_at + ) as prev_date_day + from test_executions + ), + + flip_candidates as ( + select + test_node_id + , date_day + , prev_run_started_at as failed_at + , run_started_at as passed_at + from sequenced + where status = 'pass' + and prev_status in ('fail', 'error') + and prev_date_day = date_day + ), + + test_model_parents as ( + select + child_node_id as test_node_id + , parent_node_id + from {{ ref("dim_dbt__lineage_edges") }} + where child_resource_type = 'test' + and parent_node_id like 'model.%' + ), + + candidate_parent_activity as ( + select + fc.test_node_id + , fc.date_day + , fc.failed_at + , fc.passed_at + , count(distinct tmp.parent_node_id) as n_parents + , count(distinct + case + when me.node_id is not null then me.node_id + end + ) as n_parents_rebuilt_between + from flip_candidates as fc + left join test_model_parents as tmp + on fc.test_node_id = tmp.test_node_id + left join {{ ref("stg_dbt__model_executions") }} as me + on me.node_id = tmp.parent_node_id + and me.status = 'success' + and me.run_started_at > fc.failed_at + and me.run_started_at < fc.passed_at + group by fc.test_node_id, fc.date_day, fc.failed_at, fc.passed_at + ), + + final as ( + select + {{ dbt_artifacts.generate_surrogate_key(["test_node_id", "failed_at", "passed_at"]) }} + as flaky_test_flip_id + , test_node_id + , date_day + , failed_at + , passed_at + , case when n_parents = 0 then null else false end as parent_rebuilt_between + from candidate_parent_activity + where n_parents_rebuilt_between = 0 + ) + +select * +from final diff --git a/models/fct_dbt__flaky_tests_detail.yml b/models/fct_dbt__flaky_tests_detail.yml new file mode 100644 index 00000000..4744aaae --- /dev/null +++ b/models/fct_dbt__flaky_tests_detail.yml @@ -0,0 +1,34 @@ +version: 2 + +models: +- name: fct_dbt__flaky_tests_detail + description: > + One row per flip event: a test that went fail/error -> pass on the same UTC + day with no parent model rebuilt in between. parent_rebuilt_between is false + when the test has resolvable model parents and none rebuilt between, or null + when no model parents resolve (uncertainty is surfaced, not hidden). + Snowflake-only. + columns: + - name: flaky_test_flip_id + description: Surrogate key of the flip event (hash of test_node_id, failed_at, passed_at). BI join key. + tests: + - unique + - not_null + - name: test_node_id + description: The test's unique node id. + tests: + - not_null + - name: date_day + description: UTC calendar day the flip occurred (via cast_to_utc_date()). + tests: + - not_null + - name: failed_at + description: run_started_at of the fail/error execution. + - name: passed_at + description: run_started_at of the following pass execution. + - name: parent_rebuilt_between + description: > + false = the test has model parents and none had a successful build between + failed_at and passed_at (genuine flake); null = no model parents resolved. + (Candidates where a parent rebuilt between are legitimate fixes and are + excluded from this view entirely.) From 89122aefb9c8da53d1dd939ece1aba34e88dd7ab Mon Sep 17 00:00:00 2001 From: Michael Carlone Date: Sat, 18 Jul 2026 11:12:50 -0400 Subject: [PATCH 12/18] C-04: add fct_dbt__consumption_forecast + get_smb_allowance macro Month-end forecast per billing_month x meter: MTD, weekday-aware projected month-end (trailing run_rate_days per-ISO-DOW average incl. zero days), projected allowance-breach date, pct of allowance. Allowance via get_smb_allowance() (explicit var -> plan default -> none). Remaining-day series generated with a Jinja loop (no seed, no dbt_utils). Snowflake-gated. --- macros/consumption/get_smb_allowance.sql | 28 +++ models/fct_dbt__consumption_forecast.sql | 219 +++++++++++++++++++++++ models/fct_dbt__consumption_forecast.yml | 52 ++++++ 3 files changed, 299 insertions(+) create mode 100644 macros/consumption/get_smb_allowance.sql create mode 100644 models/fct_dbt__consumption_forecast.sql create mode 100644 models/fct_dbt__consumption_forecast.yml diff --git a/macros/consumption/get_smb_allowance.sql b/macros/consumption/get_smb_allowance.sql new file mode 100644 index 00000000..fc4ca331 --- /dev/null +++ b/macros/consumption/get_smb_allowance.sql @@ -0,0 +1,28 @@ +{#- + get_smb_allowance() + + Resolves the monthly SMB allowance at compile time, returning a Python + number or none. Resolution order (per specs/consumption/design.md): + 1. var('dbt_artifacts_smb_allowance') -- explicit, wins + 2. plan default from var('dbt_artifacts_billing_plan') + (developer: 3000, starter: 15000, enterprise: 100000) + 3. none -- forecast projections still populate; breach columns stay null + + Plan allowances are published defaults that WILL drift; they are overridable + and documented here in one place. Returns none for an unknown plan name. +-#} + +{% macro get_smb_allowance() %} + {%- set explicit = var("dbt_artifacts_smb_allowance", none) -%} + {%- if explicit is not none -%} + {{ return(explicit) }} + {%- endif -%} + + {%- set plan = var("dbt_artifacts_billing_plan", none) -%} + {%- set plan_allowances = {"developer": 3000, "starter": 15000, "enterprise": 100000} -%} + {%- if plan is not none and plan in plan_allowances -%} + {{ return(plan_allowances[plan]) }} + {%- endif -%} + + {{ return(none) }} +{% endmacro %} diff --git a/models/fct_dbt__consumption_forecast.sql b/models/fct_dbt__consumption_forecast.sql new file mode 100644 index 00000000..083d4de0 --- /dev/null +++ b/models/fct_dbt__consumption_forecast.sql @@ -0,0 +1,219 @@ +{{ config(enabled = target.type == "snowflake") }} + +{#- + The forecast mart: one row per billing_month x meter with month-to-date + usage, a weekday-aware projected month-end total, the projected allowance- + breach date, and % of allowance used. + + Projection: over a trailing dbt_artifacts_run_rate_days (default 28) window + ending as_of (today, clamped to month end), compute the average quantity per + ISO day-of-week INCLUDING zero-build days (so weekends pull the rate down). + Each remaining calendar day of the month contributes its day-of-week average; + forecast_month_end = MTD + sum of those. forecast_exceeded_date is the first + remaining day whose cumulative expected (added to MTD) reaches the allowance + (or as_of when MTD already exceeds it). Allowance applies to the 'smb' meter + only. Pure SQL + Jinja day series -- no seed, no dbt_utils. +-#} + +{%- set run_rate_days = var("dbt_artifacts_run_rate_days", 28) -%} +{%- set allowance = dbt_artifacts.get_smb_allowance() -%} + +with + daily as ( + select + date_day + , meter + , quantity + from {{ ref("fct_dbt__consumption_daily") }} + ), + + month_anchors as ( + select + meter + , date_trunc('month', date_day) as billing_month + , last_day(date_trunc('month', date_day)) as month_end + , least(current_date(), last_day(date_trunc('month', date_day))) as as_of_date + , datediff( + 'day', least(current_date(), last_day(date_trunc('month', date_day))), + last_day(date_trunc('month', date_day)) + ) as days_remaining + from daily + group by meter, date_trunc('month', date_day) + ), + + month_to_date as ( + select + ma.meter + , ma.billing_month + , sum( + case when daily.date_day <= ma.as_of_date then daily.quantity else 0 end + ) as month_to_date_quantity + from month_anchors as ma + inner join daily + on daily.meter = ma.meter + and date_trunc('month', daily.date_day) = ma.billing_month + group by ma.meter, ma.billing_month + ), + + offsets as ( + {% for i in range(run_rate_days) -%} + select {{ i }} as day_offset + {%- if not loop.last %} + union all + {% endif %} + {%- endfor %} + ), + + trailing_days as ( + select + ma.meter + , ma.billing_month + , dayofweekiso(dateadd('day', -offsets.day_offset, ma.as_of_date)) as dow + , coalesce(daily.quantity, 0) as quantity + from month_anchors as ma + cross join offsets + left join daily + on daily.meter = ma.meter + and daily.date_day = dateadd('day', -offsets.day_offset, ma.as_of_date) + ), + + dow_average as ( + select + meter + , billing_month + , dow + , avg(quantity) as avg_quantity + from trailing_days + group by meter, billing_month, dow + ), + + daily_run_rate as ( + select + meter + , billing_month + , avg(quantity) as daily_run_rate + from trailing_days + group by meter, billing_month + ), + + day_numbers as ( + {% for i in range(1, 32) -%} + select {{ i }} as day_number + {%- if not loop.last %} + union all + {% endif %} + {%- endfor %} + ), + + remaining_days as ( + select + ma.meter + , ma.billing_month + , dateadd('day', day_numbers.day_number - 1, ma.billing_month) as calendar_date + , dayofweekiso( + dateadd('day', day_numbers.day_number - 1, ma.billing_month) + ) as dow + from month_anchors as ma + cross join day_numbers + where day_numbers.day_number <= day(ma.month_end) + and dateadd('day', day_numbers.day_number - 1, ma.billing_month) > ma.as_of_date + ), + + remaining_expected as ( + select + rd.meter + , rd.billing_month + , rd.calendar_date + , coalesce(da.avg_quantity, 0) as expected_quantity + , sum(coalesce(da.avg_quantity, 0)) over ( + partition by rd.meter, rd.billing_month + order by rd.calendar_date + rows between unbounded preceding and current row + ) as cumulative_expected + from remaining_days as rd + left join dow_average as da + on da.meter = rd.meter + and da.billing_month = rd.billing_month + and da.dow = rd.dow + ), + + remaining_summary as ( + select + meter + , billing_month + , sum(expected_quantity) as total_expected_remaining + from remaining_expected + group by meter, billing_month + ), + + {%- if allowance is not none %} + breach as ( + select + re.meter + , re.billing_month + , min(re.calendar_date) as forecast_exceeded_date + from remaining_expected as re + inner join month_to_date as mtd + on mtd.meter = re.meter + and mtd.billing_month = re.billing_month + where re.meter = 'smb' + and (mtd.month_to_date_quantity + re.cumulative_expected) >= {{ allowance }} + group by re.meter, re.billing_month + ), + {%- endif %} + + final as ( + select + {{ dbt_artifacts.generate_surrogate_key(["ma.meter", "ma.billing_month"]) }} + as consumption_forecast_id + , ma.billing_month + , ma.meter + , mtd.month_to_date_quantity + {%- if allowance is not none %} + , case when ma.meter = 'smb' then {{ allowance }} else null end as allowance + {%- else %} + , cast(null as {{ dbt.type_int() }}) as allowance + {%- endif %} + , drr.daily_run_rate + , mtd.month_to_date_quantity + coalesce(rs.total_expected_remaining, 0) + as forecast_month_end_quantity + {%- if allowance is not none %} + , case + when ma.meter = 'smb' and mtd.month_to_date_quantity >= {{ allowance }} + then ma.as_of_date + when ma.meter = 'smb' + then breach.forecast_exceeded_date + end as forecast_exceeded_date + , case + when ma.meter = 'smb' + then mtd.month_to_date_quantity / {{ allowance }} + end as pct_of_allowance_used + {%- else %} + , cast(null as {{ dbt.type_timestamp() }}) as forecast_exceeded_date + , cast(null as {{ dbt.type_float() }}) as pct_of_allowance_used + {%- endif %} + , ma.days_remaining + {%- if allowance is not none %} + , case + when ma.meter = 'smb' + then (mtd.month_to_date_quantity + coalesce(rs.total_expected_remaining, 0)) + > {{ allowance }} + end as is_on_pace_to_exceed + {%- else %} + , cast(null as {{ dbt_artifacts.type_boolean() }}) as is_on_pace_to_exceed + {%- endif %} + from month_anchors as ma + inner join month_to_date as mtd + on mtd.meter = ma.meter and mtd.billing_month = ma.billing_month + left join daily_run_rate as drr + on drr.meter = ma.meter and drr.billing_month = ma.billing_month + left join remaining_summary as rs + on rs.meter = ma.meter and rs.billing_month = ma.billing_month + {%- if allowance is not none %} + left join breach + on breach.meter = ma.meter and breach.billing_month = ma.billing_month + {%- endif %} + ) + +select * +from final diff --git a/models/fct_dbt__consumption_forecast.yml b/models/fct_dbt__consumption_forecast.yml new file mode 100644 index 00000000..035ee448 --- /dev/null +++ b/models/fct_dbt__consumption_forecast.yml @@ -0,0 +1,52 @@ +version: 2 + +models: +- name: fct_dbt__consumption_forecast + description: > + Month-end consumption forecast: one row per billing_month x meter with + month-to-date usage, a weekday-aware projected month-end total, the projected + allowance-breach date, and % of allowance used. Projection averages the + trailing dbt_artifacts_run_rate_days (default 28) per ISO day-of-week + (including zero-build days) and sums the remaining calendar days' expected + values. Allowance resolves via get_smb_allowance() and applies to the 'smb' + meter only; with no plan/allowance var the breach columns are null but + projections still populate. Snowflake-only in v1 (uses Snowflake date + functions); multi-adapter is fast-follow C-11. + columns: + - name: consumption_forecast_id + description: Surrogate key of the grain (hash of meter, billing_month). BI join key. + tests: + - unique + - not_null + - name: billing_month + description: First day of the calendar month (UTC). + tests: + - not_null + - name: meter + description: "Consumption meter (e.g. 'smb', 'active_target_tables')." + tests: + - not_null + - name: month_to_date_quantity + description: Sum of the meter's quantity from month start through as_of (today, clamped to month end). + tests: + - not_null + - name: allowance + description: Monthly SMB allowance (get_smb_allowance()); null for non-smb meters or when unset. + - name: daily_run_rate + description: Average daily quantity over the trailing run_rate_days window (includes zero-build days). + - name: forecast_month_end_quantity + description: month_to_date_quantity + sum of expected quantity over the remaining calendar days. + - name: forecast_exceeded_date + description: > + First remaining day whose cumulative expected (added to MTD) reaches the + allowance; as_of when MTD already exceeds it; null when no allowance or the + meter is not smb or the projection never crosses. + - name: pct_of_allowance_used + description: month_to_date_quantity / allowance for the smb meter; null otherwise. + tests: + - dbt_artifacts.is_between: + min_value: 0 + - name: days_remaining + description: Calendar days remaining in the month after as_of (0 for fully-elapsed months). + - name: is_on_pace_to_exceed + description: True when forecast_month_end_quantity > allowance (smb meter); null when no allowance. From 6f401ebc15c3a39800d19dfef70c8d6a0ef3cac7 Mon Sep 17 00:00:00 2001 From: Michael Carlone Date: Sat, 18 Jul 2026 11:12:50 -0400 Subject: [PATCH 13/18] Gate all new v1 marts to Snowflake (backwards compatibility) These marts use Snowflake-specific SQL (dayofweekiso, datediff, last_day, date_trunc semantics, etc.). dbt_artifacts is installed on 7 adapters; an un-gated Snowflake-only model would break a non-Snowflake consumer's dbt run. Gating enabled = target.type == 'snowflake' keeps v1 strictly additive: other adapters get no new models (identical behavior). Multi-adapter support with portable dispatch is fast-follow (C-11/O-12). Completes gating for C-03/C-05/ O-03/O-04 (O-02/O-05/O-06/C-04 already gated). --- models/fct_dbt__consumption_by_model.sql | 2 ++ models/fct_dbt__consumption_daily.sql | 2 ++ models/fct_dbt__consumption_daily_detail.sql | 2 ++ models/fct_dbt__model_performance.sql | 2 ++ models/fct_dbt__run_health_daily.sql | 2 ++ models/fct_dbt__run_health_daily_detail.sql | 2 ++ 6 files changed, 12 insertions(+) diff --git a/models/fct_dbt__consumption_by_model.sql b/models/fct_dbt__consumption_by_model.sql index 1d8d1c4e..6daa9b17 100644 --- a/models/fct_dbt__consumption_by_model.sql +++ b/models/fct_dbt__consumption_by_model.sql @@ -1,3 +1,5 @@ +{{ config(enabled = target.type == "snowflake") }} + {#- "Where is the consumption going" mart: one row per billing_month x node_id (models only), with SMB burn, build cadence, runtime, and the dbt State ROI diff --git a/models/fct_dbt__consumption_daily.sql b/models/fct_dbt__consumption_daily.sql index 0474ccd4..9023897b 100644 --- a/models/fct_dbt__consumption_daily.sql +++ b/models/fct_dbt__consumption_daily.sql @@ -1,3 +1,5 @@ +{{ config(enabled = target.type == "snowflake") }} + {#- Core consumption mart: one row per UTC day x meter. diff --git a/models/fct_dbt__consumption_daily_detail.sql b/models/fct_dbt__consumption_daily_detail.sql index 0748604c..033e1592 100644 --- a/models/fct_dbt__consumption_daily_detail.sql +++ b/models/fct_dbt__consumption_daily_detail.sql @@ -1,3 +1,5 @@ +{{ config(enabled = target.type == "snowflake") }} + {#- Detail grain for consumption: UTC day x meter x materialization x target_name. This is the BASE grain; fct_dbt__consumption_daily is a strict roll-up of diff --git a/models/fct_dbt__model_performance.sql b/models/fct_dbt__model_performance.sql index 6b4ed7b1..f54638d2 100644 --- a/models/fct_dbt__model_performance.sql +++ b/models/fct_dbt__model_performance.sql @@ -1,3 +1,5 @@ +{{ config(enabled = target.type == "snowflake") }} + {#- Runtime-regression detection: one row per UTC day x node_id (models). diff --git a/models/fct_dbt__run_health_daily.sql b/models/fct_dbt__run_health_daily.sql index e103fc87..a5642894 100644 --- a/models/fct_dbt__run_health_daily.sql +++ b/models/fct_dbt__run_health_daily.sql @@ -1,3 +1,5 @@ +{{ config(enabled = target.type == "snowflake") }} + {#- Run-health rollup: one row per UTC day across ALL invocations (no billing classification -- dev failures are still failures). Node counts span diff --git a/models/fct_dbt__run_health_daily_detail.sql b/models/fct_dbt__run_health_daily_detail.sql index 92c6900c..a3ca77ee 100644 --- a/models/fct_dbt__run_health_daily_detail.sql +++ b/models/fct_dbt__run_health_daily_detail.sql @@ -1,3 +1,5 @@ +{{ config(enabled = target.type == "snowflake") }} + {#- Run-health rollup at UTC day x target_name grain (companion to fct_dbt__run_health_daily). Same status mapping and definitions as the From 28bb5006fdc416c00e87d8e8a482fbc91d090b92 Mon Sep 17 00:00:00 2001 From: Michael Carlone Date: Sat, 18 Jul 2026 11:24:13 -0400 Subject: [PATCH 14/18] Make gated-model macros parse-safe on non-Snowflake adapters dbt renders disabled models' Jinja during parse, so raise_compiler_error in flatten_json_array/classify default__ broke 'dbt parse' on postgres/etc even though the models are Snowflake-gated. Guard the raises with {% if execute %} (parse has execute==false) and emit a parse-safe placeholder. Non-Snowflake consumers' dbt parse/run is now unaffected; a real build on an unsupported adapter still raises a clear error. Found via O-07's postgres-parse check. --- .../classify_invocation_billing.sql | 5 ++++- .../flatten_json_array.sql | 19 ++++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/macros/consumption/classify_invocation_billing.sql b/macros/consumption/classify_invocation_billing.sql index 2284d315..f10cf28c 100644 --- a/macros/consumption/classify_invocation_billing.sql +++ b/macros/consumption/classify_invocation_billing.sql @@ -52,7 +52,10 @@ {%- if var('dbt_artifacts_count_all_invocations', false) -%} cast('deployment' as {{ dbt.type_string() }}) {%- else -%} - {%- if var('dbt_artifacts_deployment_env_var', none) is not none -%} + {#- Raise only at run time (execute == true), never during parse, so a + non-Snowflake consumer's `dbt parse` is unaffected even with this var + set. The consumption models that call this are Snowflake-gated anyway. -#} + {%- if var('dbt_artifacts_deployment_env_var', none) is not none and execute -%} {{ exceptions.raise_compiler_error( "dbt_artifacts_deployment_env_var (billing classification rule 3) requires adapter-specific JSON support and is only implemented for Snowflake in v1. Unset the var or run on Snowflake." ) }} diff --git a/macros/database_specific_helpers/flatten_json_array.sql b/macros/database_specific_helpers/flatten_json_array.sql index 86e1b34b..42da2608 100644 --- a/macros/database_specific_helpers/flatten_json_array.sql +++ b/macros/database_specific_helpers/flatten_json_array.sql @@ -24,11 +24,20 @@ {% endmacro %} {% macro default__flatten_json_array(array_column, alias) %} - {{ exceptions.raise_compiler_error( - "dbt_artifacts.flatten_json_array() is only implemented for Snowflake in v1 (adapter '" - ~ target.type - ~ "' is unsupported for now). The models using it are Snowflake-gated; cross-adapter support is fast-follow O-12 / C-11." - ) }} + {#- The models using this helper are Snowflake-gated, so on other adapters + this branch is reached only during `dbt parse` (execute == false), never + at run time. Emit a parse-safe placeholder so parsing succeeds on every + adapter, and raise a clear error only if a model is actually built on an + unsupported adapter (execute == true). This keeps the package backwards- + compatible: a non-Snowflake consumer's `dbt parse`/`dbt run` is unaffected. -#} + {%- if execute -%} + {{ exceptions.raise_compiler_error( + "dbt_artifacts.flatten_json_array() is only implemented for Snowflake in v1 (adapter '" + ~ target.type + ~ "' is unsupported for now). The models using it are Snowflake-gated; cross-adapter support is fast-follow O-12 / C-11." + ) }} + {%- endif -%} + (select cast(null as {{ dbt.type_string() }}) as value) as {{ alias }} {% endmacro %} {% macro snowflake__flatten_json_array(array_column, alias) %} From 05f84cc2d884e1a2760744fde25da0a09c106648 Mon Sep 17 00:00:00 2001 From: Michael Carlone Date: Sat, 18 Jul 2026 11:24:13 -0400 Subject: [PATCH 15/18] C-06: integration-test coverage for consumption marts (Snowflake) Sets dbt_artifacts_billing_plan/deployment_targets in the integration project so its invocations class as deployment, and adds Snowflake-gated singular tests: consumption_daily has smb>0, forecast has one current-month smb row with non-null mtd+allowance, by_model reconciles to daily. --- integration_test_project/dbt_project.yml | 6 ++++ ..._consumption_by_model_reconciles_daily.sql | 28 +++++++++++++++++++ .../assert_consumption_daily_has_smb.sql | 8 ++++++ ...ert_consumption_forecast_current_month.sql | 14 ++++++++++ 4 files changed, 56 insertions(+) create mode 100644 integration_test_project/tests/assert_consumption_by_model_reconciles_daily.sql create mode 100644 integration_test_project/tests/assert_consumption_daily_has_smb.sql create mode 100644 integration_test_project/tests/assert_consumption_forecast_current_month.sql diff --git a/integration_test_project/dbt_project.yml b/integration_test_project/dbt_project.yml index 43cf79f1..2b9edb19 100644 --- a/integration_test_project/dbt_project.yml +++ b/integration_test_project/dbt_project.yml @@ -20,6 +20,12 @@ vars: dbt_vars: ["test_dbt_vars_1", "test_dbt_vars_2", "test_dbt_vars_3"] dbt_artifacts_exclude_all_results: true is_development: '{{ env_var("IS_DEVELOPMENT", false) | as_bool }}' + # Consumption/observability v1 (C-06/O-07): classify this project's snowflake + # invocations as deployment so the consumption marts have SMB to aggregate, + # and give the forecast a plan allowance. Only consumed by the Snowflake-gated + # v1 marts; inert on other targets. + dbt_artifacts_billing_plan: developer + dbt_artifacts_deployment_targets: ['snowflake'] models: +persist_docs: diff --git a/integration_test_project/tests/assert_consumption_by_model_reconciles_daily.sql b/integration_test_project/tests/assert_consumption_by_model_reconciles_daily.sql new file mode 100644 index 00000000..dd93d413 --- /dev/null +++ b/integration_test_project/tests/assert_consumption_by_model_reconciles_daily.sql @@ -0,0 +1,28 @@ +{{ config(enabled = target.type == "snowflake") }} +-- C-06: monthly SMB totals must reconcile between consumption_by_model and +-- consumption_daily. Fails (returns rows) on any mismatched month. +with by_model as ( + select + date_trunc('month', billing_month) as month + , sum(smb_quantity) as smb + from {{ ref("fct_dbt__consumption_by_model") }} + group by 1 +), + +daily as ( + select + date_trunc('month', date_day) as month + , sum(quantity) as smb + from {{ ref("fct_dbt__consumption_daily") }} + where meter = 'smb' + group by 1 +) + +select + by_model.month as by_model_month + , daily.month as daily_month + , by_model.smb as by_model_smb + , daily.smb as daily_smb +from by_model +full outer join daily on by_model.month = daily.month +where coalesce(by_model.smb, -1) != coalesce(daily.smb, -1) diff --git a/integration_test_project/tests/assert_consumption_daily_has_smb.sql b/integration_test_project/tests/assert_consumption_daily_has_smb.sql new file mode 100644 index 00000000..dd85afa0 --- /dev/null +++ b/integration_test_project/tests/assert_consumption_daily_has_smb.sql @@ -0,0 +1,8 @@ +{{ config(enabled = target.type == "snowflake") }} +-- C-06: consumption_daily must have at least one 'smb' row with quantity > 0 +-- after the harness's deployment-classed runs. Fails (returns a row) if none. +select count(*) as smb_rows_with_quantity +from {{ ref("fct_dbt__consumption_daily") }} +where meter = 'smb' + and quantity > 0 +having count(*) = 0 diff --git a/integration_test_project/tests/assert_consumption_forecast_current_month.sql b/integration_test_project/tests/assert_consumption_forecast_current_month.sql new file mode 100644 index 00000000..9bbc5c75 --- /dev/null +++ b/integration_test_project/tests/assert_consumption_forecast_current_month.sql @@ -0,0 +1,14 @@ +{{ config(enabled = target.type == "snowflake") }} +-- C-06: the forecast must have exactly one 'smb' row for the current billing +-- month, with non-null month_to_date_quantity and non-null allowance (developer +-- plan is set in dbt_project.yml). Fails if not exactly one, or either is null. +select + count(*) as n_rows + , count(month_to_date_quantity) as non_null_mtd + , count(allowance) as non_null_allowance +from {{ ref("fct_dbt__consumption_forecast") }} +where meter = 'smb' + and billing_month = date_trunc('month', current_date()) +having count(*) != 1 + or count(month_to_date_quantity) != 1 + or count(allowance) != 1 From f9715093b351dde31c6ce6dae8649ec9b091d8ca Mon Sep 17 00:00:00 2001 From: Michael Carlone Date: Sat, 18 Jul 2026 11:24:14 -0400 Subject: [PATCH 16/18] O-07: integration-test coverage for observability marts (Snowflake) Snowflake-gated singular tests: run_health_daily has invocations>0, every model-with-deps has a lineage edge, model_performance is consistent and non-empty, flaky_tests/dag_bottlenecks return valid (possibly empty) results. --- .../tests/assert_dag_bottlenecks_valid.sql | 11 +++++++ .../tests/assert_flaky_tests_valid.sql | 13 ++++++++ .../assert_lineage_edges_cover_models.sql | 30 +++++++++++++++++++ .../tests/assert_model_performance_valid.sql | 15 ++++++++++ ...ssert_run_health_daily_has_invocations.sql | 6 ++++ 5 files changed, 75 insertions(+) create mode 100644 integration_test_project/tests/assert_dag_bottlenecks_valid.sql create mode 100644 integration_test_project/tests/assert_flaky_tests_valid.sql create mode 100644 integration_test_project/tests/assert_lineage_edges_cover_models.sql create mode 100644 integration_test_project/tests/assert_model_performance_valid.sql create mode 100644 integration_test_project/tests/assert_run_health_daily_has_invocations.sql diff --git a/integration_test_project/tests/assert_dag_bottlenecks_valid.sql b/integration_test_project/tests/assert_dag_bottlenecks_valid.sql new file mode 100644 index 00000000..be5d6dd8 --- /dev/null +++ b/integration_test_project/tests/assert_dag_bottlenecks_valid.sql @@ -0,0 +1,11 @@ +{{ config(enabled = target.type == "snowflake") }} +-- O-07: dag_bottlenecks must be valid (possibly empty). Fails on any negative +-- stall time. +select + date_day + , parent_node_id + , total_stall_seconds + , max_stall_seconds +from {{ ref("fct_dbt__dag_bottlenecks") }} +where total_stall_seconds < 0 + or max_stall_seconds < 0 diff --git a/integration_test_project/tests/assert_flaky_tests_valid.sql b/integration_test_project/tests/assert_flaky_tests_valid.sql new file mode 100644 index 00000000..58a57414 --- /dev/null +++ b/integration_test_project/tests/assert_flaky_tests_valid.sql @@ -0,0 +1,13 @@ +{{ config(enabled = target.type == "snowflake") }} +-- O-07: flaky_tests must be valid (possibly empty). Fails on any out-of-range +-- flake_rate or flips exceeding executions. +select + test_node_id + , month + , flips + , executions + , flake_rate +from {{ ref("fct_dbt__flaky_tests") }} +where flake_rate < 0 + or flake_rate > 1 + or flips > executions diff --git a/integration_test_project/tests/assert_lineage_edges_cover_models.sql b/integration_test_project/tests/assert_lineage_edges_cover_models.sql new file mode 100644 index 00000000..a902e347 --- /dev/null +++ b/integration_test_project/tests/assert_lineage_edges_cover_models.sql @@ -0,0 +1,30 @@ +{{ config(enabled = target.type == "snowflake") }} +-- O-07: lineage_edges must be non-empty, and every model in the latest graph +-- state that declares dependencies must appear as a child of >= 1 edge. +-- Fails (returns rows) for any model-with-deps that has no edge. +with latest_models as ( + select + node_id + , depends_on_nodes + , row_number() over ( + partition by node_id order by run_started_at desc + ) as run_rank + from {{ ref("stg_dbt__models") }} +), + +models_with_deps as ( + select node_id + from latest_models + where run_rank = 1 + and array_size(depends_on_nodes) > 0 +), + +edge_children as ( + select distinct child_node_id + from {{ ref("dim_dbt__lineage_edges") }} +) + +select models_with_deps.node_id as model_missing_edge +from models_with_deps +left join edge_children on models_with_deps.node_id = edge_children.child_node_id +where edge_children.child_node_id is null diff --git a/integration_test_project/tests/assert_model_performance_valid.sql b/integration_test_project/tests/assert_model_performance_valid.sql new file mode 100644 index 00000000..83f7a20e --- /dev/null +++ b/integration_test_project/tests/assert_model_performance_valid.sql @@ -0,0 +1,15 @@ +{{ config(enabled = target.type == "snowflake") }} +-- O-07: model_performance must return rows and be internally consistent -- no +-- median_runtime without a success, and the mart must not be empty. (A null +-- median on a day whose only successes were full refreshes is CORRECT by +-- design, so we assert consistency rather than blanket non-null.) +select 'median_without_success' as issue +from {{ ref("fct_dbt__model_performance") }} +where median_runtime is not null + and success_count = 0 + +union all + +select 'empty_mart' as issue +from (select count(*) as row_count from {{ ref("fct_dbt__model_performance") }}) +where row_count = 0 diff --git a/integration_test_project/tests/assert_run_health_daily_has_invocations.sql b/integration_test_project/tests/assert_run_health_daily_has_invocations.sql new file mode 100644 index 00000000..c6950cd9 --- /dev/null +++ b/integration_test_project/tests/assert_run_health_daily_has_invocations.sql @@ -0,0 +1,6 @@ +{{ config(enabled = target.type == "snowflake") }} +-- O-07: run_health_daily must have at least one row with invocations > 0. +select count(*) as days_with_invocations +from {{ ref("fct_dbt__run_health_daily") }} +where invocations > 0 +having count(*) = 0 From d7f9ad3ef344dac7481249d25e3fe01d6808ff71 Mon Sep 17 00:00:00 2001 From: Michael Carlone Date: Sat, 18 Jul 2026 15:54:29 -0400 Subject: [PATCH 17/18] C-04: forecast_exceeded_date reports the real crossing day, not month-end Previously the 'already exceeded' branch returned as_of_date, which collapses to month_end for fully-elapsed months (e.g. June showed 06-30). Add a historical_breach CTE: the first elapsed day whose ACTUAL running total reaches the allowance, and use coalesce(historical, projected_future). Now a past or already-over month reports the true day it crossed. Verified: integration April 2025 -> 2025-04-09 (cumulative crosses 50 there), not 04-30. --- models/fct_dbt__consumption_forecast.sql | 49 ++++++++++++++++++++++-- models/fct_dbt__consumption_forecast.yml | 8 ++-- 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/models/fct_dbt__consumption_forecast.sql b/models/fct_dbt__consumption_forecast.sql index 083d4de0..23a480fc 100644 --- a/models/fct_dbt__consumption_forecast.sql +++ b/models/fct_dbt__consumption_forecast.sql @@ -147,7 +147,46 @@ with ), {%- if allowance is not none %} + elapsed_cumulative as ( + + {# Running total of ACTUAL daily quantity over elapsed days (<= as_of), + per meter x month, so we can find the real day the allowance was + first crossed rather than a month-end placeholder. #} + select + daily.meter + , date_trunc('month', daily.date_day) as billing_month + , daily.date_day + , sum(daily.quantity) over ( + partition by daily.meter, date_trunc('month', daily.date_day) + order by daily.date_day + rows between unbounded preceding and current row + ) as cumulative_actual + from daily + inner join month_anchors as ma + on ma.meter = daily.meter + and ma.billing_month = date_trunc('month', daily.date_day) + where daily.date_day <= ma.as_of_date + ), + + historical_breach as ( + + {# First elapsed day whose actual running total reaches the allowance -- + the true historical crossing date (for fully-elapsed months and for + current months already over). #} + select + meter + , billing_month + , min(date_day) as historical_exceeded_date + from elapsed_cumulative + where meter = 'smb' + and cumulative_actual >= {{ allowance }} + group by meter, billing_month + ), + breach as ( + + {# First FUTURE day the projected cumulative (MTD + expected) crosses the + allowance, for months not already over. #} select re.meter , re.billing_month @@ -179,10 +218,11 @@ with as forecast_month_end_quantity {%- if allowance is not none %} , case - when ma.meter = 'smb' and mtd.month_to_date_quantity >= {{ allowance }} - then ma.as_of_date when ma.meter = 'smb' - then breach.forecast_exceeded_date + then coalesce( + historical_breach.historical_exceeded_date, + breach.forecast_exceeded_date + ) end as forecast_exceeded_date , case when ma.meter = 'smb' @@ -210,6 +250,9 @@ with left join remaining_summary as rs on rs.meter = ma.meter and rs.billing_month = ma.billing_month {%- if allowance is not none %} + left join historical_breach + on historical_breach.meter = ma.meter + and historical_breach.billing_month = ma.billing_month left join breach on breach.meter = ma.meter and breach.billing_month = ma.billing_month {%- endif %} diff --git a/models/fct_dbt__consumption_forecast.yml b/models/fct_dbt__consumption_forecast.yml index 035ee448..b2bb1472 100644 --- a/models/fct_dbt__consumption_forecast.yml +++ b/models/fct_dbt__consumption_forecast.yml @@ -38,9 +38,11 @@ models: description: month_to_date_quantity + sum of expected quantity over the remaining calendar days. - name: forecast_exceeded_date description: > - First remaining day whose cumulative expected (added to MTD) reaches the - allowance; as_of when MTD already exceeds it; null when no allowance or the - meter is not smb or the projection never crosses. + The day the allowance is (or is projected to be) crossed: the actual day + the running total of elapsed daily quantity first reached the allowance if + it already has (historical), otherwise the first future day the projected + cumulative (MTD + weekday-aware expected) reaches it; null when there is no + allowance, the meter is not smb, or it never crosses. - name: pct_of_allowance_used description: month_to_date_quantity / allowance for the smb meter; null otherwise. tests: From 469bd697858e2baac70c2dbb65f912dafed37769 Mon Sep 17 00:00:00 2001 From: Michael Carlone Date: Fri, 31 Jul 2026 15:16:22 -0400 Subject: [PATCH 18/18] Remove internal planning references from code comments Model, macro and test comments cited internal ticket IDs and design-doc paths that don't exist in this repository, so they read as dangling references. Replace them with the information they were standing in for: dependency names instead of ticket IDs, "planned" instead of named follow-up tickets, and plain descriptions instead of doc paths. Also reword the flatten_json_array() unsupported-adapter error, which surfaced a ticket ID to end users. Comment-only apart from that error string; no behaviour change. --- integration_test_project/dbt_project.yml | 2 +- .../tests/assert_consumption_by_model_reconciles_daily.sql | 2 +- .../tests/assert_consumption_daily_has_smb.sql | 2 +- .../tests/assert_consumption_forecast_current_month.sql | 2 +- .../tests/assert_dag_bottlenecks_valid.sql | 2 +- integration_test_project/tests/assert_flaky_tests_valid.sql | 2 +- .../tests/assert_lineage_edges_cover_models.sql | 2 +- .../tests/assert_model_performance_valid.sql | 2 +- .../tests/assert_run_health_daily_has_invocations.sql | 2 +- macros/consumption/classify_invocation_billing.sql | 4 ++-- macros/consumption/get_smb_allowance.sql | 2 +- macros/database_specific_helpers/flatten_json_array.sql | 5 ++--- macros/database_specific_helpers/statistical_helpers.sql | 2 +- models/dim_dbt__lineage_edges.yml | 2 +- models/fct_dbt__consumption_by_model.yml | 2 +- models/fct_dbt__consumption_daily_detail.sql | 4 ++-- models/fct_dbt__consumption_forecast.yml | 2 +- models/fct_dbt__dag_bottlenecks.sql | 2 +- models/fct_dbt__dag_bottlenecks_detail.sql | 2 +- models/fct_dbt__flaky_tests_detail.sql | 2 +- models/fct_dbt__model_performance.sql | 3 ++- 21 files changed, 25 insertions(+), 25 deletions(-) diff --git a/integration_test_project/dbt_project.yml b/integration_test_project/dbt_project.yml index 2b9edb19..c78f188a 100644 --- a/integration_test_project/dbt_project.yml +++ b/integration_test_project/dbt_project.yml @@ -20,7 +20,7 @@ vars: dbt_vars: ["test_dbt_vars_1", "test_dbt_vars_2", "test_dbt_vars_3"] dbt_artifacts_exclude_all_results: true is_development: '{{ env_var("IS_DEVELOPMENT", false) | as_bool }}' - # Consumption/observability v1 (C-06/O-07): classify this project's snowflake + # Consumption/observability marts: classify this project's snowflake # invocations as deployment so the consumption marts have SMB to aggregate, # and give the forecast a plan allowance. Only consumed by the Snowflake-gated # v1 marts; inert on other targets. diff --git a/integration_test_project/tests/assert_consumption_by_model_reconciles_daily.sql b/integration_test_project/tests/assert_consumption_by_model_reconciles_daily.sql index dd93d413..971c9786 100644 --- a/integration_test_project/tests/assert_consumption_by_model_reconciles_daily.sql +++ b/integration_test_project/tests/assert_consumption_by_model_reconciles_daily.sql @@ -1,5 +1,5 @@ {{ config(enabled = target.type == "snowflake") }} --- C-06: monthly SMB totals must reconcile between consumption_by_model and +-- Monthly SMB totals must reconcile between consumption_by_model and -- consumption_daily. Fails (returns rows) on any mismatched month. with by_model as ( select diff --git a/integration_test_project/tests/assert_consumption_daily_has_smb.sql b/integration_test_project/tests/assert_consumption_daily_has_smb.sql index dd85afa0..9936996c 100644 --- a/integration_test_project/tests/assert_consumption_daily_has_smb.sql +++ b/integration_test_project/tests/assert_consumption_daily_has_smb.sql @@ -1,5 +1,5 @@ {{ config(enabled = target.type == "snowflake") }} --- C-06: consumption_daily must have at least one 'smb' row with quantity > 0 +-- consumption_daily must have at least one 'smb' row with quantity > 0 -- after the harness's deployment-classed runs. Fails (returns a row) if none. select count(*) as smb_rows_with_quantity from {{ ref("fct_dbt__consumption_daily") }} diff --git a/integration_test_project/tests/assert_consumption_forecast_current_month.sql b/integration_test_project/tests/assert_consumption_forecast_current_month.sql index 9bbc5c75..54719d78 100644 --- a/integration_test_project/tests/assert_consumption_forecast_current_month.sql +++ b/integration_test_project/tests/assert_consumption_forecast_current_month.sql @@ -1,5 +1,5 @@ {{ config(enabled = target.type == "snowflake") }} --- C-06: the forecast must have exactly one 'smb' row for the current billing +-- The forecast must have exactly one 'smb' row for the current billing -- month, with non-null month_to_date_quantity and non-null allowance (developer -- plan is set in dbt_project.yml). Fails if not exactly one, or either is null. select diff --git a/integration_test_project/tests/assert_dag_bottlenecks_valid.sql b/integration_test_project/tests/assert_dag_bottlenecks_valid.sql index be5d6dd8..5b3aff6e 100644 --- a/integration_test_project/tests/assert_dag_bottlenecks_valid.sql +++ b/integration_test_project/tests/assert_dag_bottlenecks_valid.sql @@ -1,5 +1,5 @@ {{ config(enabled = target.type == "snowflake") }} --- O-07: dag_bottlenecks must be valid (possibly empty). Fails on any negative +-- dag_bottlenecks must be valid (possibly empty). Fails on any negative -- stall time. select date_day diff --git a/integration_test_project/tests/assert_flaky_tests_valid.sql b/integration_test_project/tests/assert_flaky_tests_valid.sql index 58a57414..b5a15ff7 100644 --- a/integration_test_project/tests/assert_flaky_tests_valid.sql +++ b/integration_test_project/tests/assert_flaky_tests_valid.sql @@ -1,5 +1,5 @@ {{ config(enabled = target.type == "snowflake") }} --- O-07: flaky_tests must be valid (possibly empty). Fails on any out-of-range +-- flaky_tests must be valid (possibly empty). Fails on any out-of-range -- flake_rate or flips exceeding executions. select test_node_id diff --git a/integration_test_project/tests/assert_lineage_edges_cover_models.sql b/integration_test_project/tests/assert_lineage_edges_cover_models.sql index a902e347..f0dd335d 100644 --- a/integration_test_project/tests/assert_lineage_edges_cover_models.sql +++ b/integration_test_project/tests/assert_lineage_edges_cover_models.sql @@ -1,5 +1,5 @@ {{ config(enabled = target.type == "snowflake") }} --- O-07: lineage_edges must be non-empty, and every model in the latest graph +-- lineage_edges must be non-empty, and every model in the latest graph -- state that declares dependencies must appear as a child of >= 1 edge. -- Fails (returns rows) for any model-with-deps that has no edge. with latest_models as ( diff --git a/integration_test_project/tests/assert_model_performance_valid.sql b/integration_test_project/tests/assert_model_performance_valid.sql index 83f7a20e..df6e58c5 100644 --- a/integration_test_project/tests/assert_model_performance_valid.sql +++ b/integration_test_project/tests/assert_model_performance_valid.sql @@ -1,5 +1,5 @@ {{ config(enabled = target.type == "snowflake") }} --- O-07: model_performance must return rows and be internally consistent -- no +-- model_performance must return rows and be internally consistent -- no -- median_runtime without a success, and the mart must not be empty. (A null -- median on a day whose only successes were full refreshes is CORRECT by -- design, so we assert consistency rather than blanket non-null.) diff --git a/integration_test_project/tests/assert_run_health_daily_has_invocations.sql b/integration_test_project/tests/assert_run_health_daily_has_invocations.sql index c6950cd9..d9de8402 100644 --- a/integration_test_project/tests/assert_run_health_daily_has_invocations.sql +++ b/integration_test_project/tests/assert_run_health_daily_has_invocations.sql @@ -1,5 +1,5 @@ {{ config(enabled = target.type == "snowflake") }} --- O-07: run_health_daily must have at least one row with invocations > 0. +-- run_health_daily must have at least one row with invocations > 0. select count(*) as days_with_invocations from {{ ref("fct_dbt__run_health_daily") }} where invocations > 0 diff --git a/macros/consumption/classify_invocation_billing.sql b/macros/consumption/classify_invocation_billing.sql index f10cf28c..076a8609 100644 --- a/macros/consumption/classify_invocation_billing.sql +++ b/macros/consumption/classify_invocation_billing.sql @@ -10,7 +10,7 @@ (they are on stg_dbt__invocations). This macro does NOT modify any existing staging/source contract. - Precedence (first match wins), per specs/consumption/design.md: + Precedence (first match wins): 1. dbt_cloud_job_id is not null -> deployment 2. lower(target_name) in deployment_targets (lowered) -> deployment 3. optional: env var named by dbt_artifacts_deployment_env_var is @@ -24,7 +24,7 @@ (env_vars is stored via type_json() = OBJECT on Snowflake, accessed with the variant path env_vars:"NAME"). It is emitted only when the var is set, so the default path parses no JSON. default__ raises a clear compile error - if rule 3 is requested on an unsupported adapter (fast-follow C-11). + if rule 3 is requested on an adapter that does not implement it yet. -#} {% macro classify_invocation_billing() %} diff --git a/macros/consumption/get_smb_allowance.sql b/macros/consumption/get_smb_allowance.sql index fc4ca331..f890e802 100644 --- a/macros/consumption/get_smb_allowance.sql +++ b/macros/consumption/get_smb_allowance.sql @@ -2,7 +2,7 @@ get_smb_allowance() Resolves the monthly SMB allowance at compile time, returning a Python - number or none. Resolution order (per specs/consumption/design.md): + number or none. Resolution order: 1. var('dbt_artifacts_smb_allowance') -- explicit, wins 2. plan default from var('dbt_artifacts_billing_plan') (developer: 3000, starter: 15000, enterprise: 100000) diff --git a/macros/database_specific_helpers/flatten_json_array.sql b/macros/database_specific_helpers/flatten_json_array.sql index 42da2608..7df395f1 100644 --- a/macros/database_specific_helpers/flatten_json_array.sql +++ b/macros/database_specific_helpers/flatten_json_array.sql @@ -15,8 +15,7 @@ Snowflake only in v1. Other adapters raise a clear compile error; the model that uses this helper is itself gated to Snowflake. Cross-adapter overrides (BigQuery unnest, Postgres jsonb_array_elements_text, Trino unnest, Spark - explode, SQL Server openjson) are fast-follow (O-12 / C-11) and coordinated - with specs/materialize-docs/design.md. + explode, SQL Server openjson) are planned. -#} {% macro flatten_json_array(array_column, alias) %} @@ -34,7 +33,7 @@ {{ exceptions.raise_compiler_error( "dbt_artifacts.flatten_json_array() is only implemented for Snowflake in v1 (adapter '" ~ target.type - ~ "' is unsupported for now). The models using it are Snowflake-gated; cross-adapter support is fast-follow O-12 / C-11." + ~ "' is not supported yet). The models that use it are Snowflake-gated, so this should not be reachable; cross-adapter support is planned." ) }} {%- endif -%} (select cast(null as {{ dbt.type_string() }}) as value) as {{ alias }} diff --git a/macros/database_specific_helpers/statistical_helpers.sql b/macros/database_specific_helpers/statistical_helpers.sql index 728edf9c..a6cfd251 100644 --- a/macros/database_specific_helpers/statistical_helpers.sql +++ b/macros/database_specific_helpers/statistical_helpers.sql @@ -6,7 +6,7 @@ ANSI ordered-set aggregate percentile_cont(fraction) within group (order by ...), which Snowflake supports natively as an aggregate, so no snowflake__ override is required. Non-Snowflake overrides (SQL Server - window-only syntax, Spark approx_percentile) are fast-follow O-12. + window-only syntax, Spark approx_percentile) are planned. -#} {#- MEDIAN -#} diff --git a/models/dim_dbt__lineage_edges.yml b/models/dim_dbt__lineage_edges.yml index d69fe563..e4592447 100644 --- a/models/dim_dbt__lineage_edges.yml +++ b/models/dim_dbt__lineage_edges.yml @@ -8,7 +8,7 @@ models: already uploads for models, snapshots and tests. Unblocks stall-time and flaky-test parentage. **Snowflake only in v1** (enabled = target.type == 'snowflake'); the JSON-array explode uses the dispatched flatten_json_array() - helper, whose non-Snowflake overrides are fast-follow (O-12 / C-11). Latest + helper, whose non-Snowflake overrides are planned. Latest graph state = each node's most-recent appearance (row_number by run_started_at). columns: - name: lineage_edge_id diff --git a/models/fct_dbt__consumption_by_model.yml b/models/fct_dbt__consumption_by_model.yml index 1e238ef0..6bfb192e 100644 --- a/models/fct_dbt__consumption_by_model.yml +++ b/models/fct_dbt__consumption_by_model.yml @@ -43,4 +43,4 @@ models: description: > dbt State ROI estimate = distinct_days_built x dbt_artifacts_datt_price (default 0.094). A directional upper-bound of what dbt State could meter - for this model; see specs/consumption/design.md. + for this model. diff --git a/models/fct_dbt__consumption_daily_detail.sql b/models/fct_dbt__consumption_daily_detail.sql index 033e1592..37bcc7cc 100644 --- a/models/fct_dbt__consumption_daily_detail.sql +++ b/models/fct_dbt__consumption_daily_detail.sql @@ -12,8 +12,8 @@ - 'active_target_tables' : distinct node_ids (models u seeds u snapshots u tests) with >=1 successful deployment execution that day, counted per deployment target (materialization is null). On single- - deployment-target setups (the common case) this equals the - per-day distinct-node count in specs/consumption/design.md; when + deployment-target setups (the common case) this equals a plain + per-day distinct-node count; when a node runs under several deployment targets in one day it is counted once per target, which keeps the roll-up additive. -#} diff --git a/models/fct_dbt__consumption_forecast.yml b/models/fct_dbt__consumption_forecast.yml index b2bb1472..0b0f030d 100644 --- a/models/fct_dbt__consumption_forecast.yml +++ b/models/fct_dbt__consumption_forecast.yml @@ -11,7 +11,7 @@ models: values. Allowance resolves via get_smb_allowance() and applies to the 'smb' meter only; with no plan/allowance var the breach columns are null but projections still populate. Snowflake-only in v1 (uses Snowflake date - functions); multi-adapter is fast-follow C-11. + functions); multi-adapter support is planned. columns: - name: consumption_forecast_id description: Surrogate key of the grain (hash of meter, billing_month). BI join key. diff --git a/models/fct_dbt__dag_bottlenecks.sql b/models/fct_dbt__dag_bottlenecks.sql index 3067bc8f..ba8e20c2 100644 --- a/models/fct_dbt__dag_bottlenecks.sql +++ b/models/fct_dbt__dag_bottlenecks.sql @@ -5,7 +5,7 @@ waiting on it (measured stall). The "pinch point" mart. Roll-up of fct_dbt__dag_bottlenecks_detail over rows where the parent actually gated a child (stall_seconds > 0). Descendant counts / blocking scores are out of - scope for v1 (parked). Snowflake-only (inherits O-02 enablement). + scope for v1. Snowflake-only (inherits dim_dbt__lineage_edges' enablement). -#} with diff --git a/models/fct_dbt__dag_bottlenecks_detail.sql b/models/fct_dbt__dag_bottlenecks_detail.sql index 2a4811d6..fa698be9 100644 --- a/models/fct_dbt__dag_bottlenecks_detail.sql +++ b/models/fct_dbt__dag_bottlenecks_detail.sql @@ -10,7 +10,7 @@ This is *measured* wall-clock gating, not graph theory. Model parents only; seed/source parents are excluded (documented) -- seeds rarely gate and keep - the join simple. Depends on dim_dbt__lineage_edges (O-02); Snowflake-only. + the join simple. Depends on dim_dbt__lineage_edges; Snowflake-only. -#} with diff --git a/models/fct_dbt__flaky_tests_detail.sql b/models/fct_dbt__flaky_tests_detail.sql index 2fc7dab8..b532b874 100644 --- a/models/fct_dbt__flaky_tests_detail.sql +++ b/models/fct_dbt__flaky_tests_detail.sql @@ -11,7 +11,7 @@ successful build with run_started_at strictly between the fail and the pass (that's a legitimate fix, excluded). If the test has no resolvable model parents, we still count the flip but set parent_rebuilt_between = null -- - we don't hide the uncertainty. Depends on dim_dbt__lineage_edges (O-02); + we don't hide the uncertainty. Depends on dim_dbt__lineage_edges; Snowflake-only. -#} diff --git a/models/fct_dbt__model_performance.sql b/models/fct_dbt__model_performance.sql index f54638d2..ec0e0f33 100644 --- a/models/fct_dbt__model_performance.sql +++ b/models/fct_dbt__model_performance.sql @@ -5,7 +5,8 @@ Runtime stats (median/p95/rows_affected) are computed over SUCCESSFUL, NON-full-refresh executions only -- a full refresh is not a regression, and - failures are O-03's job. full_refresh_executions is exposed as a count so + failures belong to fct_dbt__run_health_daily. full_refresh_executions is + exposed as a count so the signal isn't lost. baseline_runtime = median of the SAME node's SAME-day-of-week median_runtime