From efcda3ebc887ef09ad25f4ab94f23f2a0660c83b Mon Sep 17 00:00:00 2001 From: ninazacharia-toast Date: Tue, 4 Aug 2026 10:22:24 -0400 Subject: [PATCH] add transactions for inserts --- macros/upload_results/begin_transaction.sql | 26 +++++++++++++++++++++ macros/upload_results/end_transaction.sql | 26 +++++++++++++++++++++ macros/upload_results/upload_results.sql | 5 ++++ 3 files changed, 57 insertions(+) create mode 100644 macros/upload_results/begin_transaction.sql create mode 100644 macros/upload_results/end_transaction.sql diff --git a/macros/upload_results/begin_transaction.sql b/macros/upload_results/begin_transaction.sql new file mode 100644 index 00000000..abb7fdef --- /dev/null +++ b/macros/upload_results/begin_transaction.sql @@ -0,0 +1,26 @@ +{% macro begin_transaction() -%} + + {{ return(adapter.dispatch('begin_transaction', 'dbt_artifacts')()) }} + +{%- endmacro %} + +{% macro default__begin_transaction() -%} +{%- endmacro %} + +{% macro snowflake__begin_transaction() -%} + + {% do run_query('begin transaction') %} + +{%- endmacro %} + +{% macro postgres__begin_transaction() -%} + + {% do run_query('begin') %} + +{%- endmacro %} + +{% macro sqlserver__begin_transaction() -%} + + {% do run_query('begin transaction') %} + +{%- endmacro %} diff --git a/macros/upload_results/end_transaction.sql b/macros/upload_results/end_transaction.sql new file mode 100644 index 00000000..ea4fe443 --- /dev/null +++ b/macros/upload_results/end_transaction.sql @@ -0,0 +1,26 @@ +{% macro end_transaction() -%} + + {{ return(adapter.dispatch('end_transaction', 'dbt_artifacts')()) }} + +{%- endmacro %} + +{% macro default__end_transaction() -%} +{%- endmacro %} + +{% macro snowflake__end_transaction() -%} + + {% do run_query('commit') %} + +{%- endmacro %} + +{% macro postgres__end_transaction() -%} + + {% do run_query('commit') %} + +{%- endmacro %} + +{% macro sqlserver__end_transaction() -%} + + {% do run_query('commit transaction') %} + +{%- endmacro %} diff --git a/macros/upload_results/upload_results.sql b/macros/upload_results/upload_results.sql index 114a667d..302ab879 100644 --- a/macros/upload_results/upload_results.sql +++ b/macros/upload_results/upload_results.sql @@ -25,6 +25,9 @@ {% set upload_limit = 300 if target.type == 'bigquery' else 5000 %} {% endif %} + {# Wrap the chunked inserts in a transaction, where the dialect supports it #} + {{ dbt_artifacts.begin_transaction() }} + {# Loop through each chunk in turn #} {% for i in range(0, objects | length, upload_limit) -%} @@ -42,6 +45,8 @@ {# Loop the next 'chunk' #} {% endfor %} + {{ dbt_artifacts.end_transaction() }} + {# Loop the next 'dataset' #} {% endfor %}