11"""
2- This migration script has two purposes:
2+ This script's goal is to warn users if there is both a metadata and non-metadata reference in
3+ the python environment of a model. Additionally, it warns them if there's a macro referenced
4+ in a used audit's query, in the argument list of the audits and signals properties, or in an
5+ on_virtual_update statement.
36
4- 1) Mark all python env macros referenced in audits, signals or on_virtual_update statements
5- as metadata, unless they're referenced elsewhere in the model and they're not metadata-only.
7+ Context:
68
7- 2) Warn if there is both metadata and non-metadata reference in the python environment of a model.
9+ The metadata status for macros and signals is now transitive, i.e. every dependency of a
10+ metadata macro or signal is also metadata, unless it is referenced by a non-metadata object.
811
9- The metadata status for macros and signals is now transitive, i.e. every dependency of a
10- metadata macro or signal is also metadata, unless it is referenced by a non-metadata object .
12+ This means that global references of metadata objects may now be excluded from the data hash
13+ calculation because of their new metadata status, which would lead to a diff .
1114
12- This means that global references of metadata objects may now be excluded from the
13- data hash calculation because of their new metadata status, which would lead to a
14- diff. This script detects the possibility for such a diff and warns users ahead of time.
15+ Additionally, we now implicitly treat macro refs in the aforementioned statements as "metadata-only",
16+ even though they may not be marked as such by a user. This may also lead to a diff.
1517"""
1618
1719import json
@@ -29,11 +31,13 @@ def migrate(state_sync, **kwargs): # type: ignore
2931 if schema :
3032 snapshots_table = f"{ schema } .{ snapshots_table } "
3133
32- common_msg = (
33- "Since the metadata status is now propagated transitively, this means that the next plan "
34- "command may detect unexpected changes and prompt about backfilling this model, or others, "
35- "for the same reason. If this is a concern, consider running a forward-only plan instead: "
36- "https://sqlmesh.readthedocs.io/en/stable/concepts/plans/#forward-only-plans.\n "
34+ warning = (
35+ "SQLMesh detected that it may not be able to fully migrate the state database. This should not impact "
36+ "the migration process, but may result in unexpected changes being reported by the next `sqlmesh plan` "
37+ "command. Please run the `sqlmesh diff` (https://sqlmesh.readthedocs.io/en/stable/reference/cli/?h=cli#diff) "
38+ "command after the migration has completed, before making any new changes. If any unexpected changes are reported, "
39+ "consider running a forward-only plan (https://sqlmesh.readthedocs.io/en/stable/concepts/plans/#forward-only-plans) "
40+ "to avoid unnecessary backfills.\n "
3741 )
3842
3943 for (snapshot ,) in engine_adapter .fetchall (
@@ -59,10 +63,7 @@ def migrate(state_sync, **kwargs): # type: ignore
5963 has_non_metadata = True
6064
6165 if has_metadata and has_non_metadata :
62- get_console ().log_warning (
63- f"Model '{ name } ' references both metadata and non-metadata functions (macros or signals). "
64- + common_msg
65- )
66+ get_console ().log_warning (warning )
6667 return
6768
6869 dialect = node .get ("dialect" )
@@ -89,10 +90,7 @@ def migrate(state_sync, **kwargs): # type: ignore
8990 for macro_name in extract_used_macros (metadata_hash_statements ):
9091 serialized_macro = python_env .get (macro_name )
9192 if isinstance (serialized_macro , dict ) and not serialized_macro .get ("is_metadata" ):
92- get_console ().log_warning (
93- f"Model '{ name } ' references macro '{ macro_name } ' which is now implicitly treated as metadata-only. "
94- + common_msg
95- )
93+ get_console ().log_warning (warning )
9694 return
9795
9896
0 commit comments