Feat: support dynamic blueprinting for Python models, add docs (#4177)

Author: georgesittas

docs/concepts/models/python_models.md

@@ -367,6 +367,10 @@ def entrypoint(
     )
 ```
 
+!!! note
+
+    Blueprint variable mappings can also be evaluated dynamically, by using a macro (i.e. `blueprints="@gen_blueprints()"`). This is useful in cases where the `blueprints` list needs to be sourced from external sources, e.g. CSV files.
+
 ## Examples
 ### Basic
 The following is an example of a Python model returning a static Pandas DataFrame.

docs/concepts/models/sql_models.md

@@ -175,6 +175,10 @@ SELECT
 FROM customer2.some_source
 ```
 
+!!! note
+
+    Blueprint variable mappings can also be evaluated dynamically, by using a macro (i.e. `blueprints @gen_blueprints()`). This is useful in cases where the `blueprints` list needs to be sourced from external sources, e.g. CSV files.
+
 ## Python-based definition
 
 The Python-based definition of SQL models consists of a single python function, decorated with SQLMesh's `@model` [decorator](https://wiki.python.org/moin/PythonDecorators). The decorator is required to have the `is_sql` keyword argument set to `True` to distinguish it from [Python models](./python_models.md) that return DataFrame instances.
@@ -258,6 +262,10 @@ def entrypoint(evaluator: MacroEvaluator) -> str | exp.Expression:
 
 The two models produced from this template are the same as in the [example](#SQL-model-blueprinting) for SQL-based blueprinting.
 
+!!! note
+
+    Blueprint variable mappings can also be evaluated dynamically, by using a macro (i.e. `blueprints="@gen_blueprints()"`). This is useful in cases where the `blueprints` list needs to be sourced from external sources, e.g. CSV files.
+
 ## Automatic dependencies
 
 SQLMesh parses your SQL, so it understands what the code does and how it relates to other models. There is no need for you to manually specify dependencies to other models with special tags or commands.

sqlmesh/core/model/decorator.py

@@ -24,7 +24,7 @@
 )
 from sqlmesh.core.model.kind import ModelKindName, _ModelKind
 from sqlmesh.utils import registry_decorator, DECORATOR_RETURN_TYPE
-from sqlmesh.utils.errors import ConfigError
+from sqlmesh.utils.errors import ConfigError, raise_config_error
 from sqlmesh.utils.metaprogramming import build_env, serialize_env
 
 
@@ -96,9 +96,32 @@ def models(
         default_catalog_per_gateway: t.Optional[t.Dict[str, str]] = None,
         **loader_kwargs: t.Any,
     ) -> t.List[Model]:
+        blueprints = self.kwargs.pop("blueprints", None)
+
+        if isinstance(blueprints, str):
+            blueprints = parse_one(blueprints, dialect=dialect)
+
+        if isinstance(blueprints, MacroFunc):
+            from sqlmesh.core.model.definition import render_expression
+
+            blueprints = render_expression(
+                expression=blueprints,
+                module_path=module_path,
+                macros=loader_kwargs.get("macros"),
+                jinja_macros=loader_kwargs.get("jinja_macros"),
+                variables=get_variables(None),
+                path=path,
+                dialect=dialect,
+                default_catalog=loader_kwargs.get("default_catalog"),
+            )
+            if not blueprints:
+                raise_config_error("Failed to render blueprints property", path)
+
+            blueprints = blueprints[0]
+
         return create_models_from_blueprints(
             gateway=self.kwargs.get("gateway"),
-            blueprints=self.kwargs.pop("blueprints", None),
+            blueprints=blueprints,
             get_variables=get_variables,
             loader=self.model,
             path=path,

sqlmesh/core/model/definition.py

@@ -1849,7 +1849,7 @@ def _extract_blueprints(blueprints: t.Any, path: Path) -> t.List[t.Any]:
         return blueprints
 
     raise_config_error(
-        "Expected a list or tuple consisting of key-value mappings for"
+        "Expected a list or tuple consisting of key-value mappings for "
         f"the 'blueprints' property, got '{blueprints}' instead",
         path,
     )

tests/core/test_model.py

@@ -8312,9 +8312,9 @@ def entrypoint(evaluator):
 def test_dynamic_blueprinting(tmp_path: Path) -> None:
     init_example_project(tmp_path, dialect="duckdb", template=ProjectTemplate.EMPTY)
 
-    dynamic_template = tmp_path / "models/dynamic_template.sql"
-    dynamic_template.parent.mkdir(parents=True, exist_ok=True)
-    dynamic_template.write_text(
+    dynamic_template_sql = tmp_path / "models/dynamic_template.sql"
+    dynamic_template_sql.parent.mkdir(parents=True, exist_ok=True)
+    dynamic_template_sql.write_text(
         """
         MODEL (
           name @customer.some_table,
@@ -8330,6 +8330,24 @@ def test_dynamic_blueprinting(tmp_path: Path) -> None:
         """
     )
 
+    dynamic_template_py = tmp_path / "models/dynamic_template.py"
+    dynamic_template_py.parent.mkdir(parents=True, exist_ok=True)
+    dynamic_template_py.write_text(
+        """
+from sqlmesh import model
+
+@model(
+    "@{customer}.some_other_table",
+    kind="FULL",
+    blueprints="@gen_blueprints()",
+    is_sql=True,
+)
+def entrypoint(evaluator):
+    field_a = evaluator.blueprint_var("field_a")
+    return f"SELECT {field_a}, @BLUEPRINT_VAR('field_b') AS field_b FROM @customer.some_source"
+"""
+    )
+
     gen_blueprints = tmp_path / "macros/gen_blueprints.py"
     gen_blueprints.parent.mkdir(parents=True, exist_ok=True)
     gen_blueprints.write_text(
@@ -8347,9 +8365,11 @@ def gen_blueprints(evaluator):
         config=Config(model_defaults=ModelDefaultsConfig(dialect="duckdb")), paths=tmp_path
     )
 
-    assert len(ctx.models) == 2
+    assert len(ctx.models) == 4
     assert '"memory"."customer1"."some_table"' in ctx.models
     assert '"memory"."customer2"."some_table"' in ctx.models
+    assert '"memory"."customer1"."some_other_table"' in ctx.models
+    assert '"memory"."customer2"."some_other_table"' in ctx.models
 
 
 def test_single_blueprint(tmp_path: Path) -> None: