PR feedback

Author: georgesittas

docs/reference/configuration.md

@@ -36,7 +36,7 @@ Configuration options for SQLMesh environment creation and promotion.
 | `physical_schema_mapping`     | A mapping from regular expressions to names of schemas in which physical tables for the corresponding models [will be placed](../guides/configuration.md#physical-table-schemas). (Default physical schema name: `sqlmesh__[model schema]`)                                                        | dict[string, string] | N        |
 | `environment_suffix_target`   | Whether SQLMesh views should append their environment name to the `schema` or `table` - [additional details](../guides/configuration.md#view-schema-override). (Default: `schema`)                                                                                                                 | string               | N        |
 | `gateway_managed_virtual_layer`   | Whether SQLMesh views of the virtual layer will be created by the default gateway or model specified gateways - [additional details](../guides/multi_engine.md#gateway-managed-virtual-layer). (Default: False)                                                                                                                 | boolean               | N        |
-| `infer_python_package_requirements`   | Whether SQLMesh will statically analyze Python code to automatically infer Python package requirements. (Default: True)                                                                                                                 | boolean               | N        |
+| `infer_python_dependencies`   | Whether SQLMesh will statically analyze Python code to automatically infer Python package requirements. (Default: True)                                                                                                                 | boolean               | N        |
 | `environment_catalog_mapping` | A mapping from regular expressions to catalog names. The catalog name is used to determine the target catalog for a given environment.                                                                                                                                                             | dict[string, string] | N        |
 | `log_limit`                   | The default number of logs to keep (Default: `20`)                                                                                                                                                                                                                                                 | int                  | N        |
 

sqlmesh/core/config/root.py

@@ -107,7 +107,7 @@ class Config(BaseConfig):
         physical_schema_mapping: A mapping from regular expressions to names of schemas in which physical tables for corresponding models will be placed.
         environment_suffix_target: Indicates whether to append the environment name to the schema or table name.
         gateway_managed_virtual_layer: Whether the models' views in the virtual layer are created by the model-specific gateway rather than the default gateway.
-        infer_python_package_requirements: Whether to statically analyze Python code to automatically infer Python package requirements.
+        infer_python_dependencies: Whether to statically analyze Python code to automatically infer Python package requirements.
         environment_catalog_mapping: A mapping from regular expressions to catalog names. The catalog name is used to determine the target catalog for a given environment.
         default_target_environment: The name of the environment that will be the default target for the `sqlmesh plan` and `sqlmesh run` commands.
         log_limit: The default number of logs to keep.
@@ -147,7 +147,7 @@ class Config(BaseConfig):
         default=EnvironmentSuffixTarget.default
     )
     gateway_managed_virtual_layer: bool = False
-    infer_python_package_requirements: bool = True
+    infer_python_dependencies: bool = True
     environment_catalog_mapping: RegexKeyDict = {}
     default_target_environment: str = c.PROD
     log_limit: int = c.DEFAULT_LOG_LIMIT

sqlmesh/core/context.py

@@ -383,8 +383,6 @@ def __init__(
         self.pinned_environments = Environment.sanitize_names(self.config.pinned_environments)
         self.auto_categorize_changes = self.config.plan.auto_categorize_changes
         self.selected_gateway = gateway or self.config.default_gateway_name
-        self.gateway_managed_virtual_layer = self.config.gateway_managed_virtual_layer
-        self.infer_python_package_requirements = self.config.infer_python_package_requirements
 
         gw_model_defaults = self.config.gateways[self.selected_gateway].model_defaults
         if gw_model_defaults:
@@ -2587,8 +2585,8 @@ def _context_diff(
             ensure_finalized_snapshots=ensure_finalized_snapshots,
             diff_rendered=diff_rendered,
             environment_statements=self._environment_statements,
-            gateway_managed_virtual_layer=self.gateway_managed_virtual_layer,
-            infer_python_package_requirements=self.infer_python_package_requirements,
+            gateway_managed_virtual_layer=self.config.gateway_managed_virtual_layer,
+            infer_python_dependencies=self.config.infer_python_dependencies,
         )
 
     def _destroy(self) -> None:

sqlmesh/core/context_diff.py

@@ -102,7 +102,7 @@ def create(
         diff_rendered: bool = False,
         environment_statements: t.Optional[t.List[EnvironmentStatements]] = [],
         gateway_managed_virtual_layer: bool = False,
-        infer_python_package_requirements: bool = True,
+        infer_python_dependencies: bool = True,
     ) -> ContextDiff:
         """Create a ContextDiff object.
 
@@ -121,7 +121,7 @@ def create(
             environment_statements: A list of `before_all` or `after_all` statements associated with the environment.
             gateway_managed_virtual_layer: Whether the models' views in the virtual layer are created by the
                 model-specific gateway rather than the default gateway.
-            infer_python_package_requirements: Whether to statically analyze Python code to automatically infer Python
+            infer_python_dependencies: Whether to statically analyze Python code to automatically infer Python
                 package requirements.
 
         Returns:
@@ -215,7 +215,7 @@ def create(
             provided_requirements or {},
             excluded_requirements or set(),
             snapshots.values(),
-            infer_python_package_requirements=infer_python_package_requirements,
+            infer_python_dependencies=infer_python_dependencies,
         )
 
         previous_environment_statements = state_reader.get_environment_statements(environment)
@@ -483,39 +483,41 @@ def _build_requirements(
     provided_requirements: t.Dict[str, str],
     excluded_requirements: t.Set[str],
     snapshots: t.Collection[Snapshot],
-    infer_python_package_requirements: bool = True,
+    infer_python_dependencies: bool = True,
 ) -> t.Dict[str, str]:
     requirements = {
         k: v for k, v in provided_requirements.items() if k not in excluded_requirements
     }
 
-    if infer_python_package_requirements:
-        distributions = metadata.packages_distributions()
+    if not infer_python_dependencies:
+        return requirements
 
-        for snapshot in snapshots:
-            if not snapshot.is_model:
+    distributions = metadata.packages_distributions()
+
+    for snapshot in snapshots:
+        if not snapshot.is_model:
+            continue
+
+        for executable in snapshot.model.python_env.values():
+            if executable.kind != "import":
                 continue
 
-            for executable in snapshot.model.python_env.values():
-                if executable.kind != "import":
+            try:
+                start = "from " if executable.payload.startswith("from ") else "import "
+                lib = executable.payload.split(start)[1].split()[0].split(".")[0]
+                if lib not in distributions:
                     continue
 
-                try:
-                    start = "from " if executable.payload.startswith("from ") else "import "
-                    lib = executable.payload.split(start)[1].split()[0].split(".")[0]
-                    if lib not in distributions:
-                        continue
-
-                    for dist in distributions[lib]:
-                        if (
-                            dist not in requirements
-                            and dist not in IGNORED_PACKAGES
-                            and dist not in excluded_requirements
-                        ):
-                            requirements[dist] = metadata.version(dist)
-                except metadata.PackageNotFoundError:
-                    from sqlmesh.core.console import get_console
-
-                    get_console().log_warning(f"Failed to find package for {lib}.")
+                for dist in distributions[lib]:
+                    if (
+                        dist not in requirements
+                        and dist not in IGNORED_PACKAGES
+                        and dist not in excluded_requirements
+                    ):
+                        requirements[dist] = metadata.version(dist)
+            except metadata.PackageNotFoundError:
+                from sqlmesh.core.console import get_console
+
+                get_console().log_warning(f"Failed to find package for {lib}.")
 
     return requirements

tests/core/test_context.py

@@ -1244,7 +1244,7 @@ def test_deactivate_automatic_requirement_inference(copy_to_temp_path: t.Callabl
     context_path = copy_to_temp_path("examples/sushi")[0]
     config = next(iter(load_configs("config", Config, paths=context_path).values()))
 
-    config.infer_python_package_requirements = False
+    config.infer_python_dependencies = False
     context = Context(paths=context_path, config=config)
     environment = context.plan(
         "dev", no_prompts=True, skip_tests=True, skip_backfill=True, auto_apply=True

tests/core/test_integration.py

@@ -5050,7 +5050,7 @@ def test_multi_virtual_layer(copy_to_temp_path):
     assert len(prod_environment.snapshots_) == 3
 
     # Changing the flag should show a diff
-    context.gateway_managed_virtual_layer = False
+    context.config.gateway_managed_virtual_layer = False
     plan = context.plan_builder().build()
     assert not plan.requires_backfill
     assert (