Improve execution performance in RequestManager validation and Execution.has_ended

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "retrack"
-version = "3.6.0"
+version = "3.7.0"
 description = "A business rules engine"
 authors = ["Gabriel Guarisa <gabriel.guarisa@pier.digital>"]
 license = "MIT"

retrack/engine/base.py

@@ -21,7 +21,7 @@ def __init__(
         states: pd.DataFrame,
         filters: dict = None,
         context: registry.Registry = None,
-        child_executions = None,
+        child_executions=None,
         nodes: dict = None,
         constants: dict = None,
     ):
@@ -102,7 +102,7 @@ def result(self) -> pd.DataFrame:
         ]
 
     def has_ended(self) -> bool:
-        return self.states[constants.OUTPUT_REFERENCE_COLUMN].isna().sum() == 0
+        return not self.states[constants.OUTPUT_REFERENCE_COLUMN].isna().any()
 
     def to_dict(self) -> dict:
         return {

retrack/engine/request_manager.py

@@ -1,7 +1,6 @@
 import typing
 
 import pandas as pd
-import pandera
 import pydantic
 
 from retrack.nodes.base import BaseNode, NodeKind
@@ -75,7 +74,7 @@ def model(self) -> typing.Type[pydantic.BaseModel]:
         return self._model
 
     @property
-    def dataframe_model(self) -> pandera.DataFrameSchema:
+    def dataframe_model(self):
         return self._dataframe_model
 
     def __create_model(
@@ -117,23 +116,19 @@ def __create_model(
             ),
         )
 
-    def __create_dataframe_model(self) -> pandera.DataFrameSchema:
-        """Create a pydantic model from the RequestManager's inputs"""
-        fields = {}
-        for input_field in self.inputs:
-            fields[input_field.data.name] = pandera.Column(
-                str,
-                nullable=input_field.data.default is not None,
-                coerce=True,
-                default=input_field.data.default,
-            )
+    def __create_dataframe_model(self) -> dict:
+        """Create a lightweight validation schema from the RequestManager's inputs.
 
-        return pandera.DataFrameSchema(
-            fields,
-            index=pandera.Index(int),
-            # strict=True,
-            coerce=True,
-        )
+        Returns:
+            dict: mapping column_name -> {"nullable": bool, "default": value}
+        """
+        schema = {}
+        for input_field in self.inputs:
+            schema[input_field.data.name] = {
+                "nullable": input_field.data.default is not None,
+                "default": input_field.data.default,
+            }
+        return schema
 
     def validate(
         self,
@@ -156,4 +151,29 @@ def validate(
         if not isinstance(payload, pd.DataFrame):
             raise TypeError(f"payload must be a pandas.DataFrame, not {type(payload)}")
 
-        return self.dataframe_model.validate(payload)
+        schema = self.dataframe_model
+
+        # Check required columns exist
+        missing = set(schema.keys()) - set(payload.columns)
+        if missing:
+            raise ValueError(f"Missing columns: {missing}")
+
+        result = payload.copy()
+
+        for col_name, col_schema in schema.items():
+            series = result[col_name]
+
+            # Handle nulls
+            null_mask = series.isna() | series.isin([None, "None", "", "null"])
+            if null_mask.any():
+                if not col_schema["nullable"]:
+                    raise ValueError(
+                        f"Column '{col_name}' contains null values but is not nullable"
+                    )
+                if col_schema["default"] is not None:
+                    series = series.where(~null_mask, col_schema["default"])
+
+            # Coerce to str
+            result[col_name] = series.astype(str)
+
+        return result

tests/test_engine/test_request_manager.py

@@ -1,5 +1,4 @@
 import pandas as pd
-import pandera
 import pydantic
 import pytest
 
@@ -41,7 +40,7 @@ def test_validate_payload_with_valid_payload(valid_input_dict_before_validation)
 
     assert issubclass(rm.model, pydantic.BaseModel)
 
-    assert isinstance(rm.dataframe_model, pandera.api.pandas.container.DataFrameSchema)
+    assert isinstance(rm.dataframe_model, dict)
 
     payload = rm.model(example="test")
 
@@ -63,3 +62,29 @@ def test_validate_dict_with_none_value(valid_input_dict_before_validation):
     assert issubclass(rm.model, pydantic.BaseModel)
     assert rm.model(example=None) == rm.model(example="Hello World")
     assert rm.model() == rm.model(example="Hello World")
+
+
+def test_validate_dataframe_replaces_sentinel_strings_with_default(
+    valid_input_dict_before_validation,
+):
+    """Sentinel strings count as null in the DataFrame path, matching StrFieldValidator."""
+    rm = RequestManager([Input(**valid_input_dict_before_validation)])
+
+    result = rm.validate(
+        pd.DataFrame([{"example": v} for v in ["None", "", "null", None]])
+    )
+
+    assert result["example"].tolist() == ["Hello World"] * 4
+
+
+def test_validate_dataframe_rejects_sentinel_strings_when_not_nullable(
+    valid_input_dict_before_validation,
+):
+    """Without a default the input is not nullable, so sentinel strings must raise."""
+    rm = RequestManager(
+        [Input(**{**valid_input_dict_before_validation, "data": {"name": "example"}})]
+    )
+
+    for value in ["None", "", "null", None]:
+        with pytest.raises(ValueError, match="not nullable"):
+            rm.validate(pd.DataFrame([{"example": value}]))