run conformance tests as one whole-suite invocation per module

The conformance script now receives the module's suite folder (own module)
or the required module's copied suite root instead of a per-FRID subfolder.
The current-FRID phase's single run covers every prior FRID of the module,
so the regression phase reduces to one run per required module (plus a
re-run of the own module's suite when code changed while fixing).

On failure, the run output is attributed to the implicated functionalities
by matching suite folder names; the running context is pointed at the
earliest implicated FRID so the existing fix loop, memory keying, and
conflict detection work unchanged, and the failure evidence sent to the
fix flow is scoped to that FRID (other implicated FRIDs appear only as a
summary note). Layout-level failures (suites that cannot be discovered
together, e.g. from projects rendered before whole-suite execution) fail
fast with a regeneration hint instead of entering the fix loop.

Author: pedjaradenkovic

render_machine/actions/run_conformance_tests.py

@@ -4,11 +4,20 @@
 import render_machine.render_utils as render_utils
 from plain2code_console import console
 from render_machine.actions.base_action import BaseAction
+from render_machine.failure_attribution import detect_layout_failure
 from render_machine.render_context import RenderContext
 from render_machine.render_types import RenderError
 
 UNRECOVERABLE_ERROR_EXIT_CODES = [69]
 
+LAYOUT_FAILURE_MESSAGE = (
+    "Conformance test suites of this module could not be discovered or run together. "
+    "This usually means the conformance tests were generated by an older version of the renderer - "
+    "delete the module's conformance tests folder so they get regenerated on the next render. "
+    "If the project uses a custom conformance tests script, make sure it runs all tests found "
+    "under the folder it receives as its second argument, including tests in subfolders."
+)
+
 
 class RunConformanceTests(BaseAction):
 
@@ -19,26 +28,16 @@ class RunConformanceTests(BaseAction):
     def execute(self, render_context: RenderContext, _previous_action_payload: Any | None):
         conformance_tests_script = os.path.normpath(render_context.conformance_tests_script)
 
-        if render_context.module_name == render_context.conformance_tests_running_context.current_testing_module_name:
-            conformance_tests_folder_name = (
-                render_context.conformance_tests_running_context.get_current_conformance_test_folder_name()
-            )
-        else:
-            [conformance_tests_folder_name, _] = (
-                render_context.conformance_tests.get_source_conformance_test_folder_name(
-                    render_context.module_name,
-                    render_context.required_modules,
-                    render_context.conformance_tests_running_context.current_testing_module_name,
-                    render_context.conformance_tests_running_context.get_current_conformance_test_folder_name(),
-                )
-            )
+        conformance_tests_folder_name = render_context.conformance_tests.get_module_suite_run_folder(
+            render_context.module_name,
+            render_context.required_modules,
+            render_context.conformance_tests_running_context.current_testing_module_name,
+        )
 
         console.info(
             f"Running conformance tests script {conformance_tests_script} "
-            + f"for {conformance_tests_folder_name} ("
-            + f"functionality {render_context.conformance_tests_running_context.current_testing_frid} "
-            + f"in module {render_context.conformance_tests_running_context.current_testing_module_name}"
-            + ")."
+            + f"for the test suite {conformance_tests_folder_name} "
+            + f"of module {render_context.conformance_tests_running_context.current_testing_module_name}."
         )
         exit_code, conformance_tests_issue, conformance_tests_temp_log_file_path = render_utils.execute_script(
             conformance_tests_script,
@@ -54,21 +53,23 @@ def execute(self, render_context: RenderContext, _previous_action_payload: Any |
         )
         render_context.script_execution_history.should_update_script_outputs = True
 
-        render_context.memory_manager.create_conformance_tests_memory(
-            render_context, exit_code, conformance_tests_issue
-        )
-
         if exit_code == 0:
+            render_context.memory_manager.create_conformance_tests_memory(
+                render_context, exit_code, conformance_tests_issue
+            )
+            # A passing whole-suite run of the module being rendered covers the FRID being
+            # implemented, so its unresolved memory entries can be cleared.
             if (
                 render_context.conformance_tests_running_context.current_testing_module_name
                 == render_context.module_name
-                and render_context.conformance_tests_running_context.current_testing_frid
-                == render_context.frid_context.frid
             ):
                 render_context.memory_manager.delete_unresolved_memory_files()
             return self.SUCCESSFUL_OUTCOME, None
 
         if exit_code in UNRECOVERABLE_ERROR_EXIT_CODES:
+            render_context.memory_manager.create_conformance_tests_memory(
+                render_context, exit_code, conformance_tests_issue
+            )
             console.error(conformance_tests_issue)
             return (
                 self.UNRECOVERABLE_ERROR_OUTCOME,
@@ -80,4 +81,23 @@ def execute(self, render_context: RenderContext, _previous_action_payload: Any |
                 ).to_payload(),
             )
 
-        return self.FAILED_OUTCOME, {"previous_conformance_tests_issue": conformance_tests_issue}
+        if detect_layout_failure(conformance_tests_issue):
+            console.error(conformance_tests_issue)
+            return (
+                self.UNRECOVERABLE_ERROR_OUTCOME,
+                RenderError.encode(
+                    message=LAYOUT_FAILURE_MESSAGE,
+                    error_type="ENVIRONMENT_ERROR",
+                    script=conformance_tests_script,
+                    issue=conformance_tests_issue,
+                ).to_payload(),
+            )
+
+        # Attribute the failure to a FRID (re-pointing the running context for the fix loop)
+        # before creating memory, so the memory entry is keyed to the implicated FRID.
+        conformance_tests_evidence = render_context.route_conformance_failure_to_frid(conformance_tests_issue)
+        render_context.memory_manager.create_conformance_tests_memory(
+            render_context, exit_code, conformance_tests_issue
+        )
+
+        return self.FAILED_OUTCOME, {"previous_conformance_tests_issue": conformance_tests_evidence}

render_machine/conformance_tests.py

@@ -144,6 +144,34 @@ def store_conformance_tests_files(
             style=console.OUTPUT_STYLE,
         )
 
+    def get_module_suite_run_folder(
+        self,
+        module_name: str,
+        required_modules: list[PlainModule],
+        current_testing_module_name: str,
+    ) -> str:
+        """Resolve the folder to pass to the conformance test script for a whole-module run.
+
+        For the module being rendered this is its own conformance tests folder. For a
+        required module it is the most specific existing copy of that module's tests
+        (mirroring get_source_conformance_test_folder_name at module granularity), falling
+        back to the required module's own folder when no copy exists yet.
+        """
+        if current_testing_module_name == module_name:
+            return self.get_module_conformance_tests_folder(module_name)
+
+        modules_list = [module_name] + [m.module_name for m in reversed(required_modules)]
+
+        for copy_from_module in modules_list:
+            if copy_from_module == current_testing_module_name:
+                break
+
+            candidate = self.get_module_conformance_tests_folder(copy_from_module + "/." + current_testing_module_name)
+            if os.path.exists(candidate):
+                return candidate
+
+        return self.get_module_conformance_tests_folder(current_testing_module_name)
+
     def fetch_all_existing_conformance_test_files(self, module_name: str) -> dict[str, str]:
         """Fetch the content of all existing conformance test files of the module.
 

render_machine/render_context.py

@@ -1,3 +1,4 @@
+import os.path
 import threading
 from copy import deepcopy
 from typing import Callable, Optional
@@ -11,7 +12,7 @@
 from plain2code_events import RenderContextSnapshot
 from plain2code_state import RunState
 from plain_modules import PlainModule
-from render_machine import triggers
+from render_machine import failure_attribution, triggers
 from render_machine.conformance_tests import CONFORMANCE_TESTS_DEFINITION_FILE_NAME, ConformanceTests
 from render_machine.render_types import (
     AcceptanceTestPhase,
@@ -343,7 +344,7 @@ def _start_regression_phase(self):
             ctx.code_changed_during_regression = False
 
         ctx.execution_phase = TestExecutionPhase.RUNNING_REGRESSION
-        ctx.current_testing_frid = None  # Will be set by get_first_conformance_tests_running_context
+        ctx.regression_module_index = None  # Will be advanced by _get_next_regression_module
 
     def _get_next_test_to_run(self):
         """Determine which test to run next based on current phase."""
@@ -354,6 +355,78 @@ def _get_next_test_to_run(self):
         else:
             return self.get_next_conformance_tests_running_context()
 
+    def _module_has_conformance_tests(self, module_name: str) -> bool:
+        return len(self.conformance_tests.get_conformance_tests_json(module_name)) > 0
+
+    def _get_next_regression_module(self) -> Optional[str]:
+        """Advance to the next module whose whole suite should run during regression.
+
+        The regression sequence is every required module (in requires order). When code
+        changed while fixing a conformance test, the module being rendered is appended so
+        its suite gets re-verified against the changed code.
+        """
+        ctx = self.conformance_tests_running_context
+
+        module_names = [module.module_name for module in (self.required_modules or [])]
+        if ctx.code_changed_during_regression:
+            module_names = module_names + [self.module_name]
+
+        next_index = 0 if ctx.regression_module_index is None else ctx.regression_module_index + 1
+        while next_index < len(module_names):
+            module_name = module_names[next_index]
+            if module_name == self.module_name or self._module_has_conformance_tests(module_name):
+                ctx.regression_module_index = next_index
+                return module_name
+            next_index += 1
+
+        return None
+
+    def _switch_to_module_suite(self, module_name: str):
+        """Point the running context at a module so its whole suite is the next run."""
+        ctx = self.conformance_tests_running_context
+
+        if module_name == self.module_name:
+            ctx.current_testing_module_name = self.module_name
+            ctx.current_testing_frid = ctx.frid_being_implemented
+        else:
+            if not ctx.conformance_tests_json_has_module_populated(module_name):
+                ctx.set_conformance_tests_json(
+                    module_name, self.conformance_tests.get_conformance_tests_json(module_name)
+                )
+            ctx.current_testing_module_name = module_name
+            ctx.current_testing_frid = next(iter(ctx.get_conformance_tests_json(module_name)))
+
+        self._setup_test_specifications()
+
+    def route_conformance_failure_to_frid(self, conformance_tests_issue: str) -> str:
+        """Attribute a failed whole-suite run to a functionality and scope the evidence to it.
+
+        Points the running context at the earliest implicated FRID (the fix loop, memory
+        creation, and conflict detection all key off current_testing_frid) and returns the
+        failure evidence for that FRID: its own failure blocks plus a summary note about
+        other implicated FRIDs. Returns the issue unchanged when no FRID can be identified.
+        """
+        ctx = self.conformance_tests_running_context
+        module_json = ctx.get_conformance_tests_json(ctx.current_testing_module_name)
+
+        implicated_frids = failure_attribution.attribute_failures(conformance_tests_issue, module_json)
+        if not implicated_frids:
+            return conformance_tests_issue
+
+        target_frid = implicated_frids[0]
+        if target_frid != ctx.current_testing_frid:
+            console.info(
+                f"Conformance test failure attributed to functionality {target_frid} "
+                f"in module {ctx.current_testing_module_name}."
+            )
+            ctx.current_testing_frid = target_frid
+            self._setup_test_specifications()
+
+        folder_basename = os.path.basename(module_json[target_frid]["folder_name"])
+        evidence = failure_attribution.extract_frid_failure_evidence(conformance_tests_issue, folder_basename)
+
+        return evidence + failure_attribution.format_other_frids_note(implicated_frids, target_frid)
+
     def _has_reached_implementation_frid(self) -> bool:
         """Check if regression has reached the FRID being implemented."""
         ctx = self.conformance_tests_running_context
@@ -469,32 +542,23 @@ def _handle_current_frid_testing(self):
         raise RuntimeError(f"Unexpected acceptance test phase: {ctx.acceptance_test_phase}")
 
     def _handle_regression_testing(self):
-        """Handle regression testing of all earlier FRIDs."""
-
-        # Get next test to run
-        self.conformance_tests_running_context = self._get_next_test_to_run()
+        """Handle regression testing: one whole-suite run per module.
 
-        # Get reference to the updated context
+        The module's own suite already ran in full during the current-FRID phase, so
+        regression only needs the required modules' suites (plus a re-run of the own
+        module's suite when code changed while fixing a conformance test).
+        """
         ctx = self.conformance_tests_running_context
 
-        # Set up specs and run test
-        self._setup_test_specifications()
+        next_module = self._get_next_regression_module()
 
-        if ctx.current_conformance_tests_exist():
-            # Check if this is the implementation FRID (last test to run)
-            if self._has_reached_implementation_frid():
-                # Reached implementation FRID - only re-run it if code changed during regression
-                if ctx.code_changed_during_regression:
-                    # Code changed - run the implementation FRID again to verify no regression
-                    # After it passes, mark as completed on next iteration
-                    ctx.execution_phase = TestExecutionPhase.COMPLETED
-                else:
-                    # No code changes - skip re-running implementation FRID, mark as completed immediately
-                    ctx.execution_phase = TestExecutionPhase.COMPLETED
-                    self.machine.dispatch(triggers.MARK_ALL_CONFORMANCE_TESTS_PASSED)
-                    return
+        if next_module is None:
+            ctx.execution_phase = TestExecutionPhase.COMPLETED
+            self.machine.dispatch(triggers.MARK_ALL_CONFORMANCE_TESTS_PASSED)
+            return
 
-            self.machine.dispatch(triggers.MARK_CONFORMANCE_TESTS_READY)
+        self._switch_to_module_suite(next_module)
+        self.machine.dispatch(triggers.MARK_CONFORMANCE_TESTS_READY)
 
     # ========== Main Conformance Test Orchestration ==========
 
@@ -534,7 +598,17 @@ def start_conformance_tests_for_frid(self):
             return
 
         # ========== STEP 3: Handle Current FRID Testing ==========
-        if self._should_run_current_frid_tests():
+        if ctx.execution_phase == TestExecutionPhase.TESTING_CURRENT_FRID:
+            # A failed whole-suite run may have re-pointed current_testing_frid at the
+            # implicated FRID for the fix loop; restore the FRID being implemented before
+            # continuing the current-FRID phases.
+            if (
+                ctx.current_testing_module_name != self.module_name
+                or ctx.current_testing_frid != ctx.frid_being_implemented
+            ):
+                ctx.current_testing_module_name = self.module_name
+                ctx.current_testing_frid = ctx.frid_being_implemented
+                self._setup_test_specifications()
             self._handle_current_frid_testing()
             return
 

render_machine/render_types.py

@@ -83,6 +83,8 @@ def __init__(
         self.execution_phase: TestExecutionPhase = TestExecutionPhase.TESTING_CURRENT_FRID
         self.acceptance_test_phase: AcceptanceTestPhase = AcceptanceTestPhase.NOT_STARTED
         self.acceptance_tests_completed: int = 0
+        # Index into the regression module sequence; None means regression has not started.
+        self.regression_module_index: Optional[int] = None
         self.frid_being_implemented: Optional[str] = frid_being_implemented
         self.test_that_triggered_code_change: Optional[tuple[str, str]] = None
         self.code_changed_during_regression: bool = False

tests/test_conformance_tests.py

@@ -76,3 +76,33 @@ def test_fetch_all_existing_conformance_test_files_skips_binary_files(conformanc
     files_content = conformance_tests.fetch_all_existing_conformance_test_files(MODULE_NAME)
 
     assert files_content == {os.path.join("some_functionality", "test_some.py"): "some test"}
+
+
+class _FakeModule:
+    def __init__(self, module_name):
+        self.module_name = module_name
+
+
+def test_get_module_suite_run_folder_own_module(conformance_tests, conformance_tests_dir):
+    folder = conformance_tests.get_module_suite_run_folder(MODULE_NAME, [], MODULE_NAME)
+
+    assert folder == os.path.join(conformance_tests_dir, MODULE_NAME)
+
+
+def test_get_module_suite_run_folder_required_module_with_copy(conformance_tests, conformance_tests_dir):
+    copy_folder = os.path.join(conformance_tests_dir, MODULE_NAME, ".required_module")
+    _write_file(copy_folder, os.path.join("some_frid", "test_x.py"), "copied test")
+
+    folder = conformance_tests.get_module_suite_run_folder(
+        MODULE_NAME, [_FakeModule("required_module")], "required_module"
+    )
+
+    assert folder == copy_folder
+
+
+def test_get_module_suite_run_folder_required_module_without_copy(conformance_tests, conformance_tests_dir):
+    folder = conformance_tests.get_module_suite_run_folder(
+        MODULE_NAME, [_FakeModule("required_module")], "required_module"
+    )
+
+    assert folder == os.path.join(conformance_tests_dir, "required_module")