Enable requiring zipped modules ENG-168

Author: NejcS

module_renderer.py

@@ -111,6 +111,12 @@ def _render_module(
         ):
             return False, False
 
+        # We are going to (re)render this module. If it currently exists only as a
+        # "<module>.module" archive, unpack it into the real plain_modules/<module>/ folder now,
+        # before RenderContext snapshots the build folder path and before the git repos, metadata,
+        # and memory folder are touched.
+        plain_module.ensure_module_unpacked()
+
         memory_manager = MemoryManager(
             self.codeplainAPI,
             plain_module.module_memory_folder,

partial_rendering.py

@@ -12,7 +12,7 @@ class PlainModuleRenderState:
     last_render_module: PlainModule
     last_render_frid: str | None
     change: PlainModule | None = None
-    change_type: Literal["spec_change", "code_change"] | None = None
+    change_type: Literal["spec_change", "code_change", "missing_conformance_tests"] | None = None
 
 
 @dataclass
@@ -56,6 +56,19 @@ def code_change(plain_module: PlainModule) -> PlainModule | None:
     return None
 
 
+def archive_missing_conformance_tests(plain_module: PlainModule, render_conformance_tests: bool) -> PlainModule | None:
+    """Return the first module available only as a "<module>.module" archive that has no conformance
+    tests while conformance testing is enabled. Such an archive cannot be consumed as-is (regression
+    needs the tests), so the module must be re-rendered."""
+    if not render_conformance_tests:
+        return None
+    all_modules = plain_module.all_required_modules + [plain_module]
+    for _module in all_modules:
+        if _module.is_archived_only() and not _module.archive_has_conformance_tests():
+            return _module
+    return None
+
+
 def module_comes_before_or_equal(
     all_required_modules: list[PlainModule],
     module1: PlainModule,
@@ -70,15 +83,24 @@ def module_comes_before_or_equal(
     raise ValueError(f"Module {module1.module_name} and {module2.module_name} not found in {all_required_modules}")
 
 
-def get_plain_module_render_state(plain_module: PlainModule) -> PlainModuleRenderState | None:
+def get_plain_module_render_state(
+    plain_module: PlainModule, render_conformance_tests: bool = False
+) -> PlainModuleRenderState | None:
     sc = spec_change(plain_module)
     cc = code_change(plain_module)
+    mt = archive_missing_conformance_tests(plain_module, render_conformance_tests)
     all_required_modules = plain_module.all_required_modules
     last_rendered_module_name, last_rendered_frid = plain_module.get_module_render_status()
-    if last_rendered_module_name is None and last_rendered_frid is None:
+    if last_rendered_module_name is None and last_rendered_frid is None and mt is None:
         return None
 
-    if last_rendered_module_name == plain_module.module_name:
+    if last_rendered_module_name is None:
+        # Nothing has been rendered yet, but an archived module blocks consumption (no tests).
+        # Anchor the state on that module so the user is prompted to re-render it.
+        assert mt is not None  # guaranteed by the early return above
+        module = mt
+        last_rendered_frid = None
+    elif last_rendered_module_name == plain_module.module_name:
         module = plain_module
     else:
         found_module: PlainModule | None = None
@@ -99,6 +121,13 @@ def get_plain_module_render_state(plain_module: PlainModule) -> PlainModuleRende
         change_type=None,
     )
 
+    # An archive that lacks conformance tests (while testing is enabled) is a hard blocker: it cannot
+    # be consumed as-is, so it takes precedence over spec/code changes.
+    if mt is not None:
+        pr.change = mt
+        pr.change_type = "missing_conformance_tests"
+        return pr
+
     if sc is None and cc is None:
         return pr
 
@@ -129,6 +158,8 @@ def get_all_affected_modules_from_change(
             start_module = plain_module.get_next_module(plain_module_render_state.change.module_name)
         else:
             start_module = plain_module_render_state.change
+    elif plain_module_render_state.change_type == "missing_conformance_tests":
+        start_module = plain_module_render_state.change
     else:
         raise ValueError(f"Unknown change type: {plain_module_render_state.change_type}")
 

plain2code.py

@@ -28,6 +28,7 @@
     ImportedModuleWithFunctionalitiesError,
     InvalidAPIKey,
     InvalidFridArgument,
+    InvalidModuleArchiveError,
     MissingAPIKey,
     MissingFunctionalitiesError,
     MissingPreviousFunctionalitiesError,
@@ -79,6 +80,7 @@
     UnsupportedResourceType,
     UnsupportedBase64Content,
     GitNotInstalledError,
+    InvalidModuleArchiveError,
     SystemExit,
 )
 
@@ -212,6 +214,15 @@ def render(  # noqa: C901
 
     warn_if_acceptance_tests_without_conformance_script(plain_module, args)
 
+    # A built module can be distributed as a "<module>.module" zip archive instead of an unpacked directory.
+    # Extract any such module to a scratch location up front. Scratch dirs are removed in main()'s finally.
+    # Skip an archive-only module that lacks conformance tests while testing is enabled: it cannot be
+    # consumed as-is, so leave it unmaterialized so it is detected below as needing a re-render.
+    for module in plain_module.all_required_modules + [plain_module]:
+        if args.render_conformance_tests and module.is_archived_only() and not module.archive_has_conformance_tests():
+            continue
+        module.materialize()
+
     # The module_metadata file lives outside the code git repo. That means that a crash mid-render can leave it
     # claiming a functionality was implemented even thought it wasn't yet committed (because of the crash).
     # Out of precaution, this reconciles every module_metadata against the code repo.
@@ -220,7 +231,7 @@ def render(  # noqa: C901
 
     render_choice = None
     if render_range is None:
-        plain_module_render_state = get_plain_module_render_state(plain_module)
+        plain_module_render_state = get_plain_module_render_state(plain_module, args.render_conformance_tests)
         if plain_module_render_state is not None:
             render_choices = get_render_choices(plain_module, plain_module_render_state, args.force_render)
             ask_user = True
@@ -417,6 +428,9 @@ def main():  # noqa: C901
             args.filename,
             error_message=error_message,
         )
+        # Remove any scratch extractions created for archive-only ("<module>.module") modules.
+        for module in plain_module.all_required_modules + [plain_module]:
+            module.cleanup_scratch()
 
     if args.headless and (exc_info is not None or not run_state.render_succeeded):
         sys.exit(1)

plain2code_exceptions.py

@@ -117,3 +117,11 @@ class GitNotInstalledError(Exception):
     """Raised when git is not installed or not found on PATH."""
 
     pass
+
+
+class InvalidModuleArchiveError(Exception):
+    """Raised when a ``<module>.module`` archive is missing, corrupt, or has an
+    unexpected layout (not a zip, missing ``code/``/``tests/``, ``.git`` not a real
+    directory, detached HEAD, or an unsafe member path)."""
+
+    pass