Cover compact result archives and released Plate reconstruction

tests/requirements.txt

@@ -1,2 +1,3 @@
 pytest>=8,<9
 biomero-schema>=0.3.0b1,<0.4
+biomero-shallower>=0.1.0b6,<0.2

tests/test_result_archive_validation.py

@@ -0,0 +1,81 @@
+"""Validate archive content rather than assuming small results are invalid."""
+import ast
+import logging
+import os
+import random
+from pathlib import Path
+from types import SimpleNamespace
+from typing import Tuple
+from unittest.mock import Mock
+from uuid import UUID, uuid4
+import zipfile
+
+import pytest
+
+ROOT = Path(os.environ.get("BIOMERO_SCRIPTS_ROOT", Path(__file__).parents[1]))
+SCRIPT = ROOT / "_data" / "SLURM_Import_Results.py"
+pytestmark = pytest.mark.skipif(
+    "zip_size < 1024" in SCRIPT.read_text(encoding="utf-8")
+    and os.getenv("BIOMERO_TEST_SMALL_ARCHIVES") != "1",
+    reason="selected source predates content-based result archive validation",
+)
+
+
+@pytest.fixture
+def retrieve(tmp_path):
+    tree = ast.parse(SCRIPT.read_text(encoding="utf-8"))
+    function = next(n for n in tree.body if isinstance(n, ast.FunctionDef)
+                    and n.name == "extract_slurm_results_zip")
+    move = Mock(return_value=str(tmp_path / "permanent"))
+    namespace = dict(
+        SlurmClient=object, UUID=UUID, Tuple=Tuple, os=os, zipfile=zipfile,
+        logger=logging.getLogger(__name__), shallow_remote_results=Mock(),
+        move_results_to_permanent_storage=move,
+    )
+    exec(compile(ast.Module(body=[function], type_ignores=[]), str(SCRIPT), "exec"), namespace)
+    slurm = SimpleNamespace(
+        extract_data_location_from_log=Mock(return_value="/scratch/job"),
+        run_commands=Mock(return_value=SimpleNamespace(ok=True, stdout="1\n")),
+        zip_data_on_slurm_server=Mock(return_value=SimpleNamespace(ok=True, stdout="Everything is Ok")),
+        copy_zip_locally=Mock(),
+    )
+    archive = tmp_path / "123_out.zip"
+
+    def run():
+        return namespace["extract_slurm_results_zip"](
+            slurm, "123", str(tmp_path), "group", uuid4(), "")
+
+    return SimpleNamespace(run=run, archive=archive, move=move, slurm=slurm)
+
+
+@pytest.mark.parametrize("payload", [b"II*\x00" + bytes(6144), random.Random(0).randbytes(2048)],
+                         ids=["compressed-small", "larger"])
+def test_valid_result_archive_is_accepted_regardless_of_compressed_size(retrieve, payload):
+    with zipfile.ZipFile(retrieve.archive, "w", zipfile.ZIP_DEFLATED) as archive:
+        archive.writestr("result.tif", payload)
+    if payload.startswith(b"II*"):
+        assert retrieve.archive.stat().st_size < 1024
+    result = retrieve.run()
+    assert Path(result[2]) == retrieve.archive
+    retrieve.move.assert_called_once()
+    assert Path(retrieve.move.call_args.args[-1]) == retrieve.archive
+    with zipfile.ZipFile(retrieve.archive) as archive:
+        assert archive.read("result.tif") == payload
+
+
+@pytest.mark.parametrize("contents,message", [
+    ("empty", "contains no files"),
+    ("directories", "only directories"),
+    ("corrupt", "not a valid zip"),
+    ("missing", "not found"),
+])
+def test_unusable_archive_never_reaches_permanent_storage(retrieve, contents, message):
+    if contents in ("empty", "directories"):
+        with zipfile.ZipFile(retrieve.archive, "w") as archive:
+            if contents == "directories":
+                archive.writestr("empty/", b"")
+    elif contents == "corrupt":
+        retrieve.archive.write_bytes(b"not a ZIP archive")
+    with pytest.raises(RuntimeError, match=message):
+        retrieve.run()
+    retrieve.move.assert_not_called()

tests/test_shallow_plate_reanalysis.py

@@ -0,0 +1,94 @@
+"""Use the published reconstruction return type, including its real manifest."""
+import ast
+import logging
+import os
+from pathlib import Path
+from types import SimpleNamespace
+from uuid import uuid4
+
+import pytest
+from biomero_schema.zarr import (
+    CanonicalPlateImage, CanonicalPlateSource, CanonicalZarrSource, ManagedZarrNode,
+    PixelIdentity, ShallowBindings, ShallowCollection, ShallowImageBinding,
+    ShallowImageNode, ShallowLabelBinding, ShallowLabelNode, ShallowManifest,
+    ShallowPlateReference, ZarrLabelComponent,
+)
+from biomero_shallower.result_zarr import MaterializedShallowResult
+
+ROOT = Path(os.environ.get("BIOMERO_SCRIPTS_ROOT", Path(__file__).parents[1]))
+SCRIPT = ROOT / "_data" / "_SLURM_Image_Transfer.py"
+pytestmark = pytest.mark.skipif(
+    'getattr(materialized, "manifest", None)' not in SCRIPT.read_text(encoding="utf-8")
+    and os.getenv("BIOMERO_TEST_PLATE_REANALYSIS") != "1",
+    reason="selected source predates schema-2 Plate reconstruction support",
+)
+
+
+def fixture(profile, labels):
+    nodes, bindings, label_nodes, label_bindings, managed = [], [], [], [], []
+    for i, path in enumerate(["A/1/0", "A/2/0"]):
+        ident = PixelIdentity(node_path=path, role="image", iscc_code="ISCC:KPIXEL",
+                              data_code="ISCC:GDATA", instance_code="ISCC:IINSTANCE",
+                              tool_version="0.2.0", imagewalk_revision="test",
+                              shape=(1, 1, 1, 8, 8), dtype="uint16", axes=("t", "c", "z", "y", "x"))
+        source = CanonicalZarrSource(storage_root="data", relative_path="source.zarr",
+            node_path=path, source_object_type="Plate", source_object_id=9,
+            source_generation=1, interchange_profile=profile, pixel_identity=ident,
+            pixel_identity_origin="canonical-bootstrap", canonical_pixel_verified=True)
+        nodes.append(ShallowImageNode(node_id=f"i{i}", name=f"field{i}", node_path=path))
+        bindings.append(ShallowImageBinding(node_id=f"i{i}", source=source, returned_pixel_identity=ident))
+        if labels:
+            label_path=path+"/labels/cells"
+            component=ZarrLabelComponent(logical_node_path=label_path,
+                pixel_identity=ident.model_copy(update={"node_path":label_path,"role":"label"}),
+                source=ManagedZarrNode(storage_root="data",relative_path="result.zarr",node_path=label_path))
+            managed.append(component)
+            label_nodes.append(ShallowLabelNode(node_id=f"l{i}",name=f"cells{i}",node_path=label_path,source_image_id=f"i{i}"))
+            label_bindings.append(ShallowLabelBinding(node_id=f"l{i}",component=component))
+    manifest=ShallowManifest(workflow_id=uuid4(),transfer_artifact="plate.zarr",interchange_profile=profile,
+        collection=ShallowCollection(name="Plate",images=tuple(nodes),labels=tuple(label_nodes)),
+        bindings=ShallowBindings(images=tuple(bindings),labels=tuple(label_bindings)))
+    reference=ShallowPlateReference(storage_root="data",relative_path="result.zarr",
+        workflow_id=manifest.workflow_id,transfer_artifact="plate.zarr",source_object_id=9,
+        source_generation=1,image_node_count=2,interchange_profile=profile)
+    return manifest,reference,tuple(managed)
+
+
+def load_save(reference, materialized):
+    names={"save_as_zarr","canonical_plate_source_from_collection"}
+    tree=ast.parse(SCRIPT.read_text(encoding="utf-8"))
+    body=[n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name in names]
+    namespace=dict(os=os,Path=Path,ShallowPlateReference=ShallowPlateReference,
+        CanonicalPlateImage=CanonicalPlateImage,CanonicalPlateSource=CanonicalPlateSource,
+        logger=logging.getLogger(__name__),log=lambda _:None,SHALLOW_ZARR_RESTORE_AVAILABLE=True,
+        get_shallow_plate_reference=lambda _:reference,materialize_shallow_zarr=lambda *args:materialized,
+        constants=SimpleNamespace(transfer=SimpleNamespace(DATA_TYPE_IMAGE="Image",DATA_TYPE_PLATE="Plate")))
+    exec(compile(ast.Module(body=body,type_ignores=[]),str(SCRIPT),"exec"),namespace)
+    return namespace["save_as_zarr"]
+
+
+@pytest.mark.parametrize("profile",["ngff-0.4-zarr-v2","ngff-0.5-zarr-v3"])
+@pytest.mark.parametrize("labels",[False,True])
+def test_plate_reanalysis_uses_manifest_bindings(tmp_path,profile,labels):
+    manifest,reference,managed=fixture(profile,labels)
+    materialized=MaterializedShallowResult(tmp_path/"plate.zarr",manifest,managed)
+    save=load_save(reference,materialized)
+    plate=SimpleNamespace(getName=lambda:"plate",getId=lambda:99)
+    source,artifact,restored_labels=save(None,"session",plate,str(tmp_path),"Plate",
+        storage_roots={"data":tmp_path},shallow_zarr_storage=False)
+    assert source.source_object_id==9
+    assert source.interchange_profile==profile
+    assert artifact=="plate.zarr"
+    assert restored_labels==managed
+    assert tuple(image.image_node_path for image in source.images)==("A/1/0","A/2/0")
+    for i,image in enumerate(source.images):
+        assert image.source==manifest.image_binding(f"i{i}").source
+        assert image.labels==((managed[i],) if labels else ())
+
+
+def test_missing_managed_labels_stop_reanalysis(tmp_path):
+    manifest,reference,managed=fixture("ngff-0.5-zarr-v3",True)
+    save=load_save(reference,MaterializedShallowResult(tmp_path/"plate.zarr",manifest,managed[:1]))
+    with pytest.raises(ValueError,match="lost labels"):
+        save(None,"session",SimpleNamespace(getName=lambda:"plate",getId=lambda:99),
+             str(tmp_path),"Plate",storage_roots={"data":tmp_path})