From 865cb24ca7d644298af3559e8857090cddd41a2e Mon Sep 17 00:00:00 2001 From: Luik Date: Tue, 27 Oct 2026 19:26:31 +0100 Subject: [PATCH] Cover compact ZIP results and released shallow Plate materialization --- tests/requirements.txt | 1 + tests/test_result_archive_validation.py | 81 +++++++++++++++++++++ tests/test_shallow_plate_reanalysis.py | 94 +++++++++++++++++++++++++ 3 files changed, 176 insertions(+) create mode 100644 tests/test_result_archive_validation.py create mode 100644 tests/test_shallow_plate_reanalysis.py diff --git a/tests/requirements.txt b/tests/requirements.txt index ed37696..f631b14 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,2 +1,3 @@ pytest>=8,<9 biomero-schema>=0.3.0b1,<0.4 +biomero-shallower>=0.1.0b6,<0.2 diff --git a/tests/test_result_archive_validation.py b/tests/test_result_archive_validation.py new file mode 100644 index 0000000..e6b76af --- /dev/null +++ b/tests/test_result_archive_validation.py @@ -0,0 +1,81 @@ +"""Validate archive content rather than assuming small results are invalid.""" +import ast +import logging +import os +import random +from pathlib import Path +from types import SimpleNamespace +from typing import Tuple +from unittest.mock import Mock +from uuid import UUID, uuid4 +import zipfile + +import pytest + +ROOT = Path(os.environ.get("BIOMERO_SCRIPTS_ROOT", Path(__file__).parents[1])) +SCRIPT = ROOT / "_data" / "SLURM_Import_Results.py" +pytestmark = pytest.mark.skipif( + "zip_size < 1024" in SCRIPT.read_text(encoding="utf-8") + and os.getenv("BIOMERO_TEST_SMALL_ARCHIVES") != "1", + reason="selected source predates content-based result archive validation", +) + + +@pytest.fixture +def retrieve(tmp_path): + tree = ast.parse(SCRIPT.read_text(encoding="utf-8")) + function = next(n for n in tree.body if isinstance(n, ast.FunctionDef) + and n.name == "extract_slurm_results_zip") + move = Mock(return_value=str(tmp_path / "permanent")) + namespace = dict( + SlurmClient=object, UUID=UUID, Tuple=Tuple, os=os, zipfile=zipfile, + logger=logging.getLogger(__name__), shallow_remote_results=Mock(), + move_results_to_permanent_storage=move, + ) + exec(compile(ast.Module(body=[function], type_ignores=[]), str(SCRIPT), "exec"), namespace) + slurm = SimpleNamespace( + extract_data_location_from_log=Mock(return_value="/scratch/job"), + run_commands=Mock(return_value=SimpleNamespace(ok=True, stdout="1\n")), + zip_data_on_slurm_server=Mock(return_value=SimpleNamespace(ok=True, stdout="Everything is Ok")), + copy_zip_locally=Mock(), + ) + archive = tmp_path / "123_out.zip" + + def run(): + return namespace["extract_slurm_results_zip"]( + slurm, "123", str(tmp_path), "group", uuid4(), "") + + return SimpleNamespace(run=run, archive=archive, move=move, slurm=slurm) + + +@pytest.mark.parametrize("payload", [b"II*\x00" + bytes(6144), random.Random(0).randbytes(2048)], + ids=["compressed-small", "larger"]) +def test_valid_result_archive_is_accepted_regardless_of_compressed_size(retrieve, payload): + with zipfile.ZipFile(retrieve.archive, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("result.tif", payload) + if payload.startswith(b"II*"): + assert retrieve.archive.stat().st_size < 1024 + result = retrieve.run() + assert Path(result[2]) == retrieve.archive + retrieve.move.assert_called_once() + assert Path(retrieve.move.call_args.args[-1]) == retrieve.archive + with zipfile.ZipFile(retrieve.archive) as archive: + assert archive.read("result.tif") == payload + + +@pytest.mark.parametrize("contents,message", [ + ("empty", "contains no files"), + ("directories", "only directories"), + ("corrupt", "not a valid zip"), + ("missing", "not found"), +]) +def test_unusable_archive_never_reaches_permanent_storage(retrieve, contents, message): + if contents in ("empty", "directories"): + with zipfile.ZipFile(retrieve.archive, "w") as archive: + if contents == "directories": + archive.writestr("empty/", b"") + elif contents == "corrupt": + retrieve.archive.write_bytes(b"not a ZIP archive") + with pytest.raises(RuntimeError, match=message): + retrieve.run() + retrieve.move.assert_not_called() diff --git a/tests/test_shallow_plate_reanalysis.py b/tests/test_shallow_plate_reanalysis.py new file mode 100644 index 0000000..0569f00 --- /dev/null +++ b/tests/test_shallow_plate_reanalysis.py @@ -0,0 +1,94 @@ +"""Use the published reconstruction return type, including its real manifest.""" +import ast +import logging +import os +from pathlib import Path +from types import SimpleNamespace +from uuid import uuid4 + +import pytest +from biomero_schema.zarr import ( + CanonicalPlateImage, CanonicalPlateSource, CanonicalZarrSource, ManagedZarrNode, + PixelIdentity, ShallowBindings, ShallowCollection, ShallowImageBinding, + ShallowImageNode, ShallowLabelBinding, ShallowLabelNode, ShallowManifest, + ShallowPlateReference, ZarrLabelComponent, +) +from biomero_shallower.result_zarr import MaterializedShallowResult + +ROOT = Path(os.environ.get("BIOMERO_SCRIPTS_ROOT", Path(__file__).parents[1])) +SCRIPT = ROOT / "_data" / "_SLURM_Image_Transfer.py" +pytestmark = pytest.mark.skipif( + 'getattr(materialized, "manifest", None)' not in SCRIPT.read_text(encoding="utf-8") + and os.getenv("BIOMERO_TEST_PLATE_REANALYSIS") != "1", + reason="selected source predates schema-2 Plate reconstruction support", +) + + +def fixture(profile, labels): + nodes, bindings, label_nodes, label_bindings, managed = [], [], [], [], [] + for i, path in enumerate(["A/1/0", "A/2/0"]): + ident = PixelIdentity(node_path=path, role="image", iscc_code="ISCC:KPIXEL", + data_code="ISCC:GDATA", instance_code="ISCC:IINSTANCE", + tool_version="0.2.0", imagewalk_revision="test", + shape=(1, 1, 1, 8, 8), dtype="uint16", axes=("t", "c", "z", "y", "x")) + source = CanonicalZarrSource(storage_root="data", relative_path="source.zarr", + node_path=path, source_object_type="Plate", source_object_id=9, + source_generation=1, interchange_profile=profile, pixel_identity=ident, + pixel_identity_origin="canonical-bootstrap", canonical_pixel_verified=True) + nodes.append(ShallowImageNode(node_id=f"i{i}", name=f"field{i}", node_path=path)) + bindings.append(ShallowImageBinding(node_id=f"i{i}", source=source, returned_pixel_identity=ident)) + if labels: + label_path=path+"/labels/cells" + component=ZarrLabelComponent(logical_node_path=label_path, + pixel_identity=ident.model_copy(update={"node_path":label_path,"role":"label"}), + source=ManagedZarrNode(storage_root="data",relative_path="result.zarr",node_path=label_path)) + managed.append(component) + label_nodes.append(ShallowLabelNode(node_id=f"l{i}",name=f"cells{i}",node_path=label_path,source_image_id=f"i{i}")) + label_bindings.append(ShallowLabelBinding(node_id=f"l{i}",component=component)) + manifest=ShallowManifest(workflow_id=uuid4(),transfer_artifact="plate.zarr",interchange_profile=profile, + collection=ShallowCollection(name="Plate",images=tuple(nodes),labels=tuple(label_nodes)), + bindings=ShallowBindings(images=tuple(bindings),labels=tuple(label_bindings))) + reference=ShallowPlateReference(storage_root="data",relative_path="result.zarr", + workflow_id=manifest.workflow_id,transfer_artifact="plate.zarr",source_object_id=9, + source_generation=1,image_node_count=2,interchange_profile=profile) + return manifest,reference,tuple(managed) + + +def load_save(reference, materialized): + names={"save_as_zarr","canonical_plate_source_from_collection"} + tree=ast.parse(SCRIPT.read_text(encoding="utf-8")) + body=[n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name in names] + namespace=dict(os=os,Path=Path,ShallowPlateReference=ShallowPlateReference, + CanonicalPlateImage=CanonicalPlateImage,CanonicalPlateSource=CanonicalPlateSource, + logger=logging.getLogger(__name__),log=lambda _:None,SHALLOW_ZARR_RESTORE_AVAILABLE=True, + get_shallow_plate_reference=lambda _:reference,materialize_shallow_zarr=lambda *args:materialized, + constants=SimpleNamespace(transfer=SimpleNamespace(DATA_TYPE_IMAGE="Image",DATA_TYPE_PLATE="Plate"))) + exec(compile(ast.Module(body=body,type_ignores=[]),str(SCRIPT),"exec"),namespace) + return namespace["save_as_zarr"] + + +@pytest.mark.parametrize("profile",["ngff-0.4-zarr-v2","ngff-0.5-zarr-v3"]) +@pytest.mark.parametrize("labels",[False,True]) +def test_plate_reanalysis_uses_manifest_bindings(tmp_path,profile,labels): + manifest,reference,managed=fixture(profile,labels) + materialized=MaterializedShallowResult(tmp_path/"plate.zarr",manifest,managed) + save=load_save(reference,materialized) + plate=SimpleNamespace(getName=lambda:"plate",getId=lambda:99) + source,artifact,restored_labels=save(None,"session",plate,str(tmp_path),"Plate", + storage_roots={"data":tmp_path},shallow_zarr_storage=False) + assert source.source_object_id==9 + assert source.interchange_profile==profile + assert artifact=="plate.zarr" + assert restored_labels==managed + assert tuple(image.image_node_path for image in source.images)==("A/1/0","A/2/0") + for i,image in enumerate(source.images): + assert image.source==manifest.image_binding(f"i{i}").source + assert image.labels==((managed[i],) if labels else ()) + + +def test_missing_managed_labels_stop_reanalysis(tmp_path): + manifest,reference,managed=fixture("ngff-0.5-zarr-v3",True) + save=load_save(reference,MaterializedShallowResult(tmp_path/"plate.zarr",manifest,managed[:1])) + with pytest.raises(ValueError,match="lost labels"): + save(None,"session",SimpleNamespace(getName=lambda:"plate",getId=lambda:99), + str(tmp_path),"Plate",storage_roots={"data":tmp_path})