diff --git a/README.md b/README.md index d925e59..c6c1f99 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,13 @@ missing or false feature flags leave them disabled. Within enabled shallow storage, remote normalization is preferred; set `BIOMERO_REMOTE_SHALLOW_ZARR=false` to use importer-side normalization. NL-BIOMERO's demo enables these features. +Canonical indexing records the backing store's actual OME-Zarr profile: 0.4/v2 +or 0.5/v3. Existing v2 stores are reused as v2, and importing an existing v3 +Image or Plate does not require conversion to v2. V3 indexing requires the +generic semantic-guard API from the matching v3-capable shallower/importer; +older installations retain v2-only indexing. Mixed-format Plate inventories +are rejected rather than recorded under an incorrect profile. + Result file attachments use the existing on-disk paths. Workflow-qualified attachment names are stored in OMERO metadata, so in-place image registrations and attachment links remain valid. Directory-backed Zarr stores are available diff --git a/__workflows/SLURM_CellPose_Segmentation.py b/__workflows/SLURM_CellPose_Segmentation.py index 9c1468e..3ff827d 100644 --- a/__workflows/SLURM_CellPose_Segmentation.py +++ b/__workflows/SLURM_CellPose_Segmentation.py @@ -51,7 +51,7 @@ import logging # Version constant for easy version management -VERSION = "2.9.0" +VERSION = "2.9.0b15" logger = logging.getLogger(__name__) diff --git a/__workflows/SLURM_Run_Workflow.py b/__workflows/SLURM_Run_Workflow.py index 53d496b..6c341de 100644 --- a/__workflows/SLURM_Run_Workflow.py +++ b/__workflows/SLURM_Run_Workflow.py @@ -102,7 +102,7 @@ constants.workflow.OUTPUT_CSV_TABLE, constants.workflow.OUTPUT_ATTACH_FILE_OUTPUTS, constants.workflow.OUTPUT_CREATE_ROIS] -VERSION = "2.9.0" +VERSION = "2.9.0b15" CANONICAL_INPUTS_OUTPUT = "Canonical_Inputs" diff --git a/__workflows/SLURM_Run_Workflow_Batched.py b/__workflows/SLURM_Run_Workflow_Batched.py index 8ffd502..4d9f686 100644 --- a/__workflows/SLURM_Run_Workflow_Batched.py +++ b/__workflows/SLURM_Run_Workflow_Batched.py @@ -73,7 +73,7 @@ constants.workflow.OUTPUT_CREATE_ROIS] # Version constant for easy version management -VERSION = "2.9.0" +VERSION = "2.9.0b15" # Batched runs are queued the same way single runs are: this script resolves # the batches and records them in a launcher task, and the supervisor spawns diff --git a/_data/SLURM_Get_Results.py b/_data/SLURM_Get_Results.py index b410b59..81b1943 100644 --- a/_data/SLURM_Get_Results.py +++ b/_data/SLURM_Get_Results.py @@ -78,7 +78,7 @@ from omero_metadata.populate import ParsingContext # Version constant for easy version management -VERSION = "2.9.0" +VERSION = "2.9.0b15" def get_images_by_ids(conn, image_ids): diff --git a/_data/SLURM_Get_Update.py b/_data/SLURM_Get_Update.py index c1f73de..a460c81 100644 --- a/_data/SLURM_Get_Update.py +++ b/_data/SLURM_Get_Update.py @@ -54,7 +54,7 @@ import sys # Version constant for easy version management -VERSION = "2.9.0" +VERSION = "2.9.0b15" logger = logging.getLogger(__name__) diff --git a/_data/SLURM_Import_Results.py b/_data/SLURM_Import_Results.py index d63e735..3043455 100644 --- a/_data/SLURM_Import_Results.py +++ b/_data/SLURM_Import_Results.py @@ -318,7 +318,7 @@ def _mask_url(url): return url # Fallback function if import fails ) # Version constant for easy version management -VERSION = "2.9.0" +VERSION = "2.9.0b15" def load_canonical_input_snapshot(slurm_client, workflow_id): diff --git a/_data/SLURM_Remote_Conversion.py b/_data/SLURM_Remote_Conversion.py index 8ea3cbf..ba189ed 100644 --- a/_data/SLURM_Remote_Conversion.py +++ b/_data/SLURM_Remote_Conversion.py @@ -50,7 +50,7 @@ CONV_OPTIONS_TARGET = ['tiff', 'zarr'] # Version constant for easy version management -VERSION = "2.9.0" +VERSION = "2.9.0b15" def resolve_log_fallback_target(client, conn): diff --git a/_data/_SLURM_File_Transfer.py b/_data/_SLURM_File_Transfer.py index d000ee5..d491b9f 100644 --- a/_data/_SLURM_File_Transfer.py +++ b/_data/_SLURM_File_Transfer.py @@ -59,7 +59,7 @@ logger = logging.getLogger(__name__) -VERSION = "2.9.0" +VERSION = "2.9.0b15" # --------------------------------------------------------------------------- diff --git a/_data/_SLURM_Image_Transfer.py b/_data/_SLURM_Image_Transfer.py index 38919ba..a8669a8 100644 --- a/_data/_SLURM_Image_Transfer.py +++ b/_data/_SLURM_Image_Transfer.py @@ -128,6 +128,11 @@ pixel_identities_match, read_zarr_v2_semantic_guard, ) + try: + from biomero_importer.utils.pixel_identity import read_zarr_semantic_guard + except ImportError: + # Older deployments retain their v2-only behavior until upgraded. + read_zarr_semantic_guard = read_zarr_v2_semantic_guard from biomero_importer.utils.result_zarr import ( discover_ngff_nodes, ) @@ -140,7 +145,7 @@ ) # Version constant for easy version management -VERSION = "2.9.0" +VERSION = "2.9.0b15" BIOMERO_CONFIG_FILE = os.getenv( "OMERO_BIOMERO_CONFIG_FILE", os.getenv("BIOMERO_CONFIG_FILE", "/opt/omero/server/biomero-config.json"), @@ -684,6 +689,18 @@ def discover_canonical_inputs(objects, object_type): return sources, tuple(canonical_inputs) if complete else () +def canonical_interchange_profile(guard): + """Record the supported format actually backing this canonical node.""" + profiles = { + ("0.4", 2): "ngff-0.4-zarr-v2", + ("0.5", 3): "ngff-0.5-zarr-v3", + } + try: + return profiles[(guard.ngff_version, guard.zarr_format)] + except KeyError as exc: + raise ValueError("Unsupported canonical OME-Zarr format") from exc + + def discover_canonical_label_components( source, storage_roots, @@ -703,7 +720,7 @@ def discover_canonical_label_components( or node.parent_image_node_path != source.node_path ): continue - guard = read_zarr_v2_semantic_guard(root, node.node_path) + guard = read_zarr_semantic_guard(root, node.node_path) identity = provider.generate( root, node_path=node.node_path, @@ -761,7 +778,8 @@ def build_canonical_plate_source( raise ValueError("Identity workers must be a positive integer") def generate_identity(node): - guard = read_zarr_v2_semantic_guard(root, node.node_path) + guard = read_zarr_semantic_guard(root, node.node_path) + profile = canonical_interchange_profile(guard) logger.info( "Calculating ISCC-BIO pixel identity for Plate %s %s node %s", plate.getId(), @@ -787,9 +805,10 @@ def generate_identity(node): identity.data_code, identity.instance_code, ) - return node.node_path, identity + return node.node_path, identity, profile identities = {} + profiles = set() last_keepalive = monotonic() with ThreadPoolExecutor( max_workers=min(identity_workers, len(nodes)), @@ -797,8 +816,9 @@ def generate_identity(node): ) as executor: # map preserves discovery order in the canonical manifest even though # the identity work and its diagnostic logging happen concurrently. - for node_path, identity in executor.map(generate_identity, nodes): + for node_path, identity, profile in executor.map(generate_identity, nodes): identities[node_path] = identity + profiles.add(profile) if ( keepalive is not None and monotonic() - last_keepalive >= keepalive_interval @@ -817,6 +837,9 @@ def generate_identity(node): "Lost the OMERO connection while indexing the canonical Plate Zarr" ) + if len(profiles) != 1: + raise ValueError("Canonical Plate image/label nodes use different Zarr formats") + profile = profiles.pop() relative_path = Path(relative_path).as_posix() images = [] for image_node in image_nodes: @@ -827,7 +850,7 @@ def generate_identity(node): source_object_type="Plate", source_object_id=int(plate.getId()), source_generation=source_generation, - interchange_profile="ngff-0.4-zarr-v2", + interchange_profile=profile, pixel_identity=identities[image_node.node_path], pixel_identity_origin="canonical-bootstrap", canonical_pixel_verified=False, @@ -858,7 +881,7 @@ def generate_identity(node): relative_path=relative_path, source_object_id=int(plate.getId()), source_generation=source_generation, - interchange_profile="ngff-0.4-zarr-v2", + interchange_profile=profile, images=tuple(images), ) @@ -1307,7 +1330,7 @@ def promote_exported_image_zarr( image, storage_roots) source_directory = derive_canonical_source_directory(image, storage_root) semantic_guard_reader = ( - semantic_guard_reader or read_zarr_v2_semantic_guard + semantic_guard_reader or read_zarr_semantic_guard ) guard = semantic_guard_reader(export_path, ".") validate_omero_image_semantics(image, guard) @@ -1380,6 +1403,7 @@ def promote_exported_image_zarr( original_identity=original_identity, exported_identity=exported_identity, pixel_identity_origin="omero-pixels", + interchange_profile=canonical_interchange_profile(guard), ) logger.info( "ISCC-BIO verification matched OMERO Pixels and exported Zarr for " @@ -1428,7 +1452,7 @@ def index_existing_image_zarr( storage_roots, ) semantic_guard_reader = ( - semantic_guard_reader or read_zarr_v2_semantic_guard + semantic_guard_reader or read_zarr_semantic_guard ) guard = semantic_guard_reader(existing_path, ".") validate_omero_image_semantics(image, guard) @@ -1466,7 +1490,7 @@ def index_existing_image_zarr( source_object_id=int(image.getId()), source_generation=source_generation, node_path=".", - interchange_profile="ngff-0.4-zarr-v2", + interchange_profile=canonical_interchange_profile(guard), pixel_identity=existing_identity, pixel_identity_origin="canonical-bootstrap", canonical_pixel_verified=True, diff --git a/admin/BIOMERO_Migrate_Shallow_Storage.py b/admin/BIOMERO_Migrate_Shallow_Storage.py index 89cb02d..fcc4e7b 100644 --- a/admin/BIOMERO_Migrate_Shallow_Storage.py +++ b/admin/BIOMERO_Migrate_Shallow_Storage.py @@ -47,7 +47,7 @@ logger = logging.getLogger(__name__) -VERSION = "2.9.0" +VERSION = "2.9.0b15" OBJECT_TYPE = "Object Type" OBJECT_IDS = "Object IDs (optional)" diff --git a/admin/Example_Minimal_Slurm_Script.py b/admin/Example_Minimal_Slurm_Script.py index edc2992..863d63f 100644 --- a/admin/Example_Minimal_Slurm_Script.py +++ b/admin/Example_Minimal_Slurm_Script.py @@ -66,7 +66,7 @@ def _check_command_safety(cmd: str) -> str | None: _SCMD = "Linux_Command" _DEFAULT_SCMD = "ls -la" -VERSION = "2.9.0" +VERSION = "2.9.0b15" def runScript(): diff --git a/admin/SLURM_Cownary.py b/admin/SLURM_Cownary.py index 18d8337..0b756b1 100644 --- a/admin/SLURM_Cownary.py +++ b/admin/SLURM_Cownary.py @@ -21,7 +21,7 @@ logger = logging.getLogger(__name__) -VERSION = "2.9.0" +VERSION = "2.9.0b15" _LOLCOW_IMAGE = "docker://godlovedc/lolcow" _COWNARY_SBATCH_COMMAND = ( diff --git a/admin/SLURM_Init_environment.py b/admin/SLURM_Init_environment.py index bda83c7..84d201e 100644 --- a/admin/SLURM_Init_environment.py +++ b/admin/SLURM_Init_environment.py @@ -50,7 +50,7 @@ from queue import Queue, Empty logger = logging.getLogger(__name__) -VERSION = "2.9.0" +VERSION = "2.9.0b15" def format_metadata_summary(report): diff --git a/admin/SLURM_check_setup.py b/admin/SLURM_check_setup.py index e0a7796..2169493 100644 --- a/admin/SLURM_check_setup.py +++ b/admin/SLURM_check_setup.py @@ -60,7 +60,7 @@ import pkg_resources # Version constant for easy version management -VERSION = "2.9.0" +VERSION = "2.9.0b15" logger = logging.getLogger(__name__) diff --git a/admin/Tail_logs.py b/admin/Tail_logs.py index 9bb9324..8433e5b 100755 --- a/admin/Tail_logs.py +++ b/admin/Tail_logs.py @@ -26,7 +26,7 @@ _DEFAULT_TAIL_LINES = 5000 # Version constant for easy version management -VERSION = "2.9.0" +VERSION = "2.9.0b15" logger = logging.getLogger(__name__)