Skip to content

Keep TIFF workflow inputs compatible while defaulting fresh Zarr exports to v3 #48

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
TorecLuik merged 1 commit into from
Oct 27, 2026
Merged

Keep TIFF workflow inputs compatible while defaulting fresh Zarr exports to v3 #48

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,20 @@ generic semantic-guard API from the matching v3-capable shallower/importer;
older installations retain v2-only indexing. Mixed-format Plate inventories
are rejected rather than recorded under an incorrect profile.

Fresh exports for Zarr-consuming workflows default to OME-NGFF 0.5 / Zarr v3;
an explicit 0.4 choice remains available. This is an export preference, not a
request to rewrite existing managed stores: native Zarr workflows reuse v2 and
v3 sources in their existing format.

TIFF-consuming workflows keep using the legacy HPC converter. Their transfer
route requires actual OME-NGFF 0.4 / Zarr v2 material: compatible v2 sources are
reused, while v3 sources are exported temporarily from the selected OMERO
pixels as v2. Shallow selections also export their selected pixels, including
label views, instead of reconstructing a different image. These temporary
exports are not indexed or promoted to canonical storage; the original store
and its annotations remain unchanged. Deploy the matching OMERO.biomero web
change after these scripts so the web default and transfer safeguards agree.

Result file attachments use the existing on-disk paths. Workflow-qualified
attachment names are stored in OMERO metadata, so in-place image registrations
and attachment links remain valid. Directory-backed Zarr stores are available
Expand Down
16 changes: 9 additions & 7 deletions __workflows/SLURM_Run_Workflow.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,11 +293,11 @@ def resolve_pipeline_inputs(client, conn, slurmClient, params) -> dict:
if use_zarr_format is None:
use_zarr_format = unwrap(client.getInput(
constants.workflow.USE_ZARR_FORMAT))
ome_zarr_version = params.get("ome_zarr_version")
if not ome_zarr_version:
ome_zarr_version = (unwrap(client.getInput(
constants.transfer.OME_VERSION)) if use_zarr_format
else constants.transfer.OME_ZARR_VERSION_0_4)
ome_zarr_version = (
params.get("ome_zarr_version")
or unwrap(client.getInput(constants.transfer.OME_VERSION))
or constants.transfer.OME_ZARR_VERSION_0_5
) if use_zarr_format else constants.transfer.OME_ZARR_VERSION_0_4
return {
"workflows": workflows,
"selected_output": selected_output,
Expand Down Expand Up @@ -655,7 +655,7 @@ def runScript():
omscripts.String(
constants.transfer.OME_VERSION, grouping="01.4.1",
description="Ome-zarr version", values=ome_zarr_versions,
default=constants.transfer.OME_ZARR_VERSION_0_4),
default=constants.transfer.OME_ZARR_VERSION_0_5),
omscripts.Bool(constants.workflow.SELECT_IMPORT,
optional=False,
grouping="02",
Expand Down Expand Up @@ -940,7 +940,9 @@ def runScript():
# Get ZARR format preference
use_zarr_format = unwrap(client.getInput(constants.workflow.USE_ZARR_FORMAT))
if use_zarr_format:
ome_zarr_version = unwrap(client.getInput(constants.transfer.OME_VERSION))
ome_zarr_version = (
unwrap(client.getInput(constants.transfer.OME_VERSION))
or constants.transfer.OME_ZARR_VERSION_0_5)
else:
ome_zarr_version = constants.transfer.OME_ZARR_VERSION_0_4 # backward compatible

Expand Down
2 changes: 1 addition & 1 deletion __workflows/SLURM_Run_Workflow_Batched.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ def runScript():
omscripts.String(
constants.transfer.OME_VERSION, grouping="01.5.1",
description="Ome-zarr version", values=ome_zarr_versions,
default=constants.transfer.OME_ZARR_VERSION_0_4),
default=constants.transfer.OME_ZARR_VERSION_0_5),
omscripts.Bool(constants.workflow.SELECT_IMPORT,
optional=False,
grouping="02",
Expand Down
56 changes: 38 additions & 18 deletions _data/_SLURM_Image_Transfer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2203,6 +2203,23 @@ def build_zarr_export_error(object, data_type, stderr):
return " ".join(context)


def is_legacy_tiff_conversion_source(source_path):
"""Check storage metadata without reading pixels or requiring the importer.

The legacy HPC helper uses Zarr 2. Reuse a v2 store, but export selected
OMERO pixels for v3 or unrecognized stores instead of sending unreadable
material to that helper. An ambiguous mixed root is not reusable here.
"""
root = Path(source_path)
if (root / "zarr.json").exists():
return False
try:
metadata = json.loads((root / ".zgroup").read_text(encoding="utf-8"))
return isinstance(metadata, dict) and metadata.get("zarr_format") == 2
except (OSError, ValueError):
return False


def save_as_zarr(
conn,
suuid,
Expand All @@ -2229,6 +2246,12 @@ def save_as_zarr(
Raises:
ValueError: If unsupported data_type is provided.
"""
conversion_material = not reconstruct_shallow_zarr
if conversion_material:
# This is a required format for the legacy helper, not just an export
# preference. Reused sources are checked below as well.
ome_zarr_version = constants.transfer.OME_ZARR_VERSION_0_4

extension = "zarr"
name = os.path.basename(object.getName())
img_name = "%s.%s" % (name, extension)
Expand All @@ -2246,18 +2269,6 @@ def save_as_zarr(
shallow_reference = get_shallow_reference(object)
elif data_type == constants.transfer.DATA_TYPE_PLATE:
shallow_reference = get_shallow_plate_reference(object)
if (
shallow_reference is not None
and data_type == constants.transfer.DATA_TYPE_PLATE
and not reconstruct_shallow_zarr
):
logger.warning(
"Ignoring selected-pixels transfer mode for shallow Plate %s; "
"Plate inputs always require full Zarr reconstruction",
object.getId(),
)
reconstruct_shallow_zarr = True

if (
shallow_reference is not None
and reconstruct_shallow_zarr
Expand All @@ -2280,10 +2291,10 @@ def save_as_zarr(
)
if selected_pixel_export:
logger.info(
"Bypassing full shallow reconstruction for Image %s because the "
"Bypassing full shallow reconstruction for %s %s because the "
"temporary Zarr will be converted to TIFF; exporting the exact "
"OMERO PixelBuffer and excluding it from canonical promotion",
object.getId(),
data_type, object.getId(),
)
log(" Exporting selected OMERO pixels as TIFF conversion material")
canonical_source = None
Expand Down Expand Up @@ -2330,14 +2341,23 @@ def save_as_zarr(
if not selected_pixel_export:
source_path = select_zarr_source_path(
object, canonical_source, storage_roots or {})
if (conversion_material and source_path is not None
and not is_legacy_tiff_conversion_source(source_path)):
logger.info(
"Exporting temporary NGFF 0.4 / Zarr v2 material for %s %s; "
"the managed source at %s and its canonical registration remain "
"unchanged", data_type, object.getId(), source_path)
source_path = None
canonical_source = None
if source_path is not None:
if shallow_zarr_storage and canonical_source is not None:
if (shallow_zarr_storage and canonical_source is not None
and not conversion_material):
canonical_source = upgrade_reused_canonical(
conn, object, canonical_source, source_path)
if (
shallow_zarr_storage
and canonical_source is None
and not selected_pixel_export
and not conversion_material
):
try:
if data_type == constants.transfer.DATA_TYPE_PLATE:
Expand Down Expand Up @@ -2469,7 +2489,7 @@ def save_as_zarr(
if (
shallow_zarr_storage
and canonical_source is None
and not selected_pixel_export
and not conversion_material
):
try:
if data_type == constants.transfer.DATA_TYPE_PLATE:
Expand Down Expand Up @@ -3168,7 +3188,7 @@ def run_script():
scripts.String(
constants.transfer.OME_VERSION, grouping="5.2",
description="Ome-zarr version", values=ome_zarr_versions,
default=constants.transfer.OME_ZARR_VERSION_0_4),
default=constants.transfer.OME_ZARR_VERSION_0_5),

scripts.Bool(
constants.transfer.RECONSTRUCT_SHALLOW_ZARR,
Expand Down
Loading