fix(examples): move the IDR microscopy source to livingobjects.ebi.ac.uk

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## Unreleased
 
+- **The microscopy example's `--source idr` would have stopped working when EMBL-EBI retires
+  its `uk1s3.embassy.ebi.ac.uk` S3 endpoint (#97).** The IDR image has moved to
+  `livingobjects.ebi.ac.uk`: same `idr` bucket, same key layout, same path-style anonymous
+  access, byte-identical metadata. So the fix is the endpoint constant alone. Nothing
+  exercised the real-IDR path, so the retirement would have gone unnoticed until someone ran
+  the example. It now has a `remote` test that streams a batch and checks it against a
+  direct zarr read.
+
 - **`bench/probe_decode.py`'s decode-thread sweep had stopped measuring anything, and printed
   a clean scaling curve anyway.** The decode pool went process-wide in 0.2.0 — built once by
   the first dataset in the process, a later different value ignored with a warning — and the

examples/microscopy/data.py

@@ -44,7 +44,7 @@
 # A public OME-NGFF (zarr v0.1) image in the EMBL-EBI Image Data Repository: a 3D two-channel
 # confocal stack with an expert instance-segmentation label. Read anonymously off the IDR S3.
 IDR_URL = "s3://idr/zarr/v0.1/6001240.zarr"
-IDR_ENDPOINT = "https://uk1s3.embassy.ebi.ac.uk"
+IDR_ENDPOINT = "https://livingobjects.ebi.ac.uk"
 IDR_RAW = "0"  # full-resolution image (T=1, C=2, Z=236, Y=275, X=271), Z-chunk 1
 IDR_MASK = "labels/masks/0"  # the label (T=1, C=1, Z=236, Y=275, X=271), Z-chunk 30, Y/X tiled
 

tests/test_microscopy.py

@@ -8,12 +8,15 @@
 
 from __future__ import annotations
 
+import numpy as np
 import pytest
+import zarr
 
 from examples.microscopy.data import (
     IDR_MASK,
     IDR_RAW,
     SAMPLE_AXIS,
+    _idr_store,
     inputs_and_targets,
     iou,
     make_cells_store,
@@ -70,3 +73,20 @@ def test_torch_beats_otsu(cells_store) -> None:
 
     model_iou, otsu_iou = train(segmentation_dataset(obstore_store(cells_store)), epochs=12)
     assert model_iou > otsu_iou
+
+
+@pytest.mark.remote
+def test_idr_store_streams_the_real_image() -> None:
+    """The ``--source idr`` path reads the live public IDR image, and a batch matches zarr."""
+    store = _idr_store()
+    geoms = open_geometries(store, variables=[IDR_RAW, IDR_MASK], sample_axis=SAMPLE_AXIS)
+    assert geoms[IDR_RAW].n_samples == geoms[IDR_MASK].n_samples == 236
+    assert geoms[IDR_RAW].sample_chunk_size == 1
+    assert geoms[IDR_MASK].sample_chunk_size == 30
+
+    ds = segmentation_dataset(store, sample_range=(0, 30), batch_size=4, shuffle=False)
+    batch = next(iter(ds.all))
+    z = batch.sample_indices
+    group = zarr.open_group(store=store, mode="r")
+    np.testing.assert_array_equal(batch.arrays["raw"], np.moveaxis(group[IDR_RAW][:, :, z], 2, 0))
+    np.testing.assert_array_equal(batch.arrays["mask"], np.moveaxis(group[IDR_MASK][:, :, z], 2, 0))