From 808aefd4aa5993c923f8272062919ff672e7f9a1 Mon Sep 17 00:00:00 2001 From: David Stuebe <84335963+emfdavid@users.noreply.github.com> Date: Mon, 28 Sep 2026 02:55:30 +0000 Subject: [PATCH] fix(examples): move the IDR microscopy source to livingobjects.ebi.ac.uk EMBL-EBI is retiring the uk1s3.embassy.ebi.ac.uk S3 endpoint. The same idr bucket is served from livingobjects.ebi.ac.uk with the same key layout and anonymous path-style access, so only the endpoint constant changes. Adds a remote test that streams a batch off the real image and checks it against a direct zarr read, so the --source idr path is exercised at all. Closes #97 Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 ++++++++ examples/microscopy/data.py | 2 +- tests/test_microscopy.py | 20 ++++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15d062b..f9db0f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ ## Unreleased +- **The microscopy example's `--source idr` would have stopped working when EMBL-EBI retires + its `uk1s3.embassy.ebi.ac.uk` S3 endpoint (#97).** The IDR image has moved to + `livingobjects.ebi.ac.uk`: same `idr` bucket, same key layout, same path-style anonymous + access, byte-identical metadata. So the fix is the endpoint constant alone. Nothing + exercised the real-IDR path, so the retirement would have gone unnoticed until someone ran + the example. It now has a `remote` test that streams a batch and checks it against a + direct zarr read. + - **`bench/probe_decode.py`'s decode-thread sweep had stopped measuring anything, and printed a clean scaling curve anyway.** The decode pool went process-wide in 0.2.0 — built once by the first dataset in the process, a later different value ignored with a warning — and the diff --git a/examples/microscopy/data.py b/examples/microscopy/data.py index 739340a..5231de4 100644 --- a/examples/microscopy/data.py +++ b/examples/microscopy/data.py @@ -44,7 +44,7 @@ # A public OME-NGFF (zarr v0.1) image in the EMBL-EBI Image Data Repository: a 3D two-channel # confocal stack with an expert instance-segmentation label. Read anonymously off the IDR S3. IDR_URL = "s3://idr/zarr/v0.1/6001240.zarr" -IDR_ENDPOINT = "https://uk1s3.embassy.ebi.ac.uk" +IDR_ENDPOINT = "https://livingobjects.ebi.ac.uk" IDR_RAW = "0" # full-resolution image (T=1, C=2, Z=236, Y=275, X=271), Z-chunk 1 IDR_MASK = "labels/masks/0" # the label (T=1, C=1, Z=236, Y=275, X=271), Z-chunk 30, Y/X tiled diff --git a/tests/test_microscopy.py b/tests/test_microscopy.py index 8234b36..c873a98 100644 --- a/tests/test_microscopy.py +++ b/tests/test_microscopy.py @@ -8,12 +8,15 @@ from __future__ import annotations +import numpy as np import pytest +import zarr from examples.microscopy.data import ( IDR_MASK, IDR_RAW, SAMPLE_AXIS, + _idr_store, inputs_and_targets, iou, make_cells_store, @@ -70,3 +73,20 @@ def test_torch_beats_otsu(cells_store) -> None: model_iou, otsu_iou = train(segmentation_dataset(obstore_store(cells_store)), epochs=12) assert model_iou > otsu_iou + + +@pytest.mark.remote +def test_idr_store_streams_the_real_image() -> None: + """The ``--source idr`` path reads the live public IDR image, and a batch matches zarr.""" + store = _idr_store() + geoms = open_geometries(store, variables=[IDR_RAW, IDR_MASK], sample_axis=SAMPLE_AXIS) + assert geoms[IDR_RAW].n_samples == geoms[IDR_MASK].n_samples == 236 + assert geoms[IDR_RAW].sample_chunk_size == 1 + assert geoms[IDR_MASK].sample_chunk_size == 30 + + ds = segmentation_dataset(store, sample_range=(0, 30), batch_size=4, shuffle=False) + batch = next(iter(ds.all)) + z = batch.sample_indices + group = zarr.open_group(store=store, mode="r") + np.testing.assert_array_equal(batch.arrays["raw"], np.moveaxis(group[IDR_RAW][:, :, z], 2, 0)) + np.testing.assert_array_equal(batch.arrays["mask"], np.moveaxis(group[IDR_MASK][:, :, z], 2, 0))