diff --git a/.github/workflows/run_pytests.yaml b/.github/workflows/run_pytests.yaml index ea0c9aa..b8b78d7 100644 --- a/.github/workflows/run_pytests.yaml +++ b/.github/workflows/run_pytests.yaml @@ -18,17 +18,15 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.13"] + python-version: ["3.14"] include: # we'll test the python support on ubuntu + - os: ubuntu-latest + python-version: "3.13" - os: ubuntu-latest python-version: "3.12" - os: ubuntu-latest python-version: "3.11" - - os: ubuntu-latest - python-version: "3.10" - - os: ubuntu-latest - python-version: "3.9" steps: - name: Checkout TiffSlide uses: actions/checkout@v3 @@ -55,14 +53,14 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: 3.11 - name: Install dependencies run: | python -m pip install --upgrade pip python -m pip install -e .[dev] - name: Run mypy run: | - mypy --install-types --non-interactive --python-version=3.9 tiffslide + mypy --install-types --non-interactive --python-version=3.11 tiffslide # DEPLOY TIFFSLIDE TO TEST.PYPI ON SUCCESS testdeploy: @@ -77,7 +75,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: 3.11 - name: Install dependencies run: | python -m pip install --upgrade pip @@ -114,7 +112,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: 3.11 - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/docs/generate_benchmark_plots.py b/docs/generate_benchmark_plots.py index 6836dce..eb23f6e 100644 --- a/docs/generate_benchmark_plots.py +++ b/docs/generate_benchmark_plots.py @@ -46,7 +46,7 @@ def run_pytest_benchmarks(force: bool) -> None: df = pd.DataFrame.from_records(records) for test_name, ptdf in df.groupby("test_name"): - ft_ax_order = [ + ft_ax_order_all = [ "svs", "svs_jp2k", "generic", @@ -54,8 +54,10 @@ def run_pytest_benchmarks(force: bool) -> None: "leica", "ventana", ] - assert set(ptdf["file_type"].unique()) == set(ft_ax_order) - fig, axes = plt.subplots(1, len(ft_ax_order), figsize=(10, 4)) + available = set(ptdf["file_type"].unique()) + ft_ax_order = [ft for ft in ft_ax_order_all if ft in available] + fig, axes = plt.subplots(1, len(ft_ax_order), figsize=(10, 4), squeeze=False) + axes = axes[0] # unwrap row dimension fig.suptitle(test_name, x=0.1, y=0.99) ft_ax_map = {ft: ax for ft, ax in zip(ft_ax_order, axes)} diff --git a/pyproject.toml b/pyproject.toml index a848ea1..cc6a352 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,8 +18,6 @@ classifiers = [ "License :: OSI Approved :: BSD License", "Programming Language :: Python", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", @@ -32,13 +30,13 @@ classifiers = [ "Operating System :: Unix", "Operating System :: MacOS", ] -requires-python = ">=3.9" +requires-python = ">=3.11" dependencies = [ "imagecodecs", - "fsspec>=2023.3.0", + "fsspec>=2024.10.0", "pillow>=9.1.0", - "tifffile>=2023.7.4,<2025.5.21", - "zarr>=2.16.0,<3.0", + "tifffile>=2025.5.21", + "zarr>=3.0,<4.0", "typing_extensions>=4.0", ] dynamic = ["version"] @@ -90,7 +88,7 @@ markers = [ [tool.isort] profile = "black" -py_version = "39" +py_version = "311" force_single_line = true extend_skip = ["setup.py", "docs/source/conf.py"] @@ -107,7 +105,7 @@ exclude_lines = [ ] [tool.mypy] -python_version = "3.9" +python_version = "3.11" check_untyped_defs = true disallow_any_generics = true disallow_incomplete_defs = true diff --git a/tiffslide/_compat.py b/tiffslide/_compat.py index 7040232..2f753a7 100644 --- a/tiffslide/_compat.py +++ b/tiffslide/_compat.py @@ -153,7 +153,14 @@ def size(self) -> int: return self._array.size def aszarr(self) -> zarr.storage.MemoryStore: - return zarr.creation.array(self._array).store + store = zarr.storage.MemoryStore() + zarr.create_array( + store=store, + data=self._array, + chunks=self._array.shape, + zarr_format=2, + ) + return store def asarray(self) -> NDArray[np.uint8]: return self._array diff --git a/tiffslide/_kerchunk.py b/tiffslide/_kerchunk.py index 1d2e34d..dd94deb 100644 --- a/tiffslide/_kerchunk.py +++ b/tiffslide/_kerchunk.py @@ -10,6 +10,7 @@ import json import os import sys +import threading from collections import ChainMap from io import StringIO from typing import TYPE_CHECKING @@ -17,6 +18,8 @@ from typing import TypedDict import fsspec +from fsspec.asyn import get_loop +from fsspec.asyn import sync as _fsspec_sync from imagecodecs.numcodecs import register_codecs from tiffslide.tiffslide import TiffSlide @@ -137,9 +140,10 @@ def from_kerchunk( fs: ReferenceFileSystem = fsspec.filesystem( "reference", fo=kc, + asynchronous=True, **storage_options, ) - zattrs = json.loads(fs.cat_file(".zattrs")) + zattrs = json.loads(_fsspec_sync(get_loop(), fs._cat_file, ".zattrs")) if "tiffslide.spec_version" not in zattrs or "tiffslide.properties" not in zattrs: raise ValueError("") @@ -155,6 +159,8 @@ def from_kerchunk( inst = object.__new__(TiffSlide) inst.__dict__["properties"] = properties inst._tifffile = fs # fixme: ... + inst._zarr_local = threading.local() + inst._zarr_lock = threading.Lock() return inst diff --git a/tiffslide/_zarr.py b/tiffslide/_zarr.py index e264a15..5e9624a 100644 --- a/tiffslide/_zarr.py +++ b/tiffslide/_zarr.py @@ -4,20 +4,29 @@ from __future__ import annotations +import asyncio +import builtins import json -from collections.abc import Iterator +from collections.abc import AsyncIterator +from collections.abc import Iterable from collections.abc import Mapping from typing import TYPE_CHECKING from typing import Any +from typing import Literal import numpy as np import zarr from fsspec.implementations.reference import ReferenceFileSystem from tifffile import TiffFile -from tifffile import ZarrTiffStore -from zarr.storage import BaseStore -from zarr.storage import FSStore -from zarr.storage import KVStore +from tifffile.zarr import ZarrTiffStore +from zarr.abc.store import Store +from zarr.core.buffer import Buffer +from zarr.core.buffer import BufferPrototype +from zarr.core.buffer import default_buffer_prototype +from zarr.core.buffer.cpu import Buffer as CpuBuffer +from zarr.core.sync import sync as _sync +from zarr.storage import FsspecStore +from zarr.storage import MemoryStore from tiffslide._compat import NotTiffFile from tiffslide._types import Point3D @@ -28,6 +37,7 @@ if TYPE_CHECKING: from numpy.typing import DTypeLike from numpy.typing import NDArray + from zarr.abc.store import ByteRequest __all__ = [ @@ -40,59 +50,175 @@ # --- zarr storage classes -------------------------------------------- -class _CompositedStore(Mapping[str, Any]): - """prefix zarr stores to allow mounting them in groups""" +def _store_exists_sync(store: Store, key: str) -> bool: + """synchronous wrapper for store.exists()""" + return bool(_sync(store.exists(key))) + + +def _store_get_sync( + store: Store, + key: str, + prototype: BufferPrototype | None = None, + byte_range: ByteRequest | None = None, +) -> Buffer | None: + """synchronous wrapper for store.get()""" + if prototype is None: + prototype = default_buffer_prototype() + return _sync(store.get(key, prototype, byte_range=byte_range)) + + +def _store_list_sync(store: Store) -> list[str]: + """synchronous wrapper for store.list()""" + + async def _collect() -> list[str]: + return [key async for key in store.list()] + + return list(_sync(_collect())) + + +def _store_list_prefix_sync(store: Store, prefix: str) -> list[str]: + """synchronous wrapper for store.list_prefix()""" + + async def _collect() -> list[str]: + return [key async for key in store.list_prefix(prefix)] + + return list(_sync(_collect())) + + +class _CompositedStore(Store): + """prefix-routing zarr v3 Store that mounts child stores under prefixes""" + + _stores: dict[str, Store] + _base: MemoryStore def __init__( self, - prefixed_stores: Mapping[str, Mapping[str, Any]], + prefixed_stores: Mapping[str, Store], *, zattrs: Mapping[str, Any] | None = None, ) -> None: - grp = zarr.group({}) + super().__init__(read_only=True) + + self._base = MemoryStore() + # create a zarr v2 group in the memory store with optional attributes + _zgroup = json.dumps({"zarr_format": 2}).encode() + _sync(self._base.set(".zgroup", CpuBuffer.from_bytes(_zgroup))) if zattrs: - grp.attrs["tiffslide.series-composition"] = zattrs - self._base = grp.store + _zattrs = json.dumps({"tiffslide.series-composition": zattrs}).encode() + _sync(self._base.set(".zattrs", CpuBuffer.from_bytes(_zattrs))) + self._stores = {} for prefix, store in prefixed_stores.items(): assert not prefix.endswith("/") self._stores[prefix] = store - def __len__(self) -> int: - return sum(map(len, self._stores.values())) + len(self._base) - - def __contains__(self, item: object) -> bool: - if not isinstance(item, str): - return False - - prefix, _, key = item.partition("/") - if key: + async def get( + self, + key: str, + prototype: BufferPrototype, + byte_range: ByteRequest | None = None, + ) -> Buffer | None: + prefix, _, subkey = key.partition("/") + if subkey: try: store = self._stores[prefix] except KeyError: pass else: - return key in store - - return item in self._base or prefix in self._stores - - def __iter__(self) -> Iterator[str]: - for prefix, store in self._stores.items(): - yield from (f"{prefix}/{key}" for key in store.keys()) - yield from self._base.keys() - - def __getitem__(self, item: str) -> Any: - prefix, _, key = item.partition("/") + return await store.get(subkey, prototype, byte_range=byte_range) + return await self._base.get(key, prototype, byte_range=byte_range) - if key: + async def exists(self, key: str) -> bool: + prefix, _, subkey = key.partition("/") + if subkey: try: store = self._stores[prefix] except KeyError: pass else: - return store[key] + return bool(await store.exists(subkey)) + # also return True for bare prefix (it's a "directory") + if not subkey and prefix in self._stores: + return True + return bool(await self._base.exists(key)) - return self._base[item] + async def set(self, key: str, value: Buffer) -> None: + raise ReadOnlyError() + + async def delete(self, key: str) -> None: + raise ReadOnlyError() + + async def list(self) -> AsyncIterator[str]: + for prefix, store in self._stores.items(): + async for key in store.list(): + yield f"{prefix}/{key}" + async for key in self._base.list(): + yield key + + async def list_prefix(self, prefix: str) -> AsyncIterator[str]: + async for key in self.list(): + if key.startswith(prefix): + yield key + + async def list_dir(self, prefix: str) -> AsyncIterator[str]: + if prefix: + store_prefix = prefix.rstrip("/") + if store_prefix in self._stores: + store = self._stores[store_prefix] + async for key in store.list(): + yield f"{store_prefix}/{key}" + return + # check if prefix is a sub-path within a child store + top, _, rest = store_prefix.partition("/") + if top in self._stores: + store = self._stores[top] + async for key in store.list_dir(rest): + yield f"{top}/{key}" + return + # fall back to base store + async for key in self._base.list_dir(prefix): + yield key + # also list child store prefixes as "directories" + if not prefix: + for p in self._stores: + yield p + + @property + def supports_writes(self) -> bool: + return False + + @property + def supports_deletes(self) -> bool: + return False + + @property + def supports_partial_writes(self) -> Literal[False]: + return False + + @property + def supports_listing(self) -> bool: + return True + + async def get_partial_values( + self, + prototype: BufferPrototype, + key_ranges: Iterable[tuple[str, ByteRequest | None]], + ) -> builtins.list[Buffer | None]: + return [ + await self.get(key, prototype, byte_range=byte_range) + for key, byte_range in key_ranges + ] + + def __eq__(self, value: object) -> bool: + return ( + isinstance(value, _CompositedStore) + and self._stores == value._stores + and self._base == value._base + ) + + +class ReadOnlyError(Exception): + """raised when write operations are attempted on a read-only store""" def _get_series_zarr( @@ -100,15 +226,20 @@ def _get_series_zarr( series_idx: int, *, num_decode_threads: int | None = None, -) -> Mapping[str, Any]: +) -> Store: """return a zarr store from the object""" if isinstance(obj, (TiffFile, NotTiffFile)): - zstore = obj.series[series_idx].aszarr(maxworkers=num_decode_threads) # type: ignore + zstore: Store = obj.series[series_idx].aszarr(maxworkers=num_decode_threads) # type: ignore elif isinstance(obj, ReferenceFileSystem): - zstore = FSStore(f"s{series_idx}", fs=obj) + zstore = FsspecStore(fs=obj, path=f"s{series_idx}", read_only=True) else: raise NotImplementedError(f"{type(obj).__name__} unsupported") - return zstore # type: ignore + return zstore + + +def _is_single_array_store(store: Store) -> bool: + """check if the store represents a single zarr array (not a group)""" + return _store_exists_sync(store, ".zarray") def get_zarr_store( @@ -116,7 +247,7 @@ def get_zarr_store( tf: TiffFile | ReferenceFileSystem | None, *, num_decode_threads: int | None = None, -) -> BaseStore: +) -> Store: """return a zarr store Parameters @@ -131,7 +262,7 @@ def get_zarr_store( Returns ------- store: - a zarr store of the tiff + a zarr v3 Store of the tiff """ if tf is None: raise NotImplementedError("support in future versions") @@ -140,33 +271,29 @@ def get_zarr_store( composition: SeriesCompositionInfo | None = properties.get( "tiffslide.series-composition" ) - store: BaseStore + store: Store if composition: - prefixed_stores = {} + prefixed_stores: dict[str, Store] = {} for series_idx in composition["located_series"].keys(): _store = _get_series_zarr( tf, series_idx, num_decode_threads=num_decode_threads ) # encapsulate store as group if tifffile returns a zarr array - if ".zarray" in _store: + if _is_single_array_store(_store): _store = _CompositedStore({"0": _store}) prefixed_stores[str(series_idx)] = _store - _store = _CompositedStore(prefixed_stores, zattrs=composition) - store = KVStore(_store) + store = _CompositedStore(prefixed_stores, zattrs=composition) else: series_idx = properties.get("tiffslide.series-index", 0) _store = _get_series_zarr(tf, series_idx, num_decode_threads=num_decode_threads) # encapsulate store as group if tifffile returns a zarr array - if ".zarray" in _store: - _store = _CompositedStore({"0": _store}) - store = KVStore(_store) - elif isinstance(_store, BaseStore): - store = _store + if _is_single_array_store(_store): + store = _CompositedStore({"0": _store}) else: - store = KVStore(_store) + store = _store return store @@ -177,11 +304,11 @@ def get_zarr_selection( selection: Slice3D, ) -> NDArray[np.int_]: """retrieve the selection of the zarr Group""" - composition: SeriesCompositionInfo = grp.attrs.get("tiffslide.series-composition") + composition: SeriesCompositionInfo | None = grp.attrs.get("tiffslide.series-composition") # type: ignore[assignment] if composition is None: # no composition required, simply retrieve the array - return grp[str(level)][selection] + return grp[str(level)][selection] # type: ignore[no-any-return,return-value,index] else: # we need to composite the array @@ -193,7 +320,7 @@ def get_zarr_selection( located_series = composition["located_series"] for series_idx, level_offsets in located_series.items(): - arr = grp[f"{series_idx}/{level}"] + arr: zarr.Array[Any] = grp[f"{series_idx}/{level}"] # type: ignore[assignment] offset = level_offsets[level] if dtype is None: @@ -201,7 +328,7 @@ def get_zarr_selection( if fill_value is None: fill_value = arr.fill_value - overlap = get_overlap(selection, level_shape, offset, arr.shape) + overlap = get_overlap(selection, level_shape, offset, arr.shape) # type: ignore[arg-type] if overlap is None: continue @@ -221,12 +348,25 @@ def get_zarr_selection( out = np.full(shape, fill_value=fill_value, dtype=dtype) for series_idx, (target, source) in overlaps.items(): - arr = grp[f"{series_idx}/{level}"] + arr = grp[f"{series_idx}/{level}"] # type: ignore[assignment,index] out[target] = arr[source] return out +def _unwrap_to_zarrtiffstore(store: Store) -> ZarrTiffStore: + """navigate the store wrapper chain to find the underlying ZarrTiffStore""" + if isinstance(store, ZarrTiffStore): + return store + + if isinstance(store, _CompositedStore) and {"0"} == set(store._stores): + inner = store._stores["0"] + if isinstance(inner, ZarrTiffStore): + return inner + + raise NotImplementedError(f"store type: {type(store).__name__!r}") + + def get_zarr_chunk_sizes( grp: zarr.Group, *, @@ -239,42 +379,25 @@ def get_zarr_chunk_sizes( """ store = grp.store + tiff_store = _unwrap_to_zarrtiffstore(store) - if not isinstance(store, ZarrTiffStore): - while hasattr(store, "_mutable_mapping"): - # noinspection PyProtectedMember - store = store._mutable_mapping - - # noinspection PyProtectedMember - if isinstance(store, _CompositedStore) and {"0"} == set(store._stores): - # noinspection PyProtectedMember - store = store._stores["0"] - - while hasattr(store, "_mutable_mapping"): - # noinspection PyProtectedMember - store = store._mutable_mapping - - if not isinstance(store, ZarrTiffStore): - raise NotImplementedError(f"store type: {type(store).__name__!r}") - - for key, value in store.items(): - if ".zarray" in key: - levelstr = (key.split("/")[0] + "/") if "/" in key else "" - # skip if not selected level - if levelstr == "" and level != 0: - continue - elif levelstr == "" and level == 0: - break - elif levelstr != f"{level}/": - continue - else: - break + # determine the level prefix and read metadata via zarr v2 .zarray keys + if tiff_store.is_multiscales: + levelstr = f"{level}/" + zarray_key = f"{level}/.zarray" else: + if level != 0: + raise ValueError(f"no matching level: {level}") + levelstr = "" + zarray_key = ".zarray" + + buf = _store_get_sync(tiff_store, zarray_key) + if buf is None: raise ValueError(f"no matching level: {level}") - value = json.loads(value) - shape = value["shape"] - chunks = value["chunks"] + meta = json.loads(buf.to_bytes().decode()) + shape = meta["shape"] + chunks = meta["chunks"] assert len(shape) == len(chunks) if len(shape) not in (2, 3): @@ -286,22 +409,17 @@ def get_zarr_chunk_sizes( # relies on private functionality of ZarrTiffStore, might break at any time try: # noinspection PyProtectedMember - chunkmode = store._chunkmode - # noinspection PyProtectedMember - parse_key = store._parse_key + parse_key = tiff_store._parse_key except AttributeError: raise RuntimeError("probably not supported with your tifffile version") chunk_sizes: NDArray[np.int64] = np.full(chunked, dtype=np.int64, fill_value=-1) - # _index = "" for indices in np.ndindex(*chunked): chunkindex = ".".join(str(index) for index in indices) key = levelstr + chunkindex keyframe, page, _, offset, bytecount = parse_key(key) - # key = levelstr + _index + chunkindex - if page and chunkmode and offset is None: - # offset = page.dataoffsets[0] + if page and offset is None: bytecount = keyframe.nbytes if offset and bytecount: chunk_sizes[indices] = bytecount @@ -395,8 +513,8 @@ def get_overlap( def verify_located_arrays( - shape: Size3D, located_arrays: dict[Point3D, zarr.Array] -) -> tuple[str, Any]: + shape: Size3D, located_arrays: dict[Point3D, zarr.Array[Any]] +) -> tuple[DTypeLike, Any]: """verify located arrays ensures that dtypes and fill_values agree and that @@ -427,12 +545,12 @@ def verify_located_arrays( def get_zarr_depth_and_dtype(grp: zarr.Group, axes: str) -> tuple[int, DTypeLike]: """return the image depth from the zarr group""" if "tiffslide.series-composition" in grp.attrs: - srs = next(iter(grp.attrs["series-composition"]["located_series"])) + srs = next(iter(grp.attrs["series-composition"]["located_series"])) # type: ignore[index,call-overload,arg-type] key = f"{srs}/0" else: key = "0" # -> level - zarray: zarr.core.Array = grp[key] + zarray: zarr.Array[Any] = grp[key] # type: ignore[assignment] if axes == "YXS": depth = zarray.shape[2] diff --git a/tiffslide/deepzoom.py b/tiffslide/deepzoom.py index 725b8f9..b21d227 100644 --- a/tiffslide/deepzoom.py +++ b/tiffslide/deepzoom.py @@ -23,6 +23,9 @@ import fsspec from PIL import Image from PIL import ImageFile +from tifffile import COMPRESSION +from tifffile import PHOTOMETRIC +from tifffile import PLANARCONFIG from tifffile import TIFF from tifffile import TiffFile from tifffile import TiffPage @@ -93,13 +96,13 @@ def __init__( # more assumptions to ensure programmer sanity assert page.compression in { - TIFF.COMPRESSION.JPEG, - TIFF.COMPRESSION.APERIO_JP2000_YCBC, - TIFF.COMPRESSION.JPEG_2000_LOSSY, - TIFF.COMPRESSION.APERIO_JP2000_RGB, + COMPRESSION.JPEG, + COMPRESSION.APERIO_JP2000_YCBC, + COMPRESSION.JPEG_2000_LOSSY, + COMPRESSION.APERIO_JP2000_RGB, } assert page.is_tiled - assert page.planarconfig == TIFF.PLANARCONFIG.CONTIG + assert page.planarconfig == PLANARCONFIG.CONTIG # calculate indices st_length, st_width = page.tilelength, page.tilewidth @@ -114,7 +117,7 @@ def __init__( "image_wh": (im_width, im_length), "offsets": page.dataoffsets, "bytecounts": page.databytecounts, - "requires_rgb_color_fix": page.photometric == TIFF.PHOTOMETRIC.RGB, + "requires_rgb_color_fix": page.photometric == PHOTOMETRIC.RGB, } @property diff --git a/tiffslide/tests/test_benchmark.py b/tiffslide/tests/test_benchmark.py index fdab00e..93c2148 100644 --- a/tiffslide/tests/test_benchmark.py +++ b/tiffslide/tests/test_benchmark.py @@ -18,7 +18,7 @@ "generic": "Generic-TIFF/CMU-1.tiff", "hamamatsu": "Hamamatsu/OS-3.ndpi", "leica": "Leica/Leica-2.scn", - "ventana": "Ventana/OS-2.bif", + # "ventana": "Ventana/OS-2.bif", } if OPENSLIDE_TESTDATA_DIR is None: pytestmark = pytest.mark.skip diff --git a/tiffslide/tests/test_zarr.py b/tiffslide/tests/test_zarr.py index 0932eb5..3bd7fef 100644 --- a/tiffslide/tests/test_zarr.py +++ b/tiffslide/tests/test_zarr.py @@ -1,4 +1,4 @@ -from tifffile import ZarrStore +from tifffile.zarr import ZarrTiffStore from tiffslide import TiffSlide @@ -16,14 +16,16 @@ def test_decode_only_once(wsi_file, monkeypatch): ts = TiffSlide(wsi_file) called_keys = [] - _ZarrStore___getitem__ = ZarrStore.__getitem__ + _ZarrTiffStore_get = ZarrTiffStore.get - def _cnt_getitem(self, key): + async def _cnt_get(self, key, prototype, byte_range=None): nonlocal called_keys called_keys.append(key) - return _ZarrStore___getitem__(self, key) + return await _ZarrTiffStore_get(self, key, prototype, byte_range=byte_range) - monkeypatch.setattr(ZarrStore, "__getitem__", _cnt_getitem) + monkeypatch.setattr(ZarrTiffStore, "get", _cnt_get) _ = ts.read_region((0, 0), 0, (1, 1), as_array=True) - assert len(called_keys) == len(set(called_keys)) + # filter out metadata keys — only check chunk data keys + data_keys = [k for k in called_keys if not k.startswith(".z")] + assert len(data_keys) == len(set(data_keys)) diff --git a/tiffslide/tiffslide.py b/tiffslide/tiffslide.py index f960cf0..ae5bd22 100644 --- a/tiffslide/tiffslide.py +++ b/tiffslide/tiffslide.py @@ -3,7 +3,7 @@ import io import math import os.path -import sys +import threading from collections import defaultdict from collections.abc import Iterator from collections.abc import Mapping @@ -114,6 +114,10 @@ def __init__( _cls=TiffFile, ) + # per-thread zarr_group cache + self._zarr_local: threading.local = threading.local() + self._zarr_lock: threading.Lock = threading.Lock() + @property def ts_tifffile(self) -> TiffFile: """get the underlying tifffile instance""" @@ -141,12 +145,12 @@ def __exit__( def close(self) -> None: try: - grp = self.__dict__.pop("zarr_group") - except KeyError: + grp = self._zarr_local.zarr_group + except AttributeError: pass else: try: - grp.close() + grp.store.close() except AttributeError: pass del grp @@ -247,12 +251,17 @@ def get_best_level_for_downsample(self, downsample: float) -> int: return lvl - 1 return self.level_count - 1 - @cached_property - def zarr_group(self) -> zarr.hierarchy.Group: + @property + def zarr_group(self) -> zarr.Group: """return the tiff image as a zarr-like group NOTE: this is extra functionality and not part of the drop-in behaviour """ + try: + return self._zarr_local.zarr_group # type: ignore[no-any-return] + except AttributeError: + pass + try: _num_decode = os.environ["TIFFSLIDE_NUM_DECODE_THREADS"] except KeyError: @@ -262,13 +271,19 @@ def zarr_group(self) -> zarr.hierarchy.Group: num_decode_threads = int(_num_decode) else: num_decode_threads = None - store = get_zarr_store( - self.properties, self._tifffile, num_decode_threads=num_decode_threads - ) - return zarr.open_group(store, mode="r") + + with self._zarr_lock: + store = get_zarr_store( + self.properties, + self._tifffile, + num_decode_threads=num_decode_threads, + ) + grp = zarr.open_group(store, mode="r", zarr_format=2) + self._zarr_local.zarr_group = grp + return grp @property - def ts_zarr_grp(self) -> zarr.hierarchy.Group: + def ts_zarr_grp(self) -> zarr.Group: """use .zarr_group instead""" # backwards compatibility return self.zarr_group @@ -509,6 +524,8 @@ def __init__( tifffile_options=tifffile_options, _cls=NotTiffFile, ) + self._zarr_local = threading.local() + self._zarr_lock = threading.Lock() @classmethod def detect_format(