From e27aae6dd97302c0bed8527d3dd74b7cf6ae0e41 Mon Sep 17 00:00:00 2001 From: Tejas5405 Date: Sun, 21 Jun 2026 09:00:27 +0530 Subject: [PATCH] fix(parquet): pass storage_options as kwargs to fsspec.filesystem and fix mutable defaults --- src/litdata/utilities/parquet.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/litdata/utilities/parquet.py b/src/litdata/utilities/parquet.py index f4fcad0ef..44cdc1eff 100644 --- a/src/litdata/utilities/parquet.py +++ b/src/litdata/utilities/parquet.py @@ -78,7 +78,7 @@ def __init__( self, dir_path: str | Dir | None, cache_path: str | None = None, - storage_options: dict | None = {}, + storage_options: dict | None = None, num_workers: int = 4, ): if not _PYARROW_AVAILABLE: @@ -150,7 +150,7 @@ def __init__( for provider in _CLOUD_PROVIDER: if self.dir.url.startswith(provider): # Initialize the cloud filesystem - self.fs = fsspec.filesystem(provider, *self.storage_options) + self.fs = fsspec.filesystem(provider, **self.storage_options) print(f"using provider: {provider}") break @@ -304,7 +304,7 @@ def write_index(self, chunks_info: list[dict[str, Any]], config: dict[str, Any]) def get_parquet_indexer_cls( dir_path: str, cache_path: str | None = None, - storage_options: dict | None = {}, + storage_options: dict | None = None, num_workers: int = 4, ) -> ParquetDir: """Get the appropriate ParquetDir class based on the directory path scheme.