diff --git a/.config/policies/cpu-packaging.toml b/.config/policies/cpu-packaging.toml index 2084c7e2d..b583db7e8 100644 --- a/.config/policies/cpu-packaging.toml +++ b/.config/policies/cpu-packaging.toml @@ -53,4 +53,4 @@ deployment_requirement = "The host CPU must implement the full x86-64-v3 ISA; a # `scripts/build/cpu_packaging_policy.py shipped-features`. NVML device # diagnostics are opt-in through FEATURES and are not part of this list. [server] -features = ["cugraph", "cuml", "cuvs"] +features = ["cugraph", "cuml", "cuvs", "iceberg"] diff --git a/.config/policies/public-api/algeon-datafusion.txt b/.config/policies/public-api/algeon-datafusion.txt index 13d092c33..2bf3ddee9 100644 --- a/.config/policies/public-api/algeon-datafusion.txt +++ b/.config/policies/public-api/algeon-datafusion.txt @@ -146,12 +146,14 @@ pub algeon_datafusion::capability::CapabilityReason::NativeSortPreservingMergeUn pub algeon_datafusion::capability::CapabilityReason::NativeSourceByteRange pub algeon_datafusion::capability::CapabilityReason::NativeSourceEmpty pub algeon_datafusion::capability::CapabilityReason::NativeSourceExprAdapter +pub algeon_datafusion::capability::CapabilityReason::NativeSourceFileSchemaMismatch pub algeon_datafusion::capability::CapabilityReason::NativeSourceNonLocalObjectStore pub algeon_datafusion::capability::CapabilityReason::NativeSourceOutputOrdering pub algeon_datafusion::capability::CapabilityReason::NativeSourceParquetFileReaderFactory pub algeon_datafusion::capability::CapabilityReason::NativeSourcePartitionValues pub algeon_datafusion::capability::CapabilityReason::NativeSourcePreserveOrder pub algeon_datafusion::capability::CapabilityReason::NativeSourcePushedDownParquetPredicate +pub algeon_datafusion::capability::CapabilityReason::NativeSourceRemoteFooterUnavailable pub algeon_datafusion::capability::CapabilityReason::NativeSourceRowSelection pub algeon_datafusion::capability::CapabilityReason::NativeSourceSchemaMapping pub algeon_datafusion::capability::CapabilityReason::UnsupportedExpression @@ -743,6 +745,7 @@ pub algeon_datafusion::error::ErrorCode::SourceUnauthenticated pub algeon_datafusion::error::ErrorCode::SqlParse pub algeon_datafusion::error::ErrorCode::UnsupportedExpression pub algeon_datafusion::error::ErrorCode::UnsupportedOperator +pub algeon_datafusion::error::ErrorCode::UnsupportedTableFeature pub algeon_datafusion::error::ErrorCode::UnsupportedType impl algeon_datafusion::error::ErrorCode pub const algeon_datafusion::error::ErrorCode::ALL: &'static [Self] @@ -1976,6 +1979,16 @@ pub algeon_datafusion::server::PasswordHashError::InvalidHash(alloc::string::Str impl core::error::Error for algeon_datafusion::server::PasswordHashError impl core::fmt::Display for algeon_datafusion::server::PasswordHashError pub fn algeon_datafusion::server::PasswordHashError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub enum algeon_datafusion::server::ServerIcebergCatalogKind +pub algeon_datafusion::server::ServerIcebergCatalogKind::Glue(algeon_datafusion::server::ServerIcebergGlueCatalogConfig) +pub algeon_datafusion::server::ServerIcebergCatalogKind::Rest(algeon_datafusion::server::ServerIcebergRestCatalogConfig) +pub enum algeon_datafusion::server::ServerIcebergRestAuth +pub algeon_datafusion::server::ServerIcebergRestAuth::None +pub algeon_datafusion::server::ServerIcebergRestAuth::OAuth2 +pub algeon_datafusion::server::ServerIcebergRestAuth::OAuth2::credential: algeon_datafusion::server::ServerIcebergSecret +pub algeon_datafusion::server::ServerIcebergRestAuth::OAuth2::scope: core::option::Option +pub algeon_datafusion::server::ServerIcebergRestAuth::OAuth2::server_uri: core::option::Option +pub algeon_datafusion::server::ServerIcebergRestAuth::Token(algeon_datafusion::server::ServerIcebergSecret) pub enum algeon_datafusion::server::ServerShutdown pub algeon_datafusion::server::ServerShutdown::OsSignals pub algeon_datafusion::server::ServerShutdown::Token(tokio_util::sync::cancellation_token::CancellationToken) @@ -2022,6 +2035,7 @@ pub algeon_datafusion::server::ServerConfig::data_root: core::option::Option pub algeon_datafusion::server::ServerConfig::gpu_memory_ownership: alloc::vec::Vec<(usize, query_runtime::runtime::backend_core::GpuMemoryOwnership)> pub algeon_datafusion::server::ServerConfig::http2_keepalive: algeon_datafusion::server::Http2Keepalive +pub algeon_datafusion::server::ServerConfig::iceberg_catalogs: alloc::vec::Vec pub algeon_datafusion::server::ServerConfig::init_sql: core::option::Option pub algeon_datafusion::server::ServerConfig::models: core::option::Option pub algeon_datafusion::server::ServerConfig::native_optimizer_config: algeon_datafusion::planner::AlgeonNativeOptimizerConfig @@ -2035,6 +2049,7 @@ impl algeon_datafusion::server::ServerConfig pub fn algeon_datafusion::server::ServerConfig::from_env() -> algeon_datafusion::server::Result pub fn algeon_datafusion::server::ServerConfig::from_vars(impl core::iter::traits::collect::IntoIterator) -> algeon_datafusion::server::Result where K: core::convert::Into, V: core::convert::Into pub fn algeon_datafusion::server::ServerConfig::gpu_devices(&self) -> alloc::vec::Vec +pub fn algeon_datafusion::server::ServerConfig::s3_storage(&self) -> core::option::Option<&query_runtime::runtime::object_store::S3SourceStorageProfile> pub fn algeon_datafusion::server::ServerConfig::with_gpu_memory_ownership(self, impl core::iter::traits::collect::IntoIterator) -> Self pub struct algeon_datafusion::server::ServerError impl algeon_datafusion::server::ServerError @@ -2057,6 +2072,25 @@ impl core::error::Error for algeon_datafusion::server::ServerError pub fn algeon_datafusion::server::ServerError::source(&self) -> core::option::Option<&(dyn core::error::Error + 'static)> impl core::fmt::Display for algeon_datafusion::server::ServerError pub fn algeon_datafusion::server::ServerError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct algeon_datafusion::server::ServerIcebergCatalogConfig +pub algeon_datafusion::server::ServerIcebergCatalogConfig::kind: algeon_datafusion::server::ServerIcebergCatalogKind +pub algeon_datafusion::server::ServerIcebergCatalogConfig::name: alloc::string::String +pub struct algeon_datafusion::server::ServerIcebergGlueCatalogConfig +pub algeon_datafusion::server::ServerIcebergGlueCatalogConfig::catalog_id: core::option::Option +pub algeon_datafusion::server::ServerIcebergGlueCatalogConfig::endpoint: core::option::Option +pub algeon_datafusion::server::ServerIcebergGlueCatalogConfig::region: core::option::Option +pub algeon_datafusion::server::ServerIcebergGlueCatalogConfig::warehouse: alloc::string::String +pub struct algeon_datafusion::server::ServerIcebergRestCatalogConfig +pub algeon_datafusion::server::ServerIcebergRestCatalogConfig::auth: algeon_datafusion::server::ServerIcebergRestAuth +pub algeon_datafusion::server::ServerIcebergRestCatalogConfig::prefix: core::option::Option +pub algeon_datafusion::server::ServerIcebergRestCatalogConfig::uri: alloc::string::String +pub algeon_datafusion::server::ServerIcebergRestCatalogConfig::warehouse: core::option::Option +pub struct algeon_datafusion::server::ServerIcebergSecret(_) +impl algeon_datafusion::server::ServerIcebergSecret +pub fn algeon_datafusion::server::ServerIcebergSecret::expose(&self) -> &str +pub fn algeon_datafusion::server::ServerIcebergSecret::new(impl core::convert::Into) -> Self +impl core::fmt::Debug for algeon_datafusion::server::ServerIcebergSecret +pub fn algeon_datafusion::server::ServerIcebergSecret::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub struct algeon_datafusion::server::ServerObservabilityConfig pub algeon_datafusion::server::ServerObservabilityConfig::active_logical_capture_max_bytes: core::option::Option pub algeon_datafusion::server::ServerObservabilityConfig::global_sealed_max_bytes: u64 diff --git a/.config/policies/public-api/algeon-query-planning.txt b/.config/policies/public-api/algeon-query-planning.txt index d5980debd..e00a70e80 100644 --- a/.config/policies/public-api/algeon-query-planning.txt +++ b/.config/policies/public-api/algeon-query-planning.txt @@ -449,7 +449,7 @@ pub fn query_planning::chunk::SourceChunkPlan::row_group_count(&self) -> usize #[non_exhaustive] pub struct query_planning::chunk::SourceChunkPlanningOptions impl query_planning::chunk::SourceChunkPlanningOptions pub fn query_planning::chunk::SourceChunkPlanningOptions::validate(&self) -> query_planning::error::Result<()> -pub fn query_planning::chunk::SourceChunkPlanningOptions::with_local_parquet_throughput_max_row_groups_per_chunk(self, usize) -> Self +pub fn query_planning::chunk::SourceChunkPlanningOptions::with_parquet_throughput_max_row_groups_per_chunk(self, usize) -> Self impl core::default::Default for query_planning::chunk::SourceChunkPlanningOptions pub fn query_planning::chunk::SourceChunkPlanningOptions::default() -> Self pub struct query_planning::chunk::SourceChunkPlanningPolicy @@ -482,7 +482,7 @@ pub const fn query_planning::chunk::SourceTaskMemoryModel::source_adjacent_opera pub const fn query_planning::chunk::SourceTaskMemoryModel::source_read_preflight_slack_slots(self) -> u64 pub const fn query_planning::chunk::SourceTaskMemoryModel::source_read_slots(self) -> u64 pub const fn query_planning::chunk::SourceTaskMemoryModel::unary_chain_frame() -> Self -pub const query_planning::chunk::LOCAL_PARQUET_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK: usize +pub const query_planning::chunk::PARQUET_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK: usize pub fn query_planning::chunk::annotate_source_chunk_batching_reasons(&mut [query_planning::chunk::SourceChunk], core::option::Option<&'static str>, core::option::Option) pub fn query_planning::chunk::plan_source_chunk_batches(&[query_planning::chunk::SourceChunk], core::option::Option) -> alloc::vec::Vec pub fn query_planning::chunk::plan_source_chunk_batches_with_row_cap(&[query_planning::chunk::SourceChunk], core::option::Option, core::option::Option) -> alloc::vec::Vec @@ -852,7 +852,7 @@ pub type query_planning::error::Result = core::result::Result &'static str impl core::fmt::Display for query_planning::execution_contracts::SourceChunkPlanningMode @@ -3360,6 +3360,7 @@ pub query_planning::source::RejectedSourceFeature::PartitionColumns pub query_planning::source::RejectedSourceFeature::PartitionValues pub query_planning::source::RejectedSourceFeature::PreserveOrder pub query_planning::source::RejectedSourceFeature::PushedDownParquetPredicate +pub query_planning::source::RejectedSourceFeature::RemoteFooterUnavailable pub query_planning::source::RejectedSourceFeature::RowSelection pub query_planning::source::RejectedSourceFeature::ScanLimit pub query_planning::source::RejectedSourceFeature::SchemaMapping @@ -3593,6 +3594,8 @@ pub query_planning::source::SourceDiagnostics::selected_row_group_stats: core::o pub query_planning::source::SourceDiagnostics::selected_row_groups: core::option::Option pub query_planning::source::SourceDiagnostics::source_read_strategy: alloc::string::String pub query_planning::source::SourceDiagnostics::zero_column_count_disposition: query_planning::source::ZeroColumnCountDisposition +impl query_planning::source::SourceDiagnostics +pub fn query_planning::source::SourceDiagnostics::set_resolved_read_labels(&mut self, &[query_planning::source::LocalParquetFile]) pub struct query_planning::source::SourceFooterFacts<'a> impl<'a> query_planning::source::SourceFooterFacts<'a> pub const fn query_planning::source::SourceFooterFacts<'a>::new(&'a [core::option::Option<&'a parquet::file::metadata::ParquetMetaData>]) -> Self diff --git a/Cargo.lock b/Cargo.lock index ddf0c2e03..5b0bc87a6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,41 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common 0.1.7", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + [[package]] name = "ahash" version = "0.8.12" @@ -43,14 +78,14 @@ dependencies = [ "algeon-cudf-sys", "algeon-interop", "arrow", - "bitflags", - "indexmap", + "bitflags 2.11.1", + "indexmap 2.14.0", "parquet", "smallvec", "static_assertions", "tempfile", "tokio", - "typed-builder", + "typed-builder 0.23.2", ] [[package]] @@ -152,6 +187,7 @@ dependencies = [ "prost", "rand 0.10.1", "rcgen", + "rustls", "serde", "serde_json", "sha2", @@ -167,6 +203,27 @@ dependencies = [ "workloads", ] +[[package]] +name = "algeon-iceberg" +version = "0.1.2" +dependencies = [ + "algeon-build-support", + "algeon-cudf-sys", + "algeon-datafusion", + "algeon-query-runtime", + "async-trait", + "datafusion", + "futures", + "iceberg", + "iceberg-catalog-glue", + "iceberg-catalog-rest", + "iceberg-storage-opendal", + "opendal", + "reqwest 0.12.28", + "tempfile", + "tokio", +] + [[package]] name = "algeon-interop" version = "0.1.2" @@ -244,11 +301,18 @@ dependencies = [ "algeon-cuml-sys", "algeon-cuvs-sys", "algeon-datafusion", + "algeon-iceberg", "tokio", "tracing", "tracing-subscriber", ] +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + [[package]] name = "alloc-no-stdlib" version = "2.0.4" @@ -285,6 +349,33 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "apache-avro" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "312c1ea69e5fe9966e0029fb95aca8790100b85aff4f0d3b00a9337c74069a9c" +dependencies = [ + "bigdecimal", + "bon", + "crc32fast", + "digest 0.11.3", + "log", + "miniz_oxide 0.9.1", + "num-bigint 0.4.8", + "ouroboros", + "quad-rand", + "rand 0.10.1", + "regex-lite", + "serde", + "serde_bytes", + "serde_json", + "snap", + "strum 0.28.0", + "thiserror 2.0.18", + "uuid", + "zstd", +] + [[package]] name = "ar_archive_writer" version = "0.5.1" @@ -294,6 +385,15 @@ dependencies = [ "object", ] +[[package]] +name = "arc-swap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "380962b3d93b99642e50b9febf72c071e97e8b32d86eae9cf4ef582bb960ae6e" +dependencies = [ + "rustversion", +] + [[package]] name = "argon2" version = "0.5.3" @@ -306,6 +406,12 @@ dependencies = [ "password-hash", ] +[[package]] +name = "array-init" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" + [[package]] name = "arrayref" version = "0.3.9" @@ -492,7 +598,7 @@ dependencies = [ "arrow-select", "chrono", "half", - "indexmap", + "indexmap 2.14.0", "itoa", "lexical-core", "memchr", @@ -535,7 +641,7 @@ version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10fab8d4563491417ba801fab29d205104d20d4bdf37bda6cd1cf425cff598cd" dependencies = [ - "bitflags", + "bitflags 2.11.1", "serde_core", "serde_json", ] @@ -571,6 +677,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "as-any" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0f477b951e452a0b6b4a10b53ccd569042d1d01729b519e02074a9c0958a063" + [[package]] name = "asn1-rs" version = "0.7.2" @@ -622,6 +734,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + [[package]] name = "async-trait" version = "0.1.92" @@ -633,6 +756,22 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "asyncband" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94a214ba60d6231afd0e805e3c27c45a1626d9debaa5a5061c45a1ea1b2f1ed0" +dependencies = [ + "hashbrown 0.17.1", + "slab", +] + +[[package]] +name = "asyncband" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5161fb43cb8f231f06983cb2b8f9ee9cee14be0b1356d8383398cc0f312800d1" + [[package]] name = "atoi" version = "2.0.0" @@ -655,172 +794,619 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] -name = "axum" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" -dependencies = [ - "axum-core", +name = "aws-config" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "939e256cfe0ed68e96d32f194339d88210d5c10685cff0787400158f852c6819" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", "bytes", - "form_urlencoded", - "futures-util", + "fastrand", + "hex", "http", - "http-body", - "http-body-util", - "hyper", - "hyper-util", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "serde_core", - "serde_json", - "serde_path_to_error", - "serde_urlencoded", - "sync_wrapper", + "sha1 0.10.7", + "time", "tokio", - "tower", - "tower-layer", - "tower-service", "tracing", + "url", + "zeroize", ] [[package]] -name = "axum-core" -version = "0.5.6" +name = "aws-credential-types" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +checksum = "576e8da35751dc85409ec1cd15999f692c277a1fbe9336780fe6f15e49dd637d" dependencies = [ - "bytes", - "futures-core", - "http", - "http-body", - "http-body-util", - "mime", - "pin-project-lite", - "sync_wrapper", - "tower-layer", - "tower-service", - "tracing", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", ] [[package]] -name = "base64" -version = "0.22.1" +name = "aws-lc-rs" +version = "1.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +checksum = "faac5829c2b74c28f830747e7818ccfb684261b5f48a1118b1e2a13d36dfab13" +dependencies = [ + "aws-lc-sys", + "zeroize", +] [[package]] -name = "base64" -version = "0.23.1" +name = "aws-lc-sys" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" +checksum = "f1622d8446a2d4b2ce0c7eefc73dd43a99779028d5ee5c2dd8073a658ba8a2bc" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] [[package]] -name = "base64ct" -version = "1.8.3" +name = "aws-runtime" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" - -[[package]] -name = "bench" -version = "0.1.2" +checksum = "9a0cc2144962cc5e358467b16e3d82981b1d424a7c3d34602be0036103814a1a" dependencies = [ - "algeon-build-support", - "algeon-cudf", - "algeon-cudf-sys", - "algeon-cugraph", - "algeon-cugraph-sys", - "algeon-cuml-sys", - "algeon-cuvs-sys", - "algeon-datafusion", - "algeon-query-engine", - "algeon-query-planning", - "algeon-query-runtime", - "arrow", - "datafusion", - "futures", - "parquet", - "serde", - "serde_json", - "sha2", - "tokio", - "workloads", + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "bytes-utils", + "fastrand", + "http", + "http-body", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", ] [[package]] -name = "bigdecimal" -version = "0.4.10" +name = "aws-sdk-glue" +version = "1.180.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" +checksum = "e01a203a04c200f631f81c668204ccb45cdcba4a3fe928444e2324a4fe52f0b8" dependencies = [ - "autocfg", - "libm", - "num-bigint 0.4.6", - "num-integer", - "num-traits", + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http", + "regex-lite", + "tracing", ] [[package]] -name = "bit-set" -version = "0.8.0" +name = "aws-sdk-sso" +version = "1.117.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +checksum = "5316813498fa9c403877007934ae448aa44b18136f58051f8fa3c5b79b44bdc6" dependencies = [ - "bit-vec 0.8.0", + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http", + "regex-lite", + "tracing", ] [[package]] -name = "bit-vec" -version = "0.8.0" +name = "aws-sdk-ssooidc" +version = "1.119.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +checksum = "31db35e5662787b786daba15f431e4393c7ea512b70fa112bd048d1f3ecf877a" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http", + "regex-lite", + "tracing", +] [[package]] -name = "bit-vec" -version = "0.9.1" +name = "aws-sdk-sts" +version = "1.122.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +checksum = "a10491638d9e560db90c484dbc77d73b8ad202780cdc9b9e92180603acb07ada" dependencies = [ - "serde", + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http", + "regex-lite", + "tracing", ] [[package]] -name = "bitflags" -version = "2.11.1" +name = "aws-sigv4" +version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +checksum = "9d3d48355cb86b15ccbb38b8bc6b87500c824ee4ed8cbc10a961b435242b5e16" +dependencies = [ + "aws-credential-types", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http", + "percent-encoding", + "sha2", + "time", + "tracing", +] [[package]] -name = "blake2" -version = "0.10.6" +name = "aws-smithy-async" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +checksum = "de681a9df7ff79823b759ee17061076ec049cd61a8fa28ece4293d89c9f6964e" dependencies = [ - "digest 0.10.7", + "futures-util", + "pin-project-lite", + "tokio", ] [[package]] -name = "blake3" -version = "1.8.5" +name = "aws-smithy-http" +version = "0.64.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +checksum = "ba8960956e0ee0ca3f3161e62913a61bc65a7ef638ca3ef1d4c7159f8443e398" dependencies = [ - "arrayref", - "arrayvec", - "cc", - "cfg-if", - "constant_time_eq", - "cpufeatures 0.3.0", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "percent-encoding", + "pin-project-lite", + "tracing", ] [[package]] -name = "block-buffer" -version = "0.10.4" +name = "aws-smithy-http-client" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +checksum = "aeca69f65362f7375891008ea525fd440ad6db1836458ac188c87b126804114e" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2", + "http", + "hyper", + "hyper-rustls", + "hyper-util", + "pin-project-lite", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.65.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f6835fb38bc6fc8c91e584b0366eb4ff595e5c485134dbf4f0bdeaa18856666" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e86338c869539a581bf161247762a6e87f92c5c075060057b5ed6d06632ed0c" +dependencies = [ + "aws-smithy-runtime-api", +] + +[[package]] +name = "aws-smithy-query" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "165f6265342a313ad6f7a1ac17a0cdba30c522039d3f41d8d6871ce8a2b34ecd" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-smithy-xml", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cef0c41e8ca40f74f68ed8ee4d1ddbde12bb847b95ef9b3c025681ddfae58a" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-client", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "bytes", + "fastrand", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06788004dea79018413f381891da6dcca23196acd79c4d51b767fbdacc76eae4" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api-macros", + "aws-smithy-types", + "bytes", + "http", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-runtime-api-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "aws-smithy-schema" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec8dae1015a6d959a5faf86fbdf77486cfe19e4ccb6ee009d8a46efc45028a4" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http", +] + +[[package]] +name = "aws-smithy-types" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55752025c39c0e094962fd73f1c57014b191f678612e517f219468e575af9822" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http", + "http-body", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6581618a2fd006159016d1c36881d94d61b4a017f4a8d631310875e18252413" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba0d970d3947bb064d2ae1d823efa186c1dbde88f88afa8f6549946e5d8deee" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "rustc_version", + "tracing", +] + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", + "gloo-timers", + "tokio", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bench" +version = "0.1.2" +dependencies = [ + "algeon-build-support", + "algeon-cudf", + "algeon-cudf-sys", + "algeon-cugraph", + "algeon-cugraph-sys", + "algeon-cuml-sys", + "algeon-cuvs-sys", + "algeon-datafusion", + "algeon-query-engine", + "algeon-query-planning", + "algeon-query-runtime", + "arrow", + "datafusion", + "futures", + "parquet", + "serde", + "serde_json", + "sha2", + "tokio", + "workloads", +] + +[[package]] +name = "bigdecimal" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" +dependencies = [ + "autocfg", + "libm", + "num-bigint 0.4.8", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "bimap" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec 0.8.0", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" + +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "blake3" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.0", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ "generic-array", ] @@ -834,6 +1420,39 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "bnum" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f781dba93de3a5ef6dc5b17c9958b208f6f3f021623b360fb605ea51ce443f10" +dependencies = [ + "serde", + "serde-big-array", +] + +[[package]] +name = "bon" +version = "3.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a05442cd7005532d0cfedec676b5b9333d2b4c7c3afabe2b1a9ce2742fbe2ee6" +dependencies = [ + "bon-macros", +] + +[[package]] +name = "bon-macros" +version = "3.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93833e97ac7c46ba97045afcf247238c27057538fcf2f06344662ddd5f567be6" +dependencies = [ + "darling 0.24.1", + "ident_case", + "prettyplease 0.3.0", + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "brotli" version = "8.0.2" @@ -855,18 +1474,49 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + [[package]] name = "bumpalo" version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "bzip2" version = "0.6.1" @@ -918,8 +1568,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", + "js-sys", "num-traits", "serde", + "wasm-bindgen", "windows-link", ] @@ -933,6 +1585,16 @@ dependencies = [ "phf", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + [[package]] name = "cmake" version = "0.1.58" @@ -957,6 +1619,22 @@ dependencies = [ "which", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "combine" +version = "4.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4356eba6976435498d05a311ee9fc486da3bc81c200d3f0d7dd9703b41913f92" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "comfy-table" version = "7.2.2" @@ -1054,6 +1732,16 @@ dependencies = [ "libc", ] +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest 0.10.7", + "spin", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -1063,6 +1751,24 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -1081,48 +1787,97 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ - "generic-array", - "typenum", + "generic-array", + "rand_core 0.6.4", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77727bb15fa921304124b128af125e7e3b968275d1b108b379190264f4423710" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "ctor" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d" +dependencies = [ + "link-section", + "linktime-proc-macro", +] + +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", ] [[package]] -name = "crypto-common" -version = "0.2.1" +name = "ctutils" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77727bb15fa921304124b128af125e7e3b968275d1b108b379190264f4423710" +checksum = "38d28fb6aa7dd680b3ffd1a3e64e115c177bdba63c0ae7f9cd48129ff05c4fa0" dependencies = [ - "hybrid-array", + "cmov", ] [[package]] -name = "csv" -version = "1.4.0" +name = "darling" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ - "csv-core", - "itoa", - "ryu", - "serde_core", + "darling_core 0.20.11", + "darling_macro 0.20.11", ] [[package]] -name = "csv-core" -version = "0.1.13" +name = "darling" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ - "memchr", + "darling_core 0.23.0", + "darling_macro 0.23.0", ] [[package]] name = "darling" -version = "0.20.11" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.24.1", + "darling_macro 0.24.1", ] [[package]] @@ -1139,17 +1894,65 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.117", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.3", +] + [[package]] name = "darling_macro" version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ - "darling_core", + "darling_core 0.20.11", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core 0.23.0", "quote", "syn 2.0.117", ] +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core 0.24.1", + "quote", + "syn 3.0.3", +] + [[package]] name = "dashmap" version = "6.2.1" @@ -1208,7 +2011,7 @@ dependencies = [ "datafusion-sql", "flate2", "futures", - "indexmap", + "indexmap 2.14.0", "itertools 0.15.0", "liblzma", "log", @@ -1285,7 +2088,7 @@ dependencies = [ "foldhash 0.2.0", "half", "hashbrown 0.17.1", - "indexmap", + "indexmap 2.14.0", "itertools 0.15.0", "libc", "log", @@ -1496,7 +2299,7 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", - "indexmap", + "indexmap 2.14.0", "itertools 0.15.0", "recursive", "serde_json", @@ -1511,7 +2314,7 @@ checksum = "6a18baa840f77760dcc9eaad281329bc57536c5c39e65e8649596f2a51216be7" dependencies = [ "arrow", "datafusion-common", - "indexmap", + "indexmap 2.14.0", "itertools 0.15.0", ] @@ -1678,7 +2481,7 @@ dependencies = [ "datafusion-expr", "datafusion-expr-common", "datafusion-physical-expr", - "indexmap", + "indexmap 2.14.0", "itertools 0.15.0", "log", "recursive", @@ -1701,7 +2504,7 @@ dependencies = [ "datafusion-physical-expr-common", "half", "hashbrown 0.17.1", - "indexmap", + "indexmap 2.14.0", "itertools 0.15.0", "parking_lot", "petgraph", @@ -1735,7 +2538,7 @@ dependencies = [ "datafusion-common", "datafusion-expr-common", "hashbrown 0.17.1", - "indexmap", + "indexmap 2.14.0", "itertools 0.15.0", "parking_lot", "pin-project", @@ -1786,7 +2589,7 @@ dependencies = [ "futures", "half", "hashbrown 0.17.1", - "indexmap", + "indexmap 2.14.0", "itertools 0.15.0", "log", "num-traits", @@ -1839,7 +2642,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-functions-nested", - "indexmap", + "indexmap 2.14.0", "log", "recursive", "regex", @@ -1847,6 +2650,37 @@ dependencies = [ "stacker", ] +[[package]] +name = "defmt" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "defmt-parser" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" +dependencies = [ + "thiserror 2.0.18", +] + [[package]] name = "der-parser" version = "10.0.0" @@ -1856,7 +2690,7 @@ dependencies = [ "asn1-rs", "displaydoc", "nom", - "num-bigint 0.4.6", + "num-bigint 0.4.8", "num-traits", "rusticata-macros", ] @@ -1866,6 +2700,40 @@ name = "deranged" version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "serde_core", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.117", +] [[package]] name = "digest" @@ -1887,6 +2755,7 @@ dependencies = [ "block-buffer 0.12.0", "const-oid", "crypto-common 0.2.1", + "ctutils", ] [[package]] @@ -1900,6 +2769,27 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "dlv-list" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" +dependencies = [ + "const-random", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + [[package]] name = "either" version = "1.15.0" @@ -1912,6 +2802,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "erased-serde" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + [[package]] name = "errno" version = "0.3.14" @@ -1922,6 +2823,26 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "examples" version = "0.1.2" @@ -1940,6 +2861,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "fastnum" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020d1b59a944bc239d79903fbac2eda2365138b44890c27979562f6592059dcd" +dependencies = [ + "bnum", + "num-integer", + "num-traits", + "serde", +] + [[package]] name = "fastrand" version = "2.4.1" @@ -1964,7 +2897,7 @@ version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ - "bitflags", + "bitflags 2.11.1", "rustc_version", ] @@ -1975,7 +2908,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.8.9", "zlib-rs", ] @@ -2006,6 +2939,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.32" @@ -2145,12 +3084,34 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "glob" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "h2" version = "0.4.16" @@ -2163,7 +3124,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap", + "indexmap 2.14.0", "slab", "tokio", "tokio-util", @@ -2182,6 +3143,12 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.5" @@ -2208,6 +3175,12 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "heck" version = "0.5.0" @@ -2220,6 +3193,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "http" version = "1.4.0" @@ -2316,6 +3298,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", + "webpki-roots", ] [[package]] @@ -2378,6 +3361,129 @@ dependencies = [ "cc", ] +[[package]] +name = "iceberg" +version = "0.10.1" +source = "git+https://github.com/apache/iceberg-rust.git?rev=5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3#5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" +dependencies = [ + "aes-gcm", + "anyhow", + "apache-avro", + "array-init", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", + "arrow-string", + "as-any", + "async-trait", + "backon", + "base64 0.22.1", + "bimap", + "bytes", + "chrono", + "crc32fast", + "derive_builder", + "fastnum", + "flate2", + "fnv", + "form_urlencoded", + "futures", + "iceberg-property-macro", + "itertools 0.13.0", + "moka", + "murmur3", + "once_cell", + "ordered-float", + "parquet", + "rand 0.9.4", + "reqwest 0.12.28", + "roaring", + "serde", + "serde_bytes", + "serde_derive", + "serde_json", + "serde_repr", + "serde_with", + "strum 0.27.2", + "tokio", + "tracing", + "typed-builder 0.20.1", + "typetag", + "url", + "uuid", + "zeroize", + "zstd", +] + +[[package]] +name = "iceberg-catalog-glue" +version = "0.10.1" +source = "git+https://github.com/apache/iceberg-rust.git?rev=5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3#5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" +dependencies = [ + "anyhow", + "async-trait", + "aws-config", + "aws-sdk-glue", + "iceberg", + "iceberg-storage-opendal", + "serde_json", + "tokio", +] + +[[package]] +name = "iceberg-catalog-rest" +version = "0.10.1" +source = "git+https://github.com/apache/iceberg-rust.git?rev=5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3#5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" +dependencies = [ + "async-trait", + "chrono", + "http", + "iceberg", + "itertools 0.13.0", + "reqwest 0.12.28", + "serde", + "serde_derive", + "serde_json", + "tokio", + "tracing", + "typed-builder 0.20.1", + "uuid", +] + +[[package]] +name = "iceberg-property-macro" +version = "0.10.1" +source = "git+https://github.com/apache/iceberg-rust.git?rev=5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3#5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "iceberg-storage-opendal" +version = "0.10.1" +source = "git+https://github.com/apache/iceberg-rust.git?rev=5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3#5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "cfg-if", + "futures", + "iceberg", + "iceberg-property-macro", + "opendal", + "reqsign-aws-v4", + "reqsign-core", + "serde", + "typetag", + "url", +] + [[package]] name = "icu_collections" version = "2.2.0" @@ -2493,6 +3599,17 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -2505,12 +3622,39 @@ dependencies = [ "serde_core", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "inventory" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6928282826c822ad91bf1c9a1cb90a30ba1c26770749929b4656cd6be829cd7c" +dependencies = [ + "rustversion", +] + [[package]] name = "ipnet" version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.14.0" @@ -2535,6 +3679,111 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jiff" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "js-sys", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" +dependencies = [ + "defmt", + "log", +] + +[[package]] +name = "jiff-static" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8377070c6bae868759445e5a77f66d84f0b72f3a054bfb00e6d038b8282da7" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.18", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.117", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.117", +] + [[package]] name = "jobserver" version = "0.1.34" @@ -2674,6 +3923,18 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +[[package]] +name = "link-section" +version = "0.19.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39c29a617ce3df32c08497bdc1ab6e2376e0b17948ac166a2fbe5977c5954cd9" + +[[package]] +name = "linktime-proc-macro" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e57c38c1e860fd37c604281cdfb1dd2216977fd76a50f85ba2f388ef3219616" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -2779,6 +4040,15 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", +] + [[package]] name = "mio" version = "1.2.0" @@ -2790,6 +4060,32 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "moka" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "murmur3" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" + [[package]] name = "nom" version = "7.1.3" @@ -2811,12 +4107,13 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.6" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" dependencies = [ "num-integer", "num-traits", + "serde", ] [[package]] @@ -2867,87 +4164,228 @@ dependencies = [ name = "nvml-wrapper" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f049ae562349fefb8e837eb15443da1e7c6dcbd8a11f52a228f92220c2e5c85e" +checksum = "f049ae562349fefb8e837eb15443da1e7c6dcbd8a11f52a228f92220c2e5c85e" +dependencies = [ + "bitflags 2.11.1", + "libloading", + "nvml-wrapper-sys", + "static_assertions", + "thiserror 1.0.69", + "wrapcenum-derive", +] + +[[package]] +name = "nvml-wrapper-sys" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b4d594420fcda43b1c2c4bd44d48974aa3c7a9ab2cbf10dc18e35265767bf0b" +dependencies = [ + "libloading", +] + +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + +[[package]] +name = "object_store" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "chrono", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body-util", + "humantime", + "hyper", + "itertools 0.14.0", + "md-5 0.10.6", + "parking_lot", + "percent-encoding", + "quick-xml 0.39.4", + "rand 0.10.1", + "reqwest 0.12.28", + "ring", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "opendal" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33dbff14cc9bb085224256d6a81289d2f3202e85b06f408d42534b42162a4231" +dependencies = [ + "ctor", + "opendal-core", + "opendal-http-transport-reqwest", + "opendal-layer-concurrent-limit", + "opendal-layer-logging", + "opendal-layer-retry", + "opendal-layer-timeout", + "opendal-service-fs", + "opendal-service-s3", +] + +[[package]] +name = "opendal-core" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48dbcef97d3eb7591db2c18d5cae95c836bcce07359b98d98dd6f4e861eb77b7" +dependencies = [ + "anyhow", + "asyncband 0.6.7", + "base64 0.23.1", + "bytes", + "futures", + "http", + "jiff", + "log", + "md-5 0.11.0", + "percent-encoding", + "quick-xml 0.41.0", + "reqsign-core", + "serde", + "serde_json", + "tokio", + "url", + "uuid", + "web-time", +] + +[[package]] +name = "opendal-http-transport-reqwest" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85663452ea32bbc17e8f79ab29788c846d116ec7de31451be9c787e462dcb36c" +dependencies = [ + "bytes", + "futures", + "http", + "http-body", + "opendal-core", + "reqwest 0.13.5", +] + +[[package]] +name = "opendal-layer-concurrent-limit" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03f9e144b5228d741c3763ade8711d9b72e5fb6d998e779f2d7a09da0b5a3eba" +dependencies = [ + "asyncband 0.6.7", + "futures", + "http", + "opendal-core", +] + +[[package]] +name = "opendal-layer-logging" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2de17c61cd32e9d8d7d8efb91795e714dbccbafbc3c4e219e1542f4d6324161" dependencies = [ - "bitflags", - "libloading", - "nvml-wrapper-sys", - "static_assertions", - "thiserror 1.0.69", - "wrapcenum-derive", + "log", + "opendal-core", ] [[package]] -name = "nvml-wrapper-sys" -version = "0.9.1" +name = "opendal-layer-retry" +version = "0.58.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b4d594420fcda43b1c2c4bd44d48974aa3c7a9ab2cbf10dc18e35265767bf0b" +checksum = "e94db301964a25366090484d61e6da16d5979cc8faf02c3e12210dc74fafed38" dependencies = [ - "libloading", + "backon", + "log", + "opendal-core", ] [[package]] -name = "object" -version = "0.37.3" +name = "opendal-layer-timeout" +version = "0.58.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +checksum = "08956ddda07465449bfd48825f4f0f25e0351278ac974eaa659895d9d74f2c80" dependencies = [ - "memchr", + "opendal-core", + "tokio", ] [[package]] -name = "object_store" -version = "0.13.2" +name = "opendal-service-fs" +version = "0.58.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" +checksum = "c7ef1e1c45f3f89282a59073897e0d685e51385fed0aea771714789525cff996" dependencies = [ - "async-trait", - "base64 0.22.1", "bytes", - "chrono", - "form_urlencoded", - "futures-channel", - "futures-core", - "futures-util", - "http", - "http-body-util", - "humantime", - "hyper", - "itertools 0.14.0", - "md-5 0.10.6", - "parking_lot", - "percent-encoding", - "quick-xml", - "rand 0.10.1", - "reqwest", - "ring", + "log", + "opendal-core", "serde", - "serde_json", - "serde_urlencoded", - "thiserror 2.0.18", "tokio", - "tracing", - "url", - "walkdir", - "wasm-bindgen-futures", - "web-time", + "xattr", ] [[package]] -name = "oid-registry" -version = "0.8.1" +name = "opendal-service-s3" +version = "0.58.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +checksum = "c64335f9f24ccb62ac36f1d976342b48611a75ba61979813a4f78a4ebd94de42" dependencies = [ - "asn1-rs", + "base64 0.23.1", + "bytes", + "crc-fast", + "http", + "log", + "md-5 0.11.0", + "opendal-core", + "quick-xml 0.41.0", + "reqsign-aws-v4", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "url", ] -[[package]] -name = "once_cell" -version = "1.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" - [[package]] name = "openssl-probe" version = "0.2.1" @@ -2990,6 +4428,61 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ordered-multimap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" +dependencies = [ + "dlv-list", + "hashbrown 0.14.5", +] + +[[package]] +name = "ouroboros" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0f050db9c44b97a94723127e6be766ac5c340c48f2c4bb3ffa11713744be59" +dependencies = [ + "aliasable", + "ouroboros_macro", + "static_assertions", +] + +[[package]] +name = "ouroboros_macro" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c7028bdd3d43083f6d8d4d5187680d0d3560d54df4cc9d752005268b41e64d0" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.5" @@ -3039,6 +4532,7 @@ dependencies = [ "num-integer", "num-traits", "object_store", + "ring", "seq-macro", "simdutf8", "snap", @@ -3082,7 +4576,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset", "hashbrown 0.15.5", - "indexmap", + "indexmap 2.14.0", "serde", ] @@ -3136,12 +4630,33 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +[[package]] +name = "portable-atomic-util" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +dependencies = [ + "portable-atomic", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -3176,6 +4691,16 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "prettyplease" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" +dependencies = [ + "proc-macro2", + "syn 3.0.3", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -3185,6 +4710,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "version_check", + "yansi", +] + [[package]] name = "proptest" version = "1.11.0" @@ -3193,7 +4731,7 @@ checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" dependencies = [ "bit-set", "bit-vec 0.8.0", - "bitflags", + "bitflags 2.11.1", "num-traits", "rand 0.9.4", "rand_chacha", @@ -3246,6 +4784,12 @@ dependencies = [ "cc", ] +[[package]] +name = "quad-rand" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" + [[package]] name = "quick-error" version = "1.2.3" @@ -3262,6 +4806,26 @@ dependencies = [ "serde", ] +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quick-xml" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quinn" version = "0.11.9" @@ -3288,6 +4852,7 @@ version = "0.11.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" dependencies = [ + "aws-lc-rs", "bytes", "getrandom 0.4.2", "lru-slab", @@ -3375,6 +4940,9 @@ name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] [[package]] name = "rand_core" @@ -3449,7 +5017,27 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", + "bitflags 2.11.1", +] + +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", ] [[package]] @@ -3475,23 +5063,139 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "regex-syntax" -version = "0.8.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" - +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "reqsign-aws-core" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38c902d8ec745bd9eadd1abbcbe227f4c1fc2926b2ec8929f89234af3114500" +dependencies = [ + "bytes", + "form_urlencoded", + "hex", + "http", + "log", + "percent-encoding", + "quick-xml 0.42.0", + "reqsign-core", + "rust-ini", + "serde", + "serde_json", + "serde_urlencoded", + "sha1 0.11.0", +] + +[[package]] +name = "reqsign-aws-v4" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "906bfd028f96bfe73fdb892ce26f3b3f99b4f30feb9bd024be58bd6657f68b2b" +dependencies = [ + "bytes", + "http", + "log", + "quick-xml 0.42.0", + "reqsign-aws-core", + "reqsign-core", + "serde", +] + +[[package]] +name = "reqsign-core" +version = "3.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85fed85c436b6b920e4b36310f699a77771a91e38ae483df48447bd7d162671c" +dependencies = [ + "anyhow", + "asyncband 0.7.3", + "base64 0.23.1", + "bytes", + "hex", + "hmac", + "http", + "jiff", + "log", + "percent-encoding", + "sha1 0.11.0", + "sha2", + "windows-sys 0.61.2", +] + +[[package]] +name = "reqsign-file-read-tokio" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "586a8e198057773f9616901fd723a4bc186812b9765bc0f20368661d6169d906" +dependencies = [ + "anyhow", + "reqsign-core", + "tokio", +] + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.4.2", + "web-sys", + "webpki-roots", +] + [[package]] name = "reqwest" -version = "0.12.28" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "futures-core", "futures-util", - "h2", "http", "http-body", "http-body-util", @@ -3504,11 +5208,8 @@ dependencies = [ "pin-project-lite", "quinn", "rustls", - "rustls-native-certs", "rustls-pki-types", - "serde", - "serde_json", - "serde_urlencoded", + "rustls-platform-verifier", "sync_wrapper", "tokio", "tokio-rustls", @@ -3519,7 +5220,7 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.5.0", "web-sys", ] @@ -3537,6 +5238,26 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "roaring" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18bd8a37d17a58532776dcdf6041ce64929adca78e8489d5cacbafe99229d3e1" +dependencies = [ + "bytemuck", + "byteorder", +] + +[[package]] +name = "rust-ini" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" +dependencies = [ + "cfg-if", + "ordered-multimap", +] + [[package]] name = "rustc-hash" version = "2.1.2" @@ -3567,7 +5288,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys", @@ -3580,6 +5301,7 @@ version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", @@ -3611,12 +5333,40 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" + [[package]] name = "rustls-webpki" version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -3664,6 +5414,30 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab508826f74a77ca9d5aba6ff19b522583ee3eaf28a19384ff3d0e5835fadf6e" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -3676,7 +5450,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags", + "bitflags 2.11.1", "core-foundation", "core-foundation-sys", "libc", @@ -3715,6 +5489,25 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-big-array" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "serde_core" version = "1.0.228" @@ -3737,11 +5530,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ - "indexmap", + "indexmap 2.14.0", "itoa", "memchr", "serde", @@ -3760,6 +5553,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "serde_spanned" version = "1.1.1" @@ -3781,6 +5585,61 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a" +dependencies = [ + "base64 0.22.1", + "bs58", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.14.0", + "jiff", + "schemars 0.9.0", + "schemars 1.2.3", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46" +dependencies = [ + "darling 0.23.0", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + [[package]] name = "sha2" version = "0.11.0" @@ -3823,6 +5682,16 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -3863,6 +5732,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + [[package]] name = "sqlparser" version = "0.62.0" @@ -3916,6 +5791,48 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros 0.27.2", +] + +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros 0.28.0", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "subtle" version = "2.6.1" @@ -3964,6 +5881,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "tempfile" version = "3.27.0" @@ -3971,7 +5894,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", @@ -4158,7 +6081,7 @@ version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" dependencies = [ - "indexmap", + "indexmap 2.14.0", "serde_core", "serde_spanned", "toml_datetime", @@ -4243,6 +6166,7 @@ dependencies = [ "algeon-cuml-sys", "algeon-cuvs-sys", "algeon-datafusion", + "algeon-iceberg", "algeon-query-engine", "algeon-query-planning", "algeon-query-runtime", @@ -4254,11 +6178,12 @@ dependencies = [ "bench", "datafusion", "futures", + "iceberg", "opentelemetry", "opentelemetry-proto", "parquet", "prost", - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", "sha2", @@ -4280,7 +6205,7 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", - "indexmap", + "indexmap 2.14.0", "pin-project-lite", "slab", "sync_wrapper", @@ -4297,7 +6222,7 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags", + "bitflags 2.11.1", "bytes", "futures-util", "http", @@ -4395,13 +6320,33 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +[[package]] +name = "typed-builder" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd9d30e3a08026c78f246b173243cf07b3696d274debd26680773b6773c2afc7" +dependencies = [ + "typed-builder-macro 0.20.1", +] + [[package]] name = "typed-builder" version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31aa81521b70f94402501d848ccc0ecaa8f93c8eb6999eb9747e72287757ffda" dependencies = [ - "typed-builder-macro", + "typed-builder-macro 0.23.2", +] + +[[package]] +name = "typed-builder-macro" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", ] [[package]] @@ -4415,12 +6360,42 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + [[package]] name = "typenum" version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" +[[package]] +name = "typetag" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c90e86058a30d42a1a928dfb4b49bb33c98c3a2b4909492e6b0881cd94798ec2" +dependencies = [ + "erased-serde", + "inventory", + "once_cell", + "serde", + "typetag-impl", +] + +[[package]] +name = "typetag-impl" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "unarray" version = "0.1.4" @@ -4451,6 +6426,16 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common 0.1.7", + "subtle", +] + [[package]] name = "untrusted" version = "0.9.0" @@ -4469,6 +6454,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -4477,12 +6468,13 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.23.1" +version = "1.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" dependencies = [ "getrandom 0.4.2", "js-sys", + "serde_core", "wasm-bindgen", ] @@ -4498,6 +6490,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "wait-timeout" version = "0.2.1" @@ -4622,7 +6620,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "indexmap", + "indexmap 2.14.0", "wasm-encoder", "wasmparser", ] @@ -4640,15 +6638,28 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags", + "bitflags 2.11.1", "hashbrown 0.15.5", - "indexmap", + "indexmap 2.14.0", "semver", ] @@ -4672,6 +6683,24 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e49d0e65abc00c142e8f9428da991da9f1a250c39be97459901eb12b9b49f62a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "webpki-roots" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "which" version = "8.0.5" @@ -4933,7 +6962,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" dependencies = [ "anyhow", - "heck", + "heck 0.5.0", "wit-parser", ] @@ -4944,9 +6973,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", - "heck", - "indexmap", - "prettyplease", + "heck 0.5.0", + "indexmap 2.14.0", + "prettyplease 0.2.37", "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", @@ -4960,7 +6989,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" dependencies = [ "anyhow", - "prettyplease", + "prettyplease 0.2.37", "proc-macro2", "quote", "syn 2.0.117", @@ -4975,8 +7004,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags", - "indexmap", + "bitflags 2.11.1", + "indexmap 2.14.0", "log", "serde", "serde_derive", @@ -4995,7 +7024,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", - "indexmap", + "indexmap 2.14.0", "log", "semver", "serde", @@ -5026,7 +7055,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a76ff259533532054cfbaefb115c613203c73707017459206380f03b3b3f266e" dependencies = [ - "darling", + "darling 0.20.11", "proc-macro2", "quote", "syn 2.0.117", @@ -5056,6 +7085,28 @@ dependencies = [ "time", ] +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + [[package]] name = "yasna" version = "0.6.0" diff --git a/Cargo.toml b/Cargo.toml index 200b20fb3..2ce938b3e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "crates/algeon-datafusion", + "crates/algeon-iceberg", "crates/workloads", "crates/algeon-query-engine", "crates/algeon-query-planning", @@ -71,6 +72,7 @@ drop_non_drop = "deny" # `cuvs`/`cuvs-sys` by NVIDIA's own rapidsai/cuvs. The alias key stays bare so # Rust imports remain `use cudf::…` and `dep:` feature strings keep resolving. algeon-datafusion = { path = "crates/algeon-datafusion", version = "0.1.2", default-features = false } +algeon-iceberg = { path = "crates/algeon-iceberg" } workloads = { path = "crates/workloads" } query-engine = { package = "algeon-query-engine", path = "crates/algeon-query-engine", version = "=0.1.2", default-features = false } query-planning = { package = "algeon-query-planning", path = "crates/algeon-query-planning", version = "=0.1.2", default-features = false } @@ -102,6 +104,12 @@ datafusion = "55.1" fnv = "1" futures = "0.3" getrandom = "0.4" +# Pinned to the revision apache/datafusion-iceberg pins: the first iceberg-rust +# tree on Arrow 59. Only the unpublished `algeon-iceberg` crate depends on it. +iceberg = { git = "https://github.com/apache/iceberg-rust.git", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" } +iceberg-catalog-glue = { git = "https://github.com/apache/iceberg-rust.git", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" } +iceberg-catalog-rest = { git = "https://github.com/apache/iceberg-rust.git", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" } +iceberg-storage-opendal = { git = "https://github.com/apache/iceberg-rust.git", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" } indexmap = "2" libc = "0.2" object_store = { version = "0.13", default-features = false, features = ["aws"] } diff --git a/crates/algeon-datafusion/Cargo.toml b/crates/algeon-datafusion/Cargo.toml index c8b64ec8c..7b89c1595 100644 --- a/crates/algeon-datafusion/Cargo.toml +++ b/crates/algeon-datafusion/Cargo.toml @@ -29,6 +29,7 @@ server = [ "dep:dashmap", "dep:prost", "dep:rand", + "dep:rustls", "dep:serde", "dep:tokio-util", "dep:toml", @@ -77,8 +78,10 @@ serde_json = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread", "signal", "sync", "time"] } tokio-util = { workspace = true, optional = true } toml = { version = "1.1", optional = true } -# `tls-ring` reuses the ring provider rustls already builds with. A second -# provider in the graph makes rustls require a process-wide default. +# tonic builds its TLS acceptor from rustls's process-wide provider. Builds +# that also link the AWS SDK enable rustls's aws-lc-rs backend next to ring, +# and rustls then picks no default, so the server installs ring itself. +rustls = { version = "0.23", optional = true, default-features = false, features = ["ring", "std"] } tonic = { workspace = true, optional = true, features = ["tls-ring"] } tracing = { workspace = true } url = "2.5" diff --git a/crates/algeon-datafusion/src/attempt.rs b/crates/algeon-datafusion/src/attempt.rs index 6fa7203d6..2d3d0d933 100644 --- a/crates/algeon-datafusion/src/attempt.rs +++ b/crates/algeon-datafusion/src/attempt.rs @@ -35,7 +35,7 @@ use std::{ use arrow::{datatypes::SchemaRef, record_batch::RecordBatch}; use datafusion::{ common::{DataFusionError, Result as DataFusionResult}, - execution::{SessionState, SessionStateBuilder, context::TaskContext}, + execution::{SessionState, context::TaskContext}, physical_plan::{ ExecutionPlan, RecordBatchStream, SendableRecordBatchStream, execute_stream as execute_physical_stream, }, @@ -517,10 +517,10 @@ impl AlgeonAttempt { Ok(owner) } - fn invocation_state(&self, state: SessionState) -> SessionState { - let mut builder = SessionStateBuilder::new_from_existing(state); - self.invocation.install(builder.config().get_or_insert_with(Default::default)); - builder.build() + fn invocation_state(&self, mut state: SessionState) -> SessionState { + let runtime_env = Arc::clone(state.runtime_env()); + self.invocation.install(state.config_mut(), &runtime_env); + state } fn invocation_context(&self, ctx: &SessionContext) -> SessionContext { @@ -711,7 +711,7 @@ pub(crate) fn execute_in_internal_attempt( let owner = attempt.take_owner()?; attempt.state.set_sealed_metrics_target(index); let mut session_config = task_context.session_config().clone(); - attempt.invocation.install(&mut session_config); + attempt.invocation.install(&mut session_config, &task_context.runtime_env()); let task_context = Arc::new(TaskContext::new( task_context.task_id(), task_context.session_id(), diff --git a/crates/algeon-datafusion/src/capability.rs b/crates/algeon-datafusion/src/capability.rs index 42db2372c..2fef8e2e8 100644 --- a/crates/algeon-datafusion/src/capability.rs +++ b/crates/algeon-datafusion/src/capability.rs @@ -80,6 +80,13 @@ pub enum CapabilityReason { NativeSourceRowSelection, /// Source schema mapping cannot be represented natively. NativeSourceSchemaMapping, + /// The Parquet objects of one scan differ in column names, order, or + /// Arrow types, or one declares a column optional that the table declares + /// non-null. + NativeSourceFileSchemaMismatch, + /// Planning had no footer for a remote Parquet object of a scan that + /// needs one. + NativeSourceRemoteFooterUnavailable, } impl CapabilityReason { @@ -88,7 +95,7 @@ impl CapabilityReason { /// Internal diagnostics projections use this list to assert that a new /// reason cannot accidentally ship without actionable guidance. #[cfg(test)] - pub(crate) const ALL: [Self; 23] = [ + pub(crate) const ALL: [Self; 25] = [ Self::UnsupportedOperator, Self::UnsupportedExpression, Self::UnsupportedType, @@ -112,6 +119,8 @@ impl CapabilityReason { Self::NativeSourceByteRange, Self::NativeSourceRowSelection, Self::NativeSourceSchemaMapping, + Self::NativeSourceFileSchemaMismatch, + Self::NativeSourceRemoteFooterUnavailable, ]; /// Stable snake-case reason code used by reports and tests. @@ -261,6 +270,22 @@ impl CapabilityReason { Self::NativeSourceByteRange => source_gap("native_source_byte_range"), Self::NativeSourceRowSelection => source_gap("native_source_row_selection"), Self::NativeSourceSchemaMapping => source_gap("native_source_schema_mapping"), + Self::NativeSourceFileSchemaMismatch => ReasonFacts { + remedy: QueryGpuRemedy::new( + "unify_file_schemas", + QueryGpuRemedyKind::UserAction, + "The scan's Parquet files differ in column names, column order, or Arrow types, or a file declares a column optional that the table declares non-null. Rewrite the files to one schema, or declare the column nullable in the table.", + ), + ..source_gap("native_source_file_schema_mismatch") + }, + Self::NativeSourceRemoteFooterUnavailable => ReasonFacts { + remedy: QueryGpuRemedy::new( + "cache_remote_footers", + QueryGpuRemedyKind::ConfigAction, + "Planning reads remote Parquet footers from the session's DataFusion file metadata cache, which the table scan fills while planning. Plan through an `AlgeonAttempt` or the Flight SQL server and keep `datafusion.execution.collect_statistics` enabled. A table whose footers exceed the cache limit (`datafusion.runtime.metadata_cache_limit`, 50 MB by default) evicts its own entries; an embedded session can raise the limit, the Flight SQL server keeps the default.", + ), + ..source_gap("native_source_remote_footer_unavailable") + }, } } } diff --git a/crates/algeon-datafusion/src/error/code.rs b/crates/algeon-datafusion/src/error/code.rs index 39352bec1..9c2e761cc 100644 --- a/crates/algeon-datafusion/src/error/code.rs +++ b/crates/algeon-datafusion/src/error/code.rs @@ -110,6 +110,9 @@ error_codes! { UnsupportedExpression => "unsupported_expression", Unsupported, Permanent; /// A data type, physical representation, or type combination is unsupported. UnsupportedType => "unsupported_type", Unsupported, Permanent; + /// A table uses a table-format feature that Algeon does not read, such as + /// delete files; the `reason` fact names the feature. + UnsupportedTableFeature => "unsupported_table_feature", Unsupported, Permanent; /// A DataFusion physical plan is invalid for the adapter boundary. /// /// Use this only for malformed or inconsistent plans. A well-formed plan diff --git a/crates/algeon-datafusion/src/error/tests.rs b/crates/algeon-datafusion/src/error/tests.rs index 8828f9e71..928c13329 100644 --- a/crates/algeon-datafusion/src/error/tests.rs +++ b/crates/algeon-datafusion/src/error/tests.rs @@ -627,6 +627,7 @@ const ADAPTER_STATUS_CASES: &[(ErrorCode, ErrorStatus)] = &[ (ErrorCode::UnsupportedOperator, ErrorStatus::Permanent), (ErrorCode::UnsupportedExpression, ErrorStatus::Permanent), (ErrorCode::UnsupportedType, ErrorStatus::Permanent), + (ErrorCode::UnsupportedTableFeature, ErrorStatus::Permanent), (ErrorCode::InvalidExecutionPlan, ErrorStatus::Permanent), (ErrorCode::InvalidChildCount, ErrorStatus::Permanent), (ErrorCode::CudfInvalidInput, ErrorStatus::Permanent), diff --git a/crates/algeon-datafusion/src/gpu_coverage/mod.rs b/crates/algeon-datafusion/src/gpu_coverage/mod.rs index 691c0d9ef..27173e873 100644 --- a/crates/algeon-datafusion/src/gpu_coverage/mod.rs +++ b/crates/algeon-datafusion/src/gpu_coverage/mod.rs @@ -30,6 +30,8 @@ pub(crate) use self::{ #[cfg(test)] mod tests; +use std::sync::Arc; + use datafusion::{execution::SessionStateBuilder, prelude::SessionContext}; use futures::future::BoxFuture; @@ -80,8 +82,9 @@ pub fn validate_query<'a>( let invocation = Invocation::detached(); let mut planning_state_builder = SessionStateBuilder::new_from_existing(ctx.state()); mark_coverage_validation_in_progress(&mut planning_state_builder); - invocation.install(planning_state_builder.config().get_or_insert_with(Default::default)); - let planning_state = planning_state_builder.build(); + let mut planning_state = planning_state_builder.build(); + let runtime_env = Arc::clone(planning_state.runtime_env()); + invocation.install(planning_state.config_mut(), &runtime_env); let config = optimizer_config_for_validation(&planning_state)?; let validated_under = validated_under(&planning_state, &config); if let Some(rejection) = preflight_single_query(&planning_state, sql) { diff --git a/crates/algeon-datafusion/src/native/footers.rs b/crates/algeon-datafusion/src/native/footers.rs index 7be6c487c..4f4cdc365 100644 --- a/crates/algeon-datafusion/src/native/footers.rs +++ b/crates/algeon-datafusion/src/native/footers.rs @@ -1,11 +1,18 @@ //! Parquet footers shared by the source-lowering steps of one planning run. //! -//! One lowering reads the footer of a local file from several places (schema +//! One lowering reads the footer of a scan object from several places (schema //! mapping, byte-range coalescing, row-group pruning, byte metadata, source //! diagnostics), and one optimizer run lowers the same plan more than once //! (whole plan, mixed leaf, GPU relation inputs, graph residency). A //! [`FooterScope`] entered on the planning thread lets every read in that -//! window share one parse per file. Reads outside a scope parse independently. +//! window share one parse per local file. Reads outside a scope parse +//! independently. +//! +//! A remote object's footer comes from the planning session's DataFusion file +//! metadata cache, which a `ListingTable` or table provider fills during its +//! async `scan`. Physical optimizer rules run synchronously on the caller's +//! Tokio worker, so planning never reads a remote footer from the store; an +//! object missing from that cache has no footer here. #[cfg(test)] use std::sync::atomic::{AtomicUsize, Ordering}; @@ -19,11 +26,18 @@ use std::{ time::SystemTime, }; +use datafusion::{ + datasource::physical_plan::parquet::metadata::CachedParquetMetaData, + execution::{cache::cache_manager::FileMetadataCache, object_store::ObjectStoreUrl}, + object_store::ObjectMeta, +}; use parquet::{ errors::Result as ParquetResult, file::metadata::{ParquetMetaData, ParquetMetaDataReader}, }; +use super::pruning::local_object_path_to_filesystem; + /// Footer bytes one scope retains. Every source step walks all files of a /// scan, so an unbounded cache would hold the footer of every file in a wide /// table at once. Past the budget a footer is parsed on each request. @@ -46,14 +60,25 @@ pub(crate) fn footer_parses_on_this_thread() -> usize { /// Footers keyed by path, size, and modification time, so a file rewritten /// during the scope misses the cache and is parsed again. -#[derive(Debug)] pub(crate) struct FooterCache { state: Mutex, budget_bytes: usize, + /// The planning session's DataFusion file metadata cache, which serves + /// remote footers. + remote: Option>, #[cfg(test)] parses: AtomicUsize, } +impl std::fmt::Debug for FooterCache { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("FooterCache") + .field("budget_bytes", &self.budget_bytes) + .field("remote", &self.remote.is_some()) + .finish_non_exhaustive() + } +} + #[derive(Debug, Default)] struct CacheState { footers: HashMap>, @@ -68,20 +93,43 @@ struct FooterKey { } impl FooterCache { - /// A cache with the budget every planning scope retains. - pub(crate) fn for_scope() -> Self { - Self::new(FOOTER_CACHE_BUDGET_BYTES) + /// A cache with the budget every planning scope retains. `remote` is the + /// planning session's file metadata cache; without it, remote objects + /// have no footer. + pub(crate) fn for_scope(remote: Option>) -> Self { + Self { remote, ..Self::new(FOOTER_CACHE_BUDGET_BYTES) } } fn new(budget_bytes: usize) -> Self { Self { state: Mutex::new(CacheState::default()), budget_bytes, + remote: None, #[cfg(test)] parses: AtomicUsize::new(0), } } + /// The cached footer of `object`. DataFusion keys the cache by the path + /// inside the store, without the bucket, and validates an entry by size + /// and modification time; an entry whose e_tag or version differs from + /// the object's is also refused. + fn remote_footer(&self, object: &ObjectMeta) -> Option> { + let entry = self.remote.as_ref()?.get(&object.location)?; + let same_identity = |cached: &Option, current: &Option| match (cached, current) { + (Some(cached), Some(current)) => cached == current, + _ => true, + }; + if !entry.is_valid_for(object) + || !same_identity(&entry.meta.e_tag, &object.e_tag) + || !same_identity(&entry.meta.version, &object.version) + { + return None; + } + let footer = entry.file_metadata.as_any().downcast_ref::()?; + Some(Arc::clone(footer.parquet_metadata())) + } + fn footer(&self, path: &Path) -> ParquetResult> { let file_metadata = fs::metadata(path)?; let key = @@ -125,7 +173,7 @@ pub(crate) struct FooterScope { impl FooterScope { pub(crate) fn enter() -> Self { - Self::install_with(|| Arc::new(FooterCache::for_scope())) + Self::install_with(|| Arc::new(FooterCache::for_scope(None))) } /// Installs `cache` unless a scope is already active on this thread, so a @@ -169,6 +217,33 @@ pub(crate) fn parquet_footer(path: &Path) -> ParquetResult> } } +/// Why planning has no footer for a scan object. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum FooterUnavailable { + /// The local file could not be opened or parsed. + Local, + /// The planning session's file metadata cache holds no footer for this + /// version of the remote object, or the active scope has no such cache. + RemoteNotCached, +} + +/// Returns the footer of the Parquet object `object` in the store at +/// `object_store_url`: a `file://` object is parsed from the local file, any +/// other object is read from the active scope's remote cache. +pub(crate) fn object_parquet_footer( + object_store_url: &ObjectStoreUrl, + object: &ObjectMeta, +) -> Result, FooterUnavailable> { + if object_store_url.as_str().starts_with("file://") { + let path = local_object_path_to_filesystem(object.location.as_ref()); + return parquet_footer(path.as_path()).map_err(|_error| FooterUnavailable::Local); + } + ACTIVE_CACHE + .with_borrow(Clone::clone) + .and_then(|cache| cache.remote_footer(object)) + .ok_or(FooterUnavailable::RemoteNotCached) +} + #[cfg(test)] mod tests { use arrow::{ @@ -220,4 +295,46 @@ mod tests { assert_eq!(retained.parses(), 1); assert_eq!(unretained.parses(), 2); } + + #[tokio::test] + async fn remote_footers_come_from_the_session_cache_for_the_same_object_version() { + use datafusion::{ + datasource::physical_plan::parquet::metadata::DFParquetMetadata, + execution::runtime_env::RuntimeEnv, + object_store::{ObjectStoreExt, memory::InMemory, path::Path as ObjectStorePath}, + }; + + let mut bytes = Vec::new(); + let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Int64, false)])); + let batch = + RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(Int64Array::from_iter_values(0..4))]).unwrap(); + let props = WriterProperties::builder().set_max_row_group_row_count(Some(2)).build(); + let mut writer = ArrowWriter::try_new(&mut bytes, schema, Some(props)).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + let store = InMemory::new(); + let location = ObjectStorePath::from("table/part-0.parquet"); + store.put(&location, bytes.into()).await.unwrap(); + let object = store.head(&location).await.unwrap(); + let session_cache = RuntimeEnv::default().cache_manager.get_file_metadata_cache(); + DFParquetMetadata::new(&store, &object) + .with_file_metadata_cache(Some(Arc::clone(&session_cache))) + .fetch_metadata() + .await + .unwrap(); + let s3 = ObjectStoreUrl::parse("s3://bucket").unwrap(); + let rewritten = ObjectMeta { e_tag: Some("rewritten".to_string()), ..object.clone() }; + + { + let _scope = FooterScope::enter(); + assert_eq!(object_parquet_footer(&s3, &object).unwrap_err(), FooterUnavailable::RemoteNotCached); + } + let _scope = FooterScope::join(Arc::new(FooterCache::for_scope(Some(session_cache)))); + assert_eq!(object_parquet_footer(&s3, &object).unwrap().num_row_groups(), 2); + assert_eq!( + object_parquet_footer(&s3, &rewritten).unwrap_err(), + FooterUnavailable::RemoteNotCached, + "an entry cached for another e_tag of the object must not serve its footer" + ); + } } diff --git a/crates/algeon-datafusion/src/native/lowering/source.rs b/crates/algeon-datafusion/src/native/lowering/source.rs index 84b80ed10..5bb00c7c8 100644 --- a/crates/algeon-datafusion/src/native/lowering/source.rs +++ b/crates/algeon-datafusion/src/native/lowering/source.rs @@ -6,6 +6,7 @@ use std::{ use arrow::datatypes::{Schema, SchemaRef}; use datafusion::{ common::Constraint, + datasource::{physical_plan::ParquetSource, source::DataSourceExec}, physical_expr::{PhysicalExpr, expressions::Column, projection::ProjectionExprs}, physical_plan::{ExecutionPlan, filter::FilterExec}, }; @@ -13,6 +14,7 @@ use query_planning::{ expr::Expr as NativeExpr, plan::{NodeId, QueryPlanBuilder as NativeQueryPlanBuilder}, source::SourceDiagnostics as NativeSourceDiagnostics, + triage::TriageCode, }; use crate::native::ParquetReadStrategyConfig; @@ -34,6 +36,7 @@ use super::{ use crate::{ capability::CapabilityReason, native::{LoweringFailure, normalize::NormalizedNativeSource}, + report::PlanBlocker, }; pub(super) fn lower_source_input( @@ -42,7 +45,36 @@ pub(super) fn lower_source_input( demand: &Demand, source_reads: &ParquetReadStrategyConfig, ) -> Result { - lower_local_parquet_source_input(source.scan, source.filter, builder, demand, source_reads) + lower_local_parquet_source_input(source.scan, source.filter, builder, demand, source_reads).map_err(|failure| { + match failure { + // A source rejection names the scan, so a `NativeRequired` error + // carries the reason and its subject. + LoweringFailure::Rejected(rejection) + if rejection.reason().facts().triage == TriageCode::UnsupportedSource => + { + LoweringFailure::Rejected(rejection.or_blocked_by(|| source_blocker(source.scan))) + }, + failure => failure, + } + }) +} + +fn source_blocker(scan: &DataSourceExec) -> PlanBlocker { + match scan.downcast_to_file_source::() { + Some((file_scan, _)) => { + let objects = file_scan + .file_groups + .iter() + .flat_map(|group| group.files()) + .map(|file| &file.object_meta.location) + .collect::>(); + PlanBlocker::Source { + object_store_url: file_scan.object_store_url.as_str().to_string(), + object_count: objects.len(), + } + }, + None => PlanBlocker::Operator(scan.name().to_string()), + } } pub(super) fn combine_scan_limits(left: Option, right: Option) -> Option { diff --git a/crates/algeon-datafusion/src/native/lowering/source/local_parquet.rs b/crates/algeon-datafusion/src/native/lowering/source/local_parquet.rs index 48f398fdf..1e31e0c57 100644 --- a/crates/algeon-datafusion/src/native/lowering/source/local_parquet.rs +++ b/crates/algeon-datafusion/src/native/lowering/source/local_parquet.rs @@ -198,6 +198,7 @@ pub(super) fn lower_local_parquet_source_input( Arc::clone(&table_schema), Some(&column_requirements), SourceRead { reader_pushdown, row_count_only: false, ordered: demand.ordered }, + source_reads, ); preserve_resolved_source_diagnostics(&mut native_source.diagnostics, &source_contract_diagnostics); if source_scan_limit.is_none() && post_predicate_scan_limit.is_some() { @@ -307,6 +308,7 @@ fn lower_local_parquet_row_count( table_schema, Some(&column_requirements), SourceRead { reader_pushdown: false, row_count_only: true, ordered: false }, + source_reads, ); preserve_resolved_source_diagnostics(&mut source.diagnostics, &source_contract_diagnostics); source.diagnostics.selected_row_group_stats = selected_row_group_stats; diff --git a/crates/algeon-datafusion/src/native/lowering_tests/source_contracts.rs b/crates/algeon-datafusion/src/native/lowering_tests/source_contracts.rs index 49e2bc3da..521805c9b 100644 --- a/crates/algeon-datafusion/src/native/lowering_tests/source_contracts.rs +++ b/crates/algeon-datafusion/src/native/lowering_tests/source_contracts.rs @@ -1,11 +1,24 @@ +use arrow::{ + array::{Int32Array, Int64Array}, + record_batch::RecordBatch, +}; use cudf::testing; +use datafusion::{ + execution::SessionStateBuilder, + object_store::{ObjectStoreExt, memory::InMemory, path::Path as ObjectStorePath}, + parquet::arrow::ArrowWriter, + physical_optimizer::PhysicalOptimizerRule, +}; use query_planning::source::CredentialMode; use query_runtime::runtime::{S3SourceStorageProfile, SourceStorageProfile}; use super::*; use crate::{ + error::DataFusionErrorExt, native::{ParquetReadStrategyConfig, S3_STORAGE_PROFILE_ID}, native_exec::AlgeonNativeExecutionConfig, + planner::{AlgeonGpuExecutionMode, plan_with_native_diagnostics}, + session::AlgeonSessionStateBuilderExt, }; fn pushed_down_parquet_predicate_with_filter_is_reader_pushdown() { @@ -991,26 +1004,236 @@ fn s3_parquet_object_store_scan_lowers_to_remote_native_source() { } #[test] -fn s3_multi_file_parquet_object_store_scan_rejects_unverified_footer_schema() { +fn s3_multi_file_parquet_scan_reads_footers_from_the_session_cache() { let dir = unique_temp_dir("algeon-datafusion-native-lowering-s3-store-multi-file"); - let first_path = dir.join("part-0.parquet"); - let second_path = dir.join("part-1.parquet"); - write_numeric_parquet_file(&first_path); - write_numeric_parquet_file(&second_path); - let scan = parquet_scan_with_object_store_url(&dir, ObjectStoreUrl::parse("s3://bucket").unwrap()); + write_numeric_parquet_file(&dir.join("part-0.parquet")); + write_numeric_parquet_file(&dir.join("part-1.parquet")); + let (scan, footers) = + remote_scan_with_cached_footers(parquet_scan(&dir), ObjectStoreUrl::parse("s3://bucket").unwrap()); + let _footers = remote_footer_scope(footers); + + let result = lower_physical_plan_to_native(scan, &ParquetReadStrategyConfig::default()).unwrap(); + + assert_eq!(result.selected_count(), 1, "outcomes={:?}", result.report.outcomes()); + let source = native_local_parquet_source(result.selected_plan()); + assert_eq!(source.file_count(), 2); + for file in &source.files { + assert_eq!(file.location.read_capability(), "object_store:s3"); + assert_eq!(file.row_group_rows.as_deref(), Some(&[3][..]), "the cached footer supplies row-group metadata"); + } + assert!(result.source_diagnostics[0].rejected_features.is_empty()); + assert_eq!(result.source_diagnostics[0].source_read_strategy, "remote_direct"); + + fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn s3_multi_file_parquet_scan_without_cached_footers_names_the_missing_remote_footer() { + let dir = unique_temp_dir("algeon-datafusion-native-lowering-s3-store-uncached"); + write_numeric_parquet_file(&dir.join("part-0.parquet")); + write_numeric_parquet_file(&dir.join("part-1.parquet")); + let (scan, _uncached) = + remote_scan_with_cached_footers(parquet_scan(&dir), ObjectStoreUrl::parse("s3://bucket").unwrap()); + + let result = lower_physical_plan_to_native(scan, &ParquetReadStrategyConfig::default()).unwrap(); + + assert!(result.plan.is_none()); + let CandidateOutcome::NotSupported { capability_reason, detail, .. } = &result.report.outcomes()[0] else { + panic!("outcomes={:?}", result.report.outcomes()); + }; + assert_eq!(*capability_reason, CapabilityReason::NativeSourceRemoteFooterUnavailable); + let blocking = detail.blocking().expect("a source rejection names its scan"); + assert_eq!(blocking.kind(), "source"); + assert_eq!(blocking.text(), "Parquet scan of 2 objects in `s3://bucket/`"); + let diagnostics = &result.source_diagnostics[0]; + assert_eq!( + diagnostics.rejected_features.iter().map(|feature| feature.as_str()).collect::>(), + ["remote_footer_unavailable"] + ); + assert_eq!(diagnostics.source_read_strategy, "remote_direct"); + assert_eq!(diagnostics.credential_mode, "shared_process_env"); + assert!(!diagnostics.direct_device_read); + + fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn native_required_error_names_the_missing_remote_footer() { + let dir = unique_temp_dir("algeon-datafusion-native-lowering-s3-required-uncached"); + write_numeric_parquet_file(&dir.join("part-0.parquet")); + write_numeric_parquet_file(&dir.join("part-1.parquet")); + let (scan, _uncached) = + remote_scan_with_cached_footers(parquet_scan(&dir), ObjectStoreUrl::parse("s3://bucket").unwrap()); + let rule = AlgeonNativePhysicalOptimizerRule::new( + AlgeonNativeOptimizerConfig::default().with_execution_mode(AlgeonGpuExecutionMode::NativeRequired), + ); + + let failure = rule + .optimize(aggregate_sort_limit_plan(scan, Operator::Plus), &ConfigOptions::default()) + .expect_err("NativeRequired rejects the CPU scan"); + + let error = failure.find_algeon_error().expect("typed adapter error"); + assert_eq!(error.fact_value("reason"), Some("native_source_remote_footer_unavailable")); + assert_eq!(error.fact_value("blocking_kind"), Some("source")); + assert_eq!( + error.client_detail().and_then(|detail| detail.value("blocking_subject")), + Some("Parquet scan of 2 objects in `s3://bucket/`") + ); + + fs::remove_dir_all(dir).unwrap(); +} + +#[tokio::test] +async fn remote_listing_table_footers_reach_native_planning_through_the_session() { + let dir = unique_temp_dir("algeon-datafusion-native-lowering-s3-listing-table"); + let path = dir.join("input.parquet"); + write_numeric_parquet_file(&path); + let store = Arc::new(InMemory::new()); + for name in ["part-0.parquet", "part-1.parquet"] { + store.put(&ObjectStorePath::from(format!("table/{name}")), fs::read(&path).unwrap().into()).await.unwrap(); + } + let state = SessionStateBuilder::new_with_default_features() + .with_config(SessionConfig::new().with_target_partitions(1)) + .with_algeon_native(AlgeonNativeOptimizerConfig::default()) + .build(); + state.runtime_env().register_object_store(ObjectStoreUrl::parse("s3://bucket").unwrap().as_ref(), store); + let ctx = SessionContext::new_with_state(state); + ctx.register_parquet("t", "s3://bucket/table/", ParquetReadOptions::default()).await.unwrap(); + let logical = + ctx.sql("SELECT key, SUM(m00) AS total FROM t GROUP BY key").await.unwrap().into_optimized_plan().unwrap(); + + let (_plan, diagnostics) = plan_with_native_diagnostics(&ctx.state(), &logical).await.unwrap(); + + assert!( + diagnostics.report.outcomes().iter().any(|outcome| matches!(outcome, CandidateOutcome::Selected { .. })), + "outcomes={:?}", + diagnostics.report.outcomes() + ); + assert_eq!(diagnostics.source_diagnostics[0].file_count, 2); + assert!(diagnostics.source_diagnostics[0].rejected_features.is_empty()); + + fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn s3_split_full_file_byte_ranges_coalesce_without_a_footer() { + let dir = unique_temp_dir("algeon-datafusion-native-lowering-s3-split-ranged-file"); + let path = dir.join("input.parquet"); + write_numeric_parquet_file(&path); + let (scan, _uncached) = remote_scan_with_cached_footers( + parquet_scan_with_split_full_byte_ranges(&path), + ObjectStoreUrl::parse("s3://bucket").unwrap(), + ); + let plan = aggregate_sort_limit_plan(scan, Operator::Plus); + + let result = lower_physical_plan_to_native(plan, &ParquetReadStrategyConfig::default()).unwrap(); + + assert_eq!(result.selected_count(), 1, "outcomes={:?}", result.report.outcomes()); + let source = native_local_parquet_source(result.selected_plan()); + assert_eq!(source.file_count(), 1); + assert_eq!(source.files[0].resolved_source.original_location, "s3://bucket/table/input.parquet"); + assert_eq!(result.source_diagnostics[0].ranged_file_count, 2); + assert_eq!(result.source_diagnostics[0].byte_range_disposition.as_str(), "coalesced_full_files"); + assert!(result.source_diagnostics[0].rejected_features.is_empty()); + + fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn s3_partial_byte_range_selects_row_groups_from_the_cached_footer() { + let dir = unique_temp_dir("algeon-datafusion-native-lowering-s3-partial-ranged-file"); + let path = dir.join("input.parquet"); + write_row_group_pruning_parquet_file(&path); + let offsets = row_group_range_pruning_offsets(&path); + let (scan, footers) = remote_scan_with_cached_footers( + parquet_scan_with_explicit_byte_ranges(&path, vec![(offsets[0], offsets[1])]), + ObjectStoreUrl::parse("s3://bucket").unwrap(), + ); + let _footers = remote_footer_scope(footers); + let plan = aggregate_sort_limit_plan(scan, Operator::Plus); + + let result = lower_physical_plan_to_native(plan, &ParquetReadStrategyConfig::default()).unwrap(); + + assert_eq!(result.selected_count(), 1, "outcomes={:?}", result.report.outcomes()); + let source = native_local_parquet_source(result.selected_plan()); + assert_eq!(source.files[0].row_groups.as_deref(), Some(&[0][..])); + assert_eq!(result.source_diagnostics[0].byte_range_disposition.as_str(), "row_group_selection"); + assert!(result.source_diagnostics[0].rejected_features.is_empty()); + + fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn files_that_differ_only_in_nullability_lower_under_a_nullable_table_column() { + let dir = unique_temp_dir("algeon-datafusion-native-lowering-nullability-widening"); + write_single_int64_column_parquet_file(&dir.join("part-0.parquet"), Field::new("value", DataType::Int64, false)); + write_single_int64_column_parquet_file(&dir.join("part-1.parquet"), Field::new("value", DataType::Int64, true)); + let scan = parquet_scan(&dir); + assert!(scan.schema().field_with_name("value").unwrap().is_nullable(), "DataFusion widens the merged column"); + + let result = lower_physical_plan_to_native(scan, &ParquetReadStrategyConfig::default()).unwrap(); + + assert_eq!(result.selected_count(), 1, "outcomes={:?}", result.report.outcomes()); + assert_eq!(native_local_parquet_source(result.selected_plan()).file_count(), 2); + + fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn files_with_different_column_types_are_a_file_schema_mismatch() { + let dir = unique_temp_dir("algeon-datafusion-native-lowering-type-mismatch"); + let first = dir.join("part-0.parquet"); + let second = dir.join("part-1.parquet"); + write_single_int64_column_parquet_file(&first, Field::new("value", DataType::Int64, false)); + write_single_int32_column_parquet_file(&second); + let scan = parquet_scan_with_extra_local_file(parquet_scan(&first), &second); let result = lower_physical_plan_to_native(scan, &ParquetReadStrategyConfig::default()).unwrap(); assert!(result.plan.is_none()); - assert_eq!(result.not_selected_count(), 1); assert!(matches!( result.report.outcomes()[0], - CandidateOutcome::NotSupported { capability_reason: CapabilityReason::NativeSourceSchemaMapping, .. } + CandidateOutcome::NotSupported { capability_reason: CapabilityReason::NativeSourceFileSchemaMismatch, .. } )); + let diagnostics = &result.source_diagnostics[0]; + assert_eq!(diagnostics.schema_mapping_disposition.as_str(), "rejected"); + assert!(diagnostics.rejected_features.iter().any(|feature| feature.as_str() == "schema_mapping")); fs::remove_dir_all(dir).unwrap(); } +fn write_single_int64_column_parquet_file(path: &std::path::Path, field: Field) { + let schema = Arc::new(Schema::new(vec![field])); + let batch = RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(Int64Array::from(vec![1, 2, 3]))]).unwrap(); + let mut writer = ArrowWriter::try_new(fs::File::create(path).unwrap(), schema, None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); +} + +fn write_single_int32_column_parquet_file(path: &std::path::Path) { + let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Int32, false)])); + let batch = RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(Int32Array::from(vec![1, 2, 3]))]).unwrap(); + let mut writer = ArrowWriter::try_new(fs::File::create(path).unwrap(), schema, None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); +} + +/// `scan` with the local Parquet file at `path` appended to its first file +/// group, past DataFusion's schema inference. +fn parquet_scan_with_extra_local_file(scan: Arc, path: &std::path::Path) -> Arc { + let data_source = scan.downcast_ref::().expect("parquet scan should be DataSourceExec"); + let (file_scan, _parquet_source) = + data_source.downcast_to_file_source::().expect("parquet scan should use ParquetSource"); + let mut files = file_scan.file_groups[0].files().to_vec(); + let mut extra = files[0].clone(); + extra.object_meta.location = path.to_str().unwrap().trim_start_matches('/').into(); + extra.object_meta.size = fs::metadata(path).unwrap().len(); + files.push(extra); + let file_scan = + FileScanConfigBuilder::from(file_scan.clone()).with_file_groups(vec![FileGroup::new(files)]).build(); + DataSourceExec::from_data_source(file_scan) +} + #[test] fn s3_parquet_scan_reads_through_the_configured_s3_storage() { let dir = unique_temp_dir("algeon-datafusion-native-lowering-s3-storage"); diff --git a/crates/algeon-datafusion/src/native/lowering_tests/source_fixtures.rs b/crates/algeon-datafusion/src/native/lowering_tests/source_fixtures.rs index 8160e5f43..a5810070d 100644 --- a/crates/algeon-datafusion/src/native/lowering_tests/source_fixtures.rs +++ b/crates/algeon-datafusion/src/native/lowering_tests/source_fixtures.rs @@ -1,12 +1,18 @@ use datafusion::{ common::{DataFusionError, Result as DataFusionResult}, - datasource::{listing::PartitionedFile, physical_plan::ParquetFileReaderFactory}, + datasource::{ + listing::PartitionedFile, + physical_plan::{ParquetFileReaderFactory, parquet::metadata::DFParquetMetadata}, + }, + execution::{cache::cache_manager::FileMetadataCache, runtime_env::RuntimeEnv}, + object_store::{ObjectStoreExt, memory::InMemory}, parquet::arrow::async_reader::AsyncFileReader, physical_plan::metrics::ExecutionPlanMetricsSet, }; use futures::executor::block_on; use super::*; +use crate::native::FooterCache; pub(super) fn parquet_scan(path: &std::path::Path) -> Arc { let ctx = SessionContext::new_with_config( @@ -950,6 +956,49 @@ pub(super) fn parquet_scan_with_object_store_url_and_location( DataSourceExec::from_data_source(file_scan) } +/// Moves the objects of `scan`, a scan over local files, to keys +/// `table/` of an in-memory store at `object_store_url`, so no +/// local read can reach them. Returns the moved scan and a file metadata cache +/// holding every object's footer, as DataFusion's async scan of a remote table +/// leaves the session cache. +pub(super) fn remote_scan_with_cached_footers( + scan: Arc, + object_store_url: ObjectStoreUrl, +) -> (Arc, Arc) { + let data_source = scan.downcast_ref::().expect("parquet scan should be DataSourceExec"); + let (file_scan, _parquet_source) = + data_source.downcast_to_file_source::().expect("parquet scan should use ParquetSource"); + let store = InMemory::new(); + let cache = RuntimeEnv::default().cache_manager.get_file_metadata_cache(); + let mut remote_file = |mut file: PartitionedFile| { + let local_path = std::path::Path::new("/").join(file.object_meta.location.as_ref()); + let name = local_path.file_name().unwrap().to_str().unwrap(); + file.object_meta.location = format!("table/{name}").into(); + block_on(store.put(&file.object_meta.location, fs::read(&local_path).unwrap().into())).unwrap(); + block_on( + DFParquetMetadata::new(&store, &file.object_meta) + .with_file_metadata_cache(Some(Arc::clone(&cache))) + .fetch_metadata(), + ) + .unwrap(); + file + }; + let file_groups = file_scan + .file_groups + .iter() + .map(|group| FileGroup::new(group.files().iter().cloned().map(&mut remote_file).collect())) + .collect(); + let file_scan = FileScanConfigBuilder::new(object_store_url, Arc::clone(file_scan.file_source())) + .with_file_groups(file_groups) + .build(); + (DataSourceExec::from_data_source(file_scan), cache) +} + +/// A footer scope whose remote footers come from `cache`. +pub(super) fn remote_footer_scope(cache: Arc) -> FooterScope { + FooterScope::join(Arc::new(FooterCache::for_scope(Some(cache)))) +} + pub(super) fn parquet_scan_with_partition_columns(path: &std::path::Path) -> Arc { parquet_scan_with_partition_field(path, Field::new("part", DataType::Int32, false)) } diff --git a/crates/algeon-datafusion/src/native/lowering_tests/sql_aggregate_shapes/pruning.rs b/crates/algeon-datafusion/src/native/lowering_tests/sql_aggregate_shapes/pruning.rs index e8ec726a9..592121a0c 100644 --- a/crates/algeon-datafusion/src/native/lowering_tests/sql_aggregate_shapes/pruning.rs +++ b/crates/algeon-datafusion/src/native/lowering_tests/sql_aggregate_shapes/pruning.rs @@ -271,3 +271,44 @@ async fn direct_friendly_sql_nested_prefix_pruning_uses_parquet_leaf_stats() { fs::remove_dir_all(dir).unwrap(); } + +#[tokio::test] +async fn row_group_pruning_reads_statistics_of_the_named_file_column_when_the_table_reorders_columns() { + use arrow::{array::Int64Array, record_batch::RecordBatch}; + use datafusion::parquet::{arrow::ArrowWriter, file::properties::WriterProperties}; + + let dir = unique_temp_dir("algeon-datafusion-native-lowering-reordered-pruning"); + let path = dir.join("input.parquet"); + // The file stores `a` before `b`; the table declares `b` first. + let schema = + Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false), Field::new("b", DataType::Int64, false)])); + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(Int64Array::from(vec![0, 1, 10, 11])), Arc::new(Int64Array::from(vec![100, 101, 0, 1]))], + ) + .unwrap(); + let props = WriterProperties::builder().set_max_row_group_row_count(Some(2)).build(); + let mut writer = ArrowWriter::try_new(fs::File::create(&path).unwrap(), schema, Some(props)).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + let ctx = SessionContext::new_with_config(direct_friendly_sql_session_config()); + ctx.sql(&format!( + "CREATE EXTERNAL TABLE t (b BIGINT NOT NULL, a BIGINT NOT NULL) STORED AS PARQUET LOCATION '{}'", + path.display() + )) + .await + .unwrap(); + let plan = ctx.sql("SELECT SUM(a) FROM t WHERE b <= 1").await.unwrap().create_physical_plan().await.unwrap(); + + let result = lower_physical_plan_to_native(plan, &ParquetReadStrategyConfig::default()).unwrap(); + + assert_eq!(result.selected_count(), 1, "outcomes={:?}", result.report.outcomes()); + let source = native_local_parquet_source(result.selected_plan()); + assert_eq!( + source.files[0].row_groups.as_deref(), + Some(&[1][..]), + "only the second row group holds `b <= 1`; the first holds `a <= 1`" + ); + + fs::remove_dir_all(dir).unwrap(); +} diff --git a/crates/algeon-datafusion/src/native/mod.rs b/crates/algeon-datafusion/src/native/mod.rs index cb353b8aa..1387bc6ad 100644 --- a/crates/algeon-datafusion/src/native/mod.rs +++ b/crates/algeon-datafusion/src/native/mod.rs @@ -106,6 +106,14 @@ impl LoweringRejection { Self { reason, cause: Some(Box::new(RejectionCause::Blocker(blocker))) } } + /// This rejection with `blocker` as its cause when it carries none. + pub(crate) fn or_blocked_by(self, blocker: impl FnOnce() -> PlanBlocker) -> Self { + match self.cause { + Some(_) => self, + None => Self::blocked_by(self.reason, blocker()), + } + } + /// The reason and the evidence this rejection carries, with the unsupported /// `column` the caller found in the plan schemas. pub(crate) fn into_detail(self, column: Option) -> (CapabilityReason, NotSupportedDetail) { @@ -299,7 +307,7 @@ fn lower_physical_plan_to_native_with_mode( Ok(NativeLoweringResult { plan: None, report, - source_diagnostics: source_diagnostics_for_normalized_plan(&normalized), + source_diagnostics: source_diagnostics_for_normalized_plan(&normalized, source_reads), }) }, Ok(normalized) => { @@ -319,7 +327,7 @@ fn lower_physical_plan_to_native_with_mode( Ok(NativeLoweringResult { plan: None, report, - source_diagnostics: source_diagnostics_for_normalized_plan(&normalized), + source_diagnostics: source_diagnostics_for_normalized_plan(&normalized, source_reads), }) }, Ok(mut native_plan) => { diff --git a/crates/algeon-datafusion/src/native/pruning.rs b/crates/algeon-datafusion/src/native/pruning.rs index 723b3da37..eaae676bb 100644 --- a/crates/algeon-datafusion/src/native/pruning.rs +++ b/crates/algeon-datafusion/src/native/pruning.rs @@ -17,6 +17,7 @@ use datafusion::{ parquet::{ParquetAccessPlan, RowGroupAccess}, }, }, + execution::object_store::ObjectStoreUrl, logical_expr::Operator, parquet::file::{ metadata::{ColumnChunkMetaData, RowGroupMetaData}, @@ -33,7 +34,10 @@ use datafusion::{ }; use query_planning::{expr::BinaryOp, source::PredicateDisposition as NativePredicateDisposition}; -use super::{expression_contract::validate_physical_expr_contract, footers::parquet_footer}; +use super::{ + expression_contract::validate_physical_expr_contract, + footers::{FooterUnavailable, object_parquet_footer}, +}; use crate::capability::CapabilityReason; #[derive(Debug, Clone, Copy)] @@ -162,6 +166,7 @@ struct RangeCoverage<'a> { } pub(super) fn coalesced_local_parquet_scan_files<'a>( + object_store_url: &ObjectStoreUrl, files: &[&'a PartitionedFile], ) -> Result>, CapabilityReason> { let mut coverages = Vec::>::new(); @@ -224,7 +229,7 @@ pub(super) fn coalesced_local_parquet_scan_files<'a>( continue; } - let row_groups = row_groups_for_byte_ranges(coverage)?; + let row_groups = row_groups_for_byte_ranges(object_store_url, coverage)?; coverage.byte_range_representation = LocalParquetByteRangeRepresentation::RowGroupSelection; coverage.byte_range_row_groups = Some(row_groups); } @@ -252,12 +257,20 @@ fn ranges_cover_full_file(coverage: &mut FileCoverage<'_>) -> Result) -> Result, CapabilityReason> { +fn row_groups_for_byte_ranges( + object_store_url: &ObjectStoreUrl, + coverage: &mut FileCoverage<'_>, +) -> Result, CapabilityReason> { if !ranges_are_non_overlapping(coverage) { return Err(CapabilityReason::NativeSourceByteRange); } - let path = local_object_path_to_filesystem(coverage.location.as_str()); - let metadata = parquet_footer(path.as_path()).map_err(|_error| CapabilityReason::NativeSourceByteRange)?; + let metadata = + object_parquet_footer(object_store_url, &coverage.representative.object_meta).map_err(|unavailable| { + match unavailable { + FooterUnavailable::Local => CapabilityReason::NativeSourceByteRange, + FooterUnavailable::RemoteNotCached => CapabilityReason::NativeSourceRemoteFooterUnavailable, + } + })?; let row_group_count = metadata.num_row_groups(); let ranges = coverage .ranges diff --git a/crates/algeon-datafusion/src/native/pruning/row_groups.rs b/crates/algeon-datafusion/src/native/pruning/row_groups.rs index 35adf4504..4749f6da2 100644 --- a/crates/algeon-datafusion/src/native/pruning/row_groups.rs +++ b/crates/algeon-datafusion/src/native/pruning/row_groups.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use arrow::datatypes::DataType; -use datafusion::parquet::schema::types::SchemaDescriptor; +use datafusion::{execution::object_store::ObjectStoreUrl, parquet::schema::types::SchemaDescriptor}; use parquet::file::metadata::{ColumnChunkMetaData, RowGroupMetaData}; use query_planning::parquet_stats::{integer_statistics_range, leaf_column_chunk, leaf_column_statistics}; @@ -9,9 +9,10 @@ use super::{ BinaryExpr, CastExpr, Column, FileScanConfig, FileSource, FilterExec, InListExpr, IsNotNullExpr, IsNullExpr, Literal, Operator, ParquetSource, PartitionedFile, PhysicalExpr, ScalarValue, SchemaRef, Statistics, and_may_satisfy, coalesced_local_parquet_scan_files, float_range_may_satisfy, in_list_may_satisfy, - is_not_null_may_satisfy, is_null_may_satisfy, local_object_path_to_filesystem, or_may_satisfy, - ordered_range_may_satisfy, parquet_footer, range_comparison_from_operator, validate_pruning_only_parquet_predicate, + is_not_null_may_satisfy, is_null_may_satisfy, object_parquet_footer, or_may_satisfy, ordered_range_may_satisfy, + range_comparison_from_operator, validate_pruning_only_parquet_predicate, }; +use crate::native::source::native_physical_file_schema; pub(in crate::native) fn metadata_row_group_selections( files: &[&PartitionedFile], @@ -20,10 +21,9 @@ pub(in crate::native) fn metadata_row_group_selections( filter: Option<&FilterExec>, source_schema: &SchemaRef, ) -> Option>> { - let coalesced_files = coalesced_local_parquet_scan_files(files).ok()?; + let coalesced_files = coalesced_local_parquet_scan_files(&file_scan.object_store_url, files).ok()?; let files = coalesced_files.iter().map(|file| file.file).collect::>(); if files.is_empty() - || !file_scan.object_store_url.as_str().starts_with("file://") || file_scan.expr_adapter_factory.is_some() || file_scan.limit.is_some() || file_scan.preserve_order @@ -37,9 +37,14 @@ pub(in crate::native) fn metadata_row_group_selections( { return None; } - let predicate = row_group_pruning_expr(filter?.predicate(), file_scan.file_schema())?; - let selections = - files.iter().map(|file| metadata_row_group_selection(file, predicate.clone())).collect::>>()?; + // Footer statistics are addressed by physical column position, which the + // footer schema gives; DataFusion's file schema may order columns otherwise. + let physical_schema = native_physical_file_schema(file_scan).ok()?; + let predicate = row_group_pruning_expr(filter?.predicate(), &physical_schema)?; + let selections = files + .iter() + .map(|file| metadata_row_group_selection(&file_scan.object_store_url, file, predicate.clone())) + .collect::>>()?; let selected_row_groups = selections.iter().map(|selection| selection.row_groups.len()).sum::(); let total_row_groups = selections.iter().map(|selection| selection.total_row_groups).sum::(); if selected_row_groups > 0 && selected_row_groups < total_row_groups { @@ -56,11 +61,11 @@ struct MetadataRowGroupSelection { } fn metadata_row_group_selection( + object_store_url: &ObjectStoreUrl, file: &PartitionedFile, predicate: RowGroupPruningExpr, ) -> Option { - let path = local_object_path_to_filesystem(file.object_meta.location.as_ref()); - let metadata = parquet_footer(path.as_path()).ok()?; + let metadata = object_parquet_footer(object_store_url, &file.object_meta).ok()?; let schema = metadata.file_metadata().schema_descr(); let total_row_groups = metadata.num_row_groups(); if total_row_groups == 0 { diff --git a/crates/algeon-datafusion/src/native/source.rs b/crates/algeon-datafusion/src/native/source.rs index c8850f1ee..dc95dde0c 100644 --- a/crates/algeon-datafusion/src/native/source.rs +++ b/crates/algeon-datafusion/src/native/source.rs @@ -3,7 +3,8 @@ //! A scan lowers when its object store, partition columns, reader factory, //! declared ordering, file groups and byte ranges can be represented by the //! native local or S3 Parquet source. Footers are read through the planning -//! pass's `FooterScope`; nothing here opens data pages. +//! pass's `FooterScope`, remote ones from the session's file metadata cache; +//! nothing here opens data pages. use std::collections::BTreeSet; #[cfg(feature = "cugraph")] @@ -17,6 +18,8 @@ use datafusion::{ listing::PartitionedFile, physical_plan::{FileScanConfig, FileSource, ParquetSource}, }, + execution::object_store::ObjectStoreUrl, + object_store::ObjectMeta, physical_plan::filter::FilterExec, }; use parquet::file::metadata::{KeyValue, ParquetMetaData}; @@ -40,7 +43,7 @@ use serde_json::Value; #[cfg(feature = "cugraph")] use super::footers::FooterScope; use super::{ - footers::parquet_footer, + footers::object_parquet_footer, literal::{NativeLiteralContext, native_literal_value}, pruning::{ coalesced_local_parquet_scan_files, explicit_row_group_selection, local_object_path_to_filesystem, @@ -54,8 +57,8 @@ mod diagnostics; mod s3_parquet; pub(super) use columns::{ - NativeParquetColumnRequirements, native_source_output_schema, parquet_source_has_row_level_predicate, - required_parquet_column_requirements_for_scan_columns, + NativeParquetColumnRequirements, native_physical_file_schema, native_source_output_schema, + parquet_source_has_row_level_predicate, required_parquet_column_requirements_for_scan_columns, }; pub(super) use diagnostics::{SourceRead, local_parquet_source_diagnostics, source_diagnostics_for_normalized_plan}; pub(crate) use s3_parquet::{ParquetReadStrategyConfig, S3_STORAGE_PROFILE_ID, ambient_s3_endpoint}; @@ -310,7 +313,7 @@ pub(super) fn local_parquet_files( if local_parquet_scan_is_empty(file_scan) { return Err(CapabilityReason::NativeSourceEmpty); } - let scan_files = coalesced_local_parquet_scan_files(&files)?; + let scan_files = coalesced_local_parquet_scan_files(&file_scan.object_store_url, &files)?; let metadata_row_group_selections = metadata_row_group_selections(&files, file_scan, parquet_source, filter, &source_schema); let mut native_files = Vec::with_capacity(scan_files.len()); @@ -330,7 +333,13 @@ pub(super) fn local_parquet_files( { native_file = native_file.with_row_groups(row_groups.clone()); } - native_file = attach_row_group_byte_metadata(native_file, &source_schema, column_requirements); + native_file = attach_row_group_byte_metadata( + native_file, + &file_scan.object_store_url, + &file.object_meta, + &source_schema, + column_requirements, + ); native_files.push(native_file); } Ok(native_files) @@ -340,7 +349,7 @@ pub(super) fn local_parquet_scan_is_empty(file_scan: &FileScanConfig) -> bool { file_scan.file_groups.iter().all(|group| group.files().is_empty()) } -fn native_parquet_file_for_scan( +pub(super) fn native_parquet_file_for_scan( file_scan: &FileScanConfig, file: &PartitionedFile, source_reads: &ParquetReadStrategyConfig, @@ -387,12 +396,16 @@ fn object_store_file_uri(file_scan: &FileScanConfig, file: &PartitionedFile) -> Ok(format!("{base}/{key}")) } +/// Attaches the row-group byte, row, null-count, and graph metadata of +/// `object`'s footer. An object without a footer keeps none of it. pub(super) fn attach_row_group_byte_metadata( mut native_file: NativeLocalParquetFile, + object_store_url: &ObjectStoreUrl, + object: &ObjectMeta, source_schema: &Schema, column_requirements: &NativeParquetColumnRequirements, ) -> NativeLocalParquetFile { - let Ok(footer) = parquet_footer(native_file.path.as_path()) else { + let Ok(footer) = object_parquet_footer(object_store_url, object) else { return native_file; }; let metadata = footer.as_ref(); diff --git a/crates/algeon-datafusion/src/native/source/columns.rs b/crates/algeon-datafusion/src/native/source/columns.rs index 6b4e9fd67..3e7a27a60 100644 --- a/crates/algeon-datafusion/src/native/source/columns.rs +++ b/crates/algeon-datafusion/src/native/source/columns.rs @@ -1,12 +1,14 @@ use std::{ - collections::{BTreeSet, HashSet}, + collections::{BTreeSet, HashMap, HashSet}, sync::Arc, }; -use arrow::datatypes::{Field, Schema, SchemaRef}; +use arrow::datatypes::{DataType, Field, FieldRef, Schema, SchemaRef}; use cudf::interop::arrow_host::is_supported_arrow_type; use datafusion::{ datasource::physical_plan::{FileScanConfig, FileSource, ParquetSource}, + execution::object_store::ObjectStoreUrl, + object_store::ObjectMeta, physical_expr::{PhysicalExpr, expressions::Column, projection::ProjectionExprs}, physical_plan::filter::FilterExec, }; @@ -19,11 +21,8 @@ use query_planning::source::{ use crate::{ capability::CapabilityReason, native::{ - footers::parquet_footer, - pruning::{ - local_object_path_to_filesystem, scan_predicate_is_dynamic_filter_hint, - scan_predicate_without_dynamic_filter_hints, - }, + footers::{FooterUnavailable, object_parquet_footer}, + pruning::{scan_predicate_is_dynamic_filter_hint, scan_predicate_without_dynamic_filter_hints}, validate::collect_required_columns, }, }; @@ -205,37 +204,96 @@ fn collect_source_required_columns( collect_required_columns(&source_expr, table_schema, required) } -fn native_physical_file_schema(file_scan: &FileScanConfig) -> Result { - let files = file_scan.file_groups.iter().flat_map(|group| group.files()).collect::>(); - if !file_scan.object_store_url.as_str().starts_with("file://") { - if files.len() > 1 { - return Err(CapabilityReason::NativeSourceSchemaMapping); - } +/// The physical schema of the scan's Parquet objects, which column indices +/// address. +/// +/// Every distinct object's footer must hold the same columns in the same +/// order with the same Arrow types. A column may be required in one file and +/// optional in another when the table schema declares it nullable; DataFusion +/// widens nullability the same way when it merges file schemas. Column +/// indices and footer statistics address this schema, which can order columns +/// differently from DataFusion's file schema. A remote scan of one object +/// whose footer planning cannot read keeps the DataFusion file schema; nothing +/// that needs the footer applies to it then. +pub(in crate::native) fn native_physical_file_schema( + file_scan: &FileScanConfig, +) -> Result { + let mut checked_locations = HashSet::new(); + // Byte-range splits of one object share its footer, so each distinct object is checked once. + let objects = file_scan + .file_groups + .iter() + .flat_map(|group| group.files()) + .map(|file| &file.object_meta) + .filter(|object| checked_locations.insert(&object.location)) + .collect::>(); + if objects.is_empty() { return Ok(Arc::clone(file_scan.file_schema())); } - let Some(first_file) = files.first() else { - return Ok(Arc::clone(file_scan.file_schema())); - }; - let first_location = first_file.object_meta.location.as_ref(); - let first_schema = local_parquet_footer_schema(first_location)?; - // Byte-range splits of one object share its footer, so each distinct object is parsed once. - let mut checked_locations = HashSet::from([first_location]); - for file in files.iter().skip(1) { - let location = file.object_meta.location.as_ref(); - if !checked_locations.insert(location) { - continue; + let single_object = objects.len() == 1; + let table_schema = file_scan.file_schema(); + let mut physical_schema: Option = None; + for object in objects { + let schema = match parquet_footer_schema(&file_scan.object_store_url, object) { + Err(CapabilityReason::NativeSourceRemoteFooterUnavailable) if single_object => { + return Ok(Arc::clone(file_scan.file_schema())); + }, + schema => schema?, + }; + let nullable_in_table = |field: &Field| { + table_schema.field_with_name(field.name()).map_or(true, |table_field| table_field.is_nullable()) + }; + if schema.fields().iter().any(|field| field.is_nullable() && !nullable_in_table(field)) { + return Err(CapabilityReason::NativeSourceFileSchemaMismatch); } - let schema = local_parquet_footer_schema(location)?; - if schema.as_ref() != first_schema.as_ref() { - return Err(CapabilityReason::NativeSourceSchemaMapping); + match &physical_schema { + None => physical_schema = Some(schema), + Some(first) => { + if first.fields().len() != schema.fields().len() + || first.fields().iter().zip(schema.fields()).any(|(first, field)| { + first.name() != field.name() + || data_type_without_metadata(first.data_type()) + != data_type_without_metadata(field.data_type()) + }) + { + return Err(CapabilityReason::NativeSourceFileSchemaMismatch); + } + }, } } - Ok(first_schema) + Ok(physical_schema.unwrap_or_else(|| Arc::clone(file_scan.file_schema()))) +} + +/// `data_type` with the metadata of every nested field removed: writers +/// attach field ids and other annotations to nested fields, which do not +/// change how the column decodes. +fn data_type_without_metadata(data_type: &DataType) -> DataType { + let field_without_metadata = |field: &FieldRef| -> FieldRef { + Arc::new( + Field::new(field.name(), data_type_without_metadata(field.data_type()), field.is_nullable()) + .with_metadata(HashMap::new()), + ) + }; + match data_type { + DataType::List(field) => DataType::List(field_without_metadata(field)), + DataType::LargeList(field) => DataType::LargeList(field_without_metadata(field)), + DataType::ListView(field) => DataType::ListView(field_without_metadata(field)), + DataType::LargeListView(field) => DataType::LargeListView(field_without_metadata(field)), + DataType::FixedSizeList(field, size) => DataType::FixedSizeList(field_without_metadata(field), *size), + DataType::Map(field, sorted) => DataType::Map(field_without_metadata(field), *sorted), + DataType::Struct(fields) => DataType::Struct(fields.iter().map(field_without_metadata).collect()), + other => other.clone(), + } } -fn local_parquet_footer_schema(location: &str) -> Result { - let path = local_object_path_to_filesystem(location); - let footer = parquet_footer(path.as_path()).map_err(|_error| CapabilityReason::NativeSourceSchemaMapping)?; +fn parquet_footer_schema( + object_store_url: &ObjectStoreUrl, + object: &ObjectMeta, +) -> Result { + let footer = object_parquet_footer(object_store_url, object).map_err(|unavailable| match unavailable { + FooterUnavailable::Local => CapabilityReason::NativeSourceSchemaMapping, + FooterUnavailable::RemoteNotCached => CapabilityReason::NativeSourceRemoteFooterUnavailable, + })?; let metadata = footer.file_metadata(); let schema = parquet_to_arrow_schema(metadata.schema_descr(), metadata.key_value_metadata()) .map_err(|_error| CapabilityReason::NativeSourceSchemaMapping)?; diff --git a/crates/algeon-datafusion/src/native/source/diagnostics.rs b/crates/algeon-datafusion/src/native/source/diagnostics.rs index 7e98ff782..08dd5ba66 100644 --- a/crates/algeon-datafusion/src/native/source/diagnostics.rs +++ b/crates/algeon-datafusion/src/native/source/diagnostics.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use arrow::datatypes::SchemaRef; use datafusion::{ datasource::physical_plan::{FileScanConfig, ParquetSource, parquet::ParquetAccessPlan}, + execution::object_store::ObjectStoreUrl, physical_plan::filter::FilterExec, }; use query_planning::source::{ @@ -13,18 +14,22 @@ use query_planning::source::{ }; use super::{ - NativeParquetColumnRequirements, native_declared_output_ordering_scan_is_supported, - native_parquet_file_reader_factory_is_supported, native_parquet_object_store_scan_is_supported, + NativeParquetColumnRequirements, ParquetReadStrategyConfig, native_declared_output_ordering_scan_is_supported, + native_parquet_file_for_scan, native_parquet_file_reader_factory_is_supported, + native_parquet_object_store_scan_is_supported, native_physical_file_schema, native_preserve_order_scan_is_supported, partition_column_disposition, }; -use crate::native::{ - footers::parquet_footer, - lowering::{aggregate_input_demand, parquet_column_requirements, parquet_reads_row_count_only}, - normalize::{NormalizedNativeAggregate, NormalizedNativeInput, NormalizedNativePlan, NormalizedNativeSource}, - pruning::{ - CoalescedLocalParquetFile, LocalParquetByteRangeRepresentation, coalesced_local_parquet_scan_files, - explicit_row_group_selection, has_row_level_selection, local_object_path_to_filesystem, - metadata_row_group_selections, parquet_predicate_disposition, selected_row_groups_from_access_plans, +use crate::{ + capability::CapabilityReason, + native::{ + footers::object_parquet_footer, + lowering::{aggregate_input_demand, parquet_column_requirements, parquet_reads_row_count_only}, + normalize::{NormalizedNativeAggregate, NormalizedNativeInput, NormalizedNativePlan, NormalizedNativeSource}, + pruning::{ + CoalescedLocalParquetFile, LocalParquetByteRangeRepresentation, coalesced_local_parquet_scan_files, + explicit_row_group_selection, has_row_level_selection, metadata_row_group_selections, + parquet_predicate_disposition, selected_row_groups_from_access_plans, + }, }, }; @@ -34,9 +39,10 @@ use crate::native::{ /// enums and counters so they can be reported without exposing DataFusion types. pub(in crate::native) fn source_diagnostics_for_normalized_plan( normalized: &NormalizedNativePlan<'_>, + source_reads: &ParquetReadStrategyConfig, ) -> Vec { let mut diagnostics = Vec::new(); - collect_source_diagnostics_for_input(&normalized.input, None, &mut diagnostics); + collect_source_diagnostics_for_input(&normalized.input, None, source_reads, &mut diagnostics); diagnostics } @@ -45,49 +51,50 @@ pub(in crate::native) fn source_diagnostics_for_normalized_plan( fn collect_source_diagnostics_for_input( input: &NormalizedNativeInput<'_>, aggregate: Option<&NormalizedNativeAggregate<'_>>, + source_reads: &ParquetReadStrategyConfig, diagnostics: &mut Vec, ) { match input { NormalizedNativeInput::Source(source) => { - if let Some(diagnostic) = source_diagnostics_for_source(source, aggregate) { + if let Some(diagnostic) = source_diagnostics_for_source(source, aggregate, source_reads) { diagnostics.push(diagnostic); } }, NormalizedNativeInput::GpuResidentSource(_) => {}, NormalizedNativeInput::LiteralSource(_) => {}, NormalizedNativeInput::Filter(filter) => { - collect_source_diagnostics_for_input(&filter.input, None, diagnostics); + collect_source_diagnostics_for_input(&filter.input, None, source_reads, diagnostics); }, NormalizedNativeInput::Projection(projection) => { - collect_source_diagnostics_for_input(&projection.input, None, diagnostics); + collect_source_diagnostics_for_input(&projection.input, None, source_reads, diagnostics); }, NormalizedNativeInput::Limit(limit) => { - collect_source_diagnostics_for_input(&limit.input, None, diagnostics); + collect_source_diagnostics_for_input(&limit.input, None, source_reads, diagnostics); }, NormalizedNativeInput::Aggregate(aggregate) => { - collect_source_diagnostics_for_input(&aggregate.input, Some(aggregate), diagnostics); + collect_source_diagnostics_for_input(&aggregate.input, Some(aggregate), source_reads, diagnostics); }, NormalizedNativeInput::Sort(sort) => { - collect_source_diagnostics_for_input(&sort.input, None, diagnostics); + collect_source_diagnostics_for_input(&sort.input, None, source_reads, diagnostics); }, NormalizedNativeInput::Window(window) => { - collect_source_diagnostics_for_input(&window.input, None, diagnostics); + collect_source_diagnostics_for_input(&window.input, None, source_reads, diagnostics); }, NormalizedNativeInput::HashJoin(join) => { - collect_source_diagnostics_for_input(&join.left, None, diagnostics); - collect_source_diagnostics_for_input(&join.right, None, diagnostics); + collect_source_diagnostics_for_input(&join.left, None, source_reads, diagnostics); + collect_source_diagnostics_for_input(&join.right, None, source_reads, diagnostics); }, NormalizedNativeInput::CrossJoin(join) => { - collect_source_diagnostics_for_input(&join.left, None, diagnostics); - collect_source_diagnostics_for_input(&join.right, None, diagnostics); + collect_source_diagnostics_for_input(&join.left, None, source_reads, diagnostics); + collect_source_diagnostics_for_input(&join.right, None, source_reads, diagnostics); }, NormalizedNativeInput::NestedLoopJoin(join) => { - collect_source_diagnostics_for_input(&join.left, None, diagnostics); - collect_source_diagnostics_for_input(&join.right, None, diagnostics); + collect_source_diagnostics_for_input(&join.left, None, source_reads, diagnostics); + collect_source_diagnostics_for_input(&join.right, None, source_reads, diagnostics); }, NormalizedNativeInput::Union(union) => { for input in &union.inputs { - collect_source_diagnostics_for_input(input, None, diagnostics); + collect_source_diagnostics_for_input(input, None, source_reads, diagnostics); } }, } @@ -96,6 +103,7 @@ fn collect_source_diagnostics_for_input( fn source_diagnostics_for_source( source: &NormalizedNativeSource<'_>, aggregate: Option<&NormalizedNativeAggregate<'_>>, + source_reads: &ParquetReadStrategyConfig, ) -> Option { let source_exec = source.scan; let (file_scan, parquet_source) = source_exec.downcast_to_file_source::()?; @@ -117,6 +125,7 @@ fn source_diagnostics_for_source( // Outside an aggregate this walk does not know what the parent reads. ordered: demand.as_ref().is_none_or(|demand| demand.ordered), }, + source_reads, )) } @@ -143,10 +152,11 @@ pub(in crate::native) fn local_parquet_source_diagnostics( source_schema: SchemaRef, column_requirements: Option<&NativeParquetColumnRequirements>, read: SourceRead, + source_reads: &ParquetReadStrategyConfig, ) -> NativeSourceDiagnostics { let SourceRead { reader_pushdown, row_count_only: zero_column_count_safe, ordered } = read; let files = file_scan.file_groups.iter().flat_map(|group| group.files()).collect::>(); - let coalesced_files = coalesced_local_parquet_scan_files(&files); + let coalesced_files = coalesced_local_parquet_scan_files(&file_scan.object_store_url, &files); let diagnostic_files = coalesced_files .as_ref() .map_or_else(|_| files.clone(), |coalesced| coalesced.iter().map(|file| file.file).collect::>()); @@ -169,8 +179,10 @@ pub(in crate::native) fn local_parquet_source_diagnostics( } else { NativeByteRangeDisposition::Rejected }; - let coalesced_selected_row_groups = - coalesced_files.as_ref().ok().and_then(|files| selected_row_groups_from_coalesced_files(files)); + let coalesced_selected_row_groups = coalesced_files + .as_ref() + .ok() + .and_then(|files| selected_row_groups_from_coalesced_files(&file_scan.object_store_url, files)); let selected_row_groups = coalesced_selected_row_groups.or_else(|| selected_row_groups_from_access_plans(&files)).or_else(|| { metadata_row_group_selections(&files, file_scan, parquet_source, filter, &source_schema) @@ -192,16 +204,24 @@ pub(in crate::native) fn local_parquet_source_diagnostics( NativeZeroColumnCountDisposition::NotRequired } }); - let schema_mapping_disposition = - column_requirements.map_or(NativeSchemaMappingDisposition::DirectByName, |requirements| { - if !requirements.missing_columns.is_empty() { - NativeSchemaMappingDisposition::MissingNullableColumns - } else { - NativeSchemaMappingDisposition::DirectByName - } - }); + let physical_schema = native_physical_file_schema(file_scan); + let schema_mapping_disposition = match &physical_schema { + Err(CapabilityReason::NativeSourceRemoteFooterUnavailable) | Ok(_) => { + column_requirements.map_or(NativeSchemaMappingDisposition::DirectByName, |requirements| { + if !requirements.missing_columns.is_empty() { + NativeSchemaMappingDisposition::MissingNullableColumns + } else { + NativeSchemaMappingDisposition::DirectByName + } + }) + }, + Err(_) => NativeSchemaMappingDisposition::Rejected, + }; let mut rejected_features = Vec::new(); + if physical_schema == Err(CapabilityReason::NativeSourceRemoteFooterUnavailable) { + rejected_features.push(NativeRejectedSourceFeature::RemoteFooterUnavailable); + } if !native_parquet_object_store_scan_is_supported(file_scan) { rejected_features.push(NativeRejectedSourceFeature::NonLocalObjectStore); } @@ -241,7 +261,6 @@ pub(in crate::native) fn local_parquet_source_diagnostics( rejected_features.push(NativeRejectedSourceFeature::SchemaMapping); } let ranges_are_safely_coalesced = ranged_file_count > 0 - && file_scan.object_store_url.as_str().starts_with("file://") && partition_column_disposition != NativePartitionColumnDisposition::Rejected && file_scan.expr_adapter_factory.is_none() && scan_limit_disposition != NativeScanLimitDisposition::Rejected @@ -264,13 +283,19 @@ pub(in crate::native) fn local_parquet_source_diagnostics( rejected_features.sort_by_key(|feature| feature.as_str()); rejected_features.dedup(); - NativeSourceDiagnostics { - source_read_strategy: "local_kvikio".to_string(), - credential_mode: "none".to_string(), - lakehouse_cache_status: "disabled".to_string(), - lakehouse_cache_read_mode: "disabled".to_string(), - lakehouse_cache_fill_mode: "disabled".to_string(), - direct_device_read: rejected_features.is_empty(), + // The files resolve with the read strategy lowering uses, so a rejected + // remote scan reports the remote read it would have made. + let resolved_files = diagnostic_files + .iter() + .filter_map(|file| native_parquet_file_for_scan(file_scan, file, source_reads).ok()) + .collect::>(); + let mut diagnostics = NativeSourceDiagnostics { + source_read_strategy: String::new(), + credential_mode: String::new(), + lakehouse_cache_status: String::new(), + lakehouse_cache_read_mode: String::new(), + lakehouse_cache_fill_mode: String::new(), + direct_device_read: false, file_count: diagnostic_files.len(), ranged_file_count, known_bytes, @@ -286,7 +311,9 @@ pub(in crate::native) fn local_parquet_source_diagnostics( projected_output_column_count, pure_filter_column_count, rejected_features, - } + }; + diagnostics.set_resolved_read_labels(&resolved_files); + diagnostics } fn scan_limit_disposition(file_scan: &FileScanConfig, ordered: bool) -> NativeScanLimitDisposition { @@ -302,24 +329,29 @@ fn scan_limit_disposition(file_scan: &FileScanConfig, ordered: bool) -> NativeSc } } -fn selected_row_groups_from_coalesced_files(files: &[CoalescedLocalParquetFile<'_>]) -> Option { +fn selected_row_groups_from_coalesced_files( + object_store_url: &ObjectStoreUrl, + files: &[CoalescedLocalParquetFile<'_>], +) -> Option { let has_row_group_selection = files .iter() .any(|file| file.byte_range_row_groups.is_some() || file.file.extensions.get::().is_some()); has_row_group_selection.then(|| { files.iter().try_fold(0usize, |total, file| { - selected_row_group_count_for_coalesced_file(file).map(|count| total.saturating_add(count)) + selected_row_group_count_for_coalesced_file(object_store_url, file).map(|count| total.saturating_add(count)) }) })? } -fn selected_row_group_count_for_coalesced_file(file: &CoalescedLocalParquetFile<'_>) -> Option { +fn selected_row_group_count_for_coalesced_file( + object_store_url: &ObjectStoreUrl, + file: &CoalescedLocalParquetFile<'_>, +) -> Option { if let Some(row_groups) = &file.byte_range_row_groups { return Some(row_groups.len()); } if let Some(row_groups) = explicit_row_group_selection(file.file).ok()? { return Some(row_groups.len()); } - let path = local_object_path_to_filesystem(file.file.object_meta.location.as_ref()); - Some(parquet_footer(path.as_path()).ok()?.num_row_groups()) + Some(object_parquet_footer(object_store_url, &file.file.object_meta).ok()?.num_row_groups()) } diff --git a/crates/algeon-datafusion/src/native_exec/config/mod.rs b/crates/algeon-datafusion/src/native_exec/config/mod.rs index 380f5d448..6a0898350 100644 --- a/crates/algeon-datafusion/src/native_exec/config/mod.rs +++ b/crates/algeon-datafusion/src/native_exec/config/mod.rs @@ -285,7 +285,7 @@ impl AlgeonNativeExecutionConfig { self } - /// Set expert local-throughput source chunk planning policy. + /// Set expert throughput source chunk planning policy. /// /// This policy is intentionally Rust-only; public source chunk settings remain /// the bounded memory policy and source chunk planning mode. @@ -380,7 +380,7 @@ impl AlgeonNativeExecutionConfig { self.aggregate_strategy_execution_options } - /// Expert local-throughput source chunk planning policy copied into native contexts. + /// Expert throughput source chunk planning policy copied into native contexts. pub fn source_chunk_planning_options(&self) -> SourceChunkPlanningOptions { self.source_chunk_planning_options } diff --git a/crates/algeon-datafusion/src/native_exec/config/tests.rs b/crates/algeon-datafusion/src/native_exec/config/tests.rs index 54b77b41d..ff43d2e40 100644 --- a/crates/algeon-datafusion/src/native_exec/config/tests.rs +++ b/crates/algeon-datafusion/src/native_exec/config/tests.rs @@ -56,7 +56,7 @@ fn config_validation_errors_carry_configured_correlation_ids() { .with_query_id("query-config-error") .with_trace_id("trace-config-error") .with_source_chunk_planning_options( - SourceChunkPlanningOptions::default().with_local_parquet_throughput_max_row_groups_per_chunk(0), + SourceChunkPlanningOptions::default().with_parquet_throughput_max_row_groups_per_chunk(0), ) .validate_query_policies() .unwrap_err(); diff --git a/crates/algeon-datafusion/src/native_exec/metrics/tests.rs b/crates/algeon-datafusion/src/native_exec/metrics/tests.rs index 0b8eff46b..5ca3c8bb3 100644 --- a/crates/algeon-datafusion/src/native_exec/metrics/tests.rs +++ b/crates/algeon-datafusion/src/native_exec/metrics/tests.rs @@ -348,8 +348,8 @@ fn source_read_runtime_rows_bridge_runtime_labels() { reader_predicate: true, reader_dynamic_filter_count: 1, source_batching_reason: "batched".to_string(), - source_chunk_planning_mode: "local_parquet_throughput".to_string(), - source_chunk_planning_reason: "local_parquet_throughput_admitted".to_string(), + source_chunk_planning_mode: "parquet_throughput".to_string(), + source_chunk_planning_reason: "parquet_throughput_admitted".to_string(), elapsed: std::time::Duration::from_nanos(1234), }); native_metrics.record_query_metrics(&query_metrics); @@ -383,8 +383,8 @@ fn source_read_runtime_rows_bridge_runtime_labels() { assert!(row.reader_predicate); assert_eq!(row.reader_dynamic_filter_count, 1); assert_eq!(row.source_batching_reason, "batched"); - assert_eq!(row.source_chunk_planning_mode, "local_parquet_throughput"); - assert_eq!(row.source_chunk_planning_reason, "local_parquet_throughput_admitted"); + assert_eq!(row.source_chunk_planning_mode, "parquet_throughput"); + assert_eq!(row.source_chunk_planning_reason, "parquet_throughput_admitted"); assert_eq!(row.elapsed_ns, 1234); } diff --git a/crates/algeon-datafusion/src/native_report/diagnostics.rs b/crates/algeon-datafusion/src/native_report/diagnostics.rs index 68c2af92c..0a60ca8ca 100644 --- a/crates/algeon-datafusion/src/native_report/diagnostics.rs +++ b/crates/algeon-datafusion/src/native_report/diagnostics.rs @@ -155,7 +155,10 @@ fn plan_blocker_category(blocker: &PlanBlocker) -> Option { WindowDefect::UnsupportedFrame(frame) => Some(format!("window_frame_{}", window_frame_identifier(frame))), WindowDefect::Arguments => None, }, - PlanBlocker::ParentOperator(_) | PlanBlocker::DistributedAggregate(_) | PlanBlocker::Ordering { .. } => None, + PlanBlocker::ParentOperator(_) + | PlanBlocker::DistributedAggregate(_) + | PlanBlocker::Ordering { .. } + | PlanBlocker::Source { .. } => None, } } diff --git a/crates/algeon-datafusion/src/planner/native.rs b/crates/algeon-datafusion/src/planner/native.rs index 5765f373e..8c70d80ac 100644 --- a/crates/algeon-datafusion/src/planner/native.rs +++ b/crates/algeon-datafusion/src/planner/native.rs @@ -208,7 +208,8 @@ pub async fn plan_with_native_diagnostics( ) -> DataFusionResult<(Arc, NativePlanningDiagnostics)> { let invocation = Invocation::detached(); let mut state = state.clone(); - invocation.install(state.config_mut()); + let runtime_env = Arc::clone(state.runtime_env()); + invocation.install(state.config_mut(), &runtime_env); let plan = state.create_physical_plan(logical).await?; let diagnostics = invocation.planning_diagnostics().ok_or_else(|| { DataFusionError::from( @@ -329,6 +330,9 @@ impl AlgeonNativePhysicalOptimizerRule { options: &ConfigOptions, ) -> DataFusionResult<(Arc, Option)> { let invocation = Invocation::detached(); + if let Some(outer) = Invocation::of_options(options) { + invocation.share_remote_footers(outer); + } let mut options = options.clone(); options.extensions.insert(invocation.clone()); let plan = self.optimize(plan, &options)?; diff --git a/crates/algeon-datafusion/src/planner/native/invocation.rs b/crates/algeon-datafusion/src/planner/native/invocation.rs index a408137ac..6fe20d271 100644 --- a/crates/algeon-datafusion/src/planner/native/invocation.rs +++ b/crates/algeon-datafusion/src/planner/native/invocation.rs @@ -18,7 +18,7 @@ use datafusion::{ DataFusionError, Result as DataFusionResult, config::{ConfigEntry, ConfigExtension, ConfigOptions, ExtensionOptions}, }, - execution::TaskContext, + execution::{TaskContext, cache::cache_manager::FileMetadataCache, runtime_env::RuntimeEnv}, prelude::SessionConfig, }; use query_planning::admission::AttemptCapabilityEvidence; @@ -45,7 +45,6 @@ use crate::{ #[derive(Debug, Clone)] pub(crate) struct Invocation(Arc); -#[derive(Debug)] struct InvocationState { attempt: Option, admission: OnceLock>, @@ -56,6 +55,20 @@ struct InvocationState { // rewrite ends it, so the count-only probe and the final rewrite parse each // footer once without keeping footers alive while the plan executes. footers: Mutex>>, + // The file metadata cache of the runtime this invocation was last + // installed with. Planning reads remote footers from it. + remote_footers: Mutex>>, +} + +impl std::fmt::Debug for InvocationState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("InvocationState") + .field("attempt", &self.attempt) + .field("admission", &self.admission) + .field("planning", &self.planning) + .field("footers", &self.footers) + .finish_non_exhaustive() + } } /// Execution inputs that the Flight server attaches to an attempt's @@ -89,6 +102,7 @@ impl Invocation { execution: Mutex::new(None), planning: Mutex::new(Vec::new()), footers: Mutex::new(None), + remote_footers: Mutex::new(None), })) } @@ -104,11 +118,21 @@ impl Invocation { } /// Makes this invocation the one that `config` plans and executes under, - /// replacing any invocation the config already carried. - pub(crate) fn install(&self, config: &mut SessionConfig) { + /// replacing any invocation the config already carried. Planning reads + /// remote footers from the file metadata cache of `runtime_env`, the + /// runtime the session plans with. + pub(crate) fn install(&self, config: &mut SessionConfig, runtime_env: &RuntimeEnv) { + *self.0.remote_footers.lock().unwrap_or_else(PoisonError::into_inner) = + Some(runtime_env.cache_manager.get_file_metadata_cache()); config.options_mut().extensions.insert(self.clone()); } + /// Makes `self` read remote footers from the same cache as `other`. + pub(crate) fn share_remote_footers(&self, other: &Self) { + let remote = other.0.remote_footers.lock().unwrap_or_else(PoisonError::into_inner).clone(); + *self.0.remote_footers.lock().unwrap_or_else(PoisonError::into_inner) = remote; + } + /// The attempt this invocation belongs to, if any. pub(crate) fn attempt(&self) -> Option<&InvocationAttempt> { self.0.attempt.as_ref() @@ -160,13 +184,11 @@ impl Invocation { /// Enters a footer scope over this invocation's current planning pass. pub(crate) fn footer_scope(&self) -> FooterScope { - let cache = Arc::clone( - self.0 - .footers - .lock() - .unwrap_or_else(PoisonError::into_inner) - .get_or_insert_with(|| Arc::new(FooterCache::for_scope())), - ); + let cache = + Arc::clone(self.0.footers.lock().unwrap_or_else(PoisonError::into_inner).get_or_insert_with(|| { + let remote = self.0.remote_footers.lock().unwrap_or_else(PoisonError::into_inner).clone(); + Arc::new(FooterCache::for_scope(remote)) + })); FooterScope::join(cache) } diff --git a/crates/algeon-datafusion/src/report/blocking.rs b/crates/algeon-datafusion/src/report/blocking.rs index 0b01d2de2..70897bd5a 100644 --- a/crates/algeon-datafusion/src/report/blocking.rs +++ b/crates/algeon-datafusion/src/report/blocking.rs @@ -33,6 +33,12 @@ pub(crate) enum PlanBlocker { required_by: &'static str, ordering: Option, }, + /// A Parquet scan that the native source contract cannot read; the + /// candidate's `NativeSource*` reason names the feature. + Source { + object_store_url: String, + object_count: usize, + }, } /// What planning the query for one partition (`SET @@ -125,7 +131,7 @@ impl Blocking { /// Stable machine kind: `operator`, `parent_operator`, `aggregate_function`, /// `distributed_aggregate`, `window_function`, `scalar_function`, `cast`, /// `binary_operator`, `unary_operator`, `literal`, `expression`, - /// `column_type`, `ordering`, or `compiled_contract`. + /// `column_type`, `ordering`, `source`, or `compiled_contract`. pub const fn kind(&self) -> &'static str { self.kind } @@ -332,5 +338,13 @@ fn plan_blocking(blocker: &PlanBlocker) -> Blocking { }; Blocking::new("ordering", text, fields) }, + PlanBlocker::Source { object_store_url, object_count } => { + let objects = if *object_count == 1 { "object" } else { "objects" }; + Blocking::new( + "source", + format!("Parquet scan of {object_count} {objects} in `{object_store_url}`"), + vec![("object_store_url", object_store_url.clone()), ("object_count", object_count.to_string())], + ) + }, } } diff --git a/crates/algeon-datafusion/src/server/config/effective.rs b/crates/algeon-datafusion/src/server/config/effective.rs index 5d36beeb6..89fa259fd 100644 --- a/crates/algeon-datafusion/src/server/config/effective.rs +++ b/crates/algeon-datafusion/src/server/config/effective.rs @@ -3,12 +3,16 @@ use std::{fmt::Write as _, sync::OnceLock}; +use query_runtime::runtime::S3SourceStorageCredential; + use super::{ - AlgeonGpuDeviceProfile, ConfigSources, ConfigValueSource, Duration, ServerConfig, ServerFile, ServerTlsFileConfig, + AlgeonGpuDeviceProfile, ConfigSources, ConfigValueSource, Duration, ServerConfig, ServerFile, + ServerIcebergCatalogConfig, ServerIcebergCatalogKind, ServerIcebergRestAuth, ServerTlsFileConfig, env::scalar_leaves, file::{ ServerAdmissionDeviceProfileFileConfig, ServerAdmissionFileConfig, ServerCugraphFileConfig, - ServerDataFileConfig, ServerModelsFileConfig, ServerNativeFileConfig, ServerObservabilityFileConfig, + ServerDataFileConfig, ServerIcebergFileConfig, ServerModelsFileConfig, ServerNativeFileConfig, + ServerObservabilityFileConfig, ServerS3FileConfig, }, table_lookup, }; @@ -16,8 +20,10 @@ use crate::server::backend::GpuDeviceMemoryLimit; /// `config` in the file schema. A leaf left `None` is unset and takes an /// automatic value. TLS paths come from `tls`, because the tonic config keeps -/// no paths. The struct literals are exhaustive so a new file field must be -/// projected here before the crate compiles. +/// no paths. Iceberg catalogs stay out of the table so their secrets cannot +/// reach a rendering; [`render_iceberg_catalogs`] prints them. The struct +/// literals are exhaustive so a new file field must be projected here before +/// the crate compiles. fn project(config: &ServerConfig, tls: Option<&ServerTlsFileConfig>) -> ServerFile { let native = config.native_optimizer_config.native_execution_config(); let chunking = native.source_chunking(); @@ -75,6 +81,13 @@ fn project(config: &ServerConfig, tls: Option<&ServerTlsFileConfig>) -> ServerFi None => ServerModelsFileConfig { root: None, max_model_bytes: None }, }, data: ServerDataFileConfig { root: config.data_root.clone() }, + s3: native.s3_storage().map(|storage| ServerS3FileConfig { + endpoint: storage.endpoint.clone(), + region: storage.region.clone(), + path_style: storage.virtual_hosted_style_request.map(|virtual_hosted| !virtual_hosted), + unsigned: Some(storage.credential == S3SourceStorageCredential::Unsigned), + }), + iceberg: ServerIcebergFileConfig { catalogs: Vec::new() }, auth: config.auth.clone(), tls: tls.cloned(), } @@ -181,9 +194,63 @@ pub(super) fn render(config: &ServerConfig, sources: &ConfigSources) -> String { _ => {}, } } + render_iceberg_catalogs(&mut out, &config.iceberg_catalogs, sources); out } +/// `[[iceberg.catalogs]]` from the typed config. `credential` and `token` +/// render as `is set` comments. +fn render_iceberg_catalogs(out: &mut String, catalogs: &[ServerIcebergCatalogConfig], sources: &ConfigSources) { + let file_catalogs = table_lookup(&sources.file_table, "iceberg.catalogs").and_then(toml::Value::as_array); + for (index, catalog) in catalogs.iter().enumerate() { + let file_entry = file_catalogs.and_then(|entries| entries.get(index)).and_then(toml::Value::as_table); + let file_auth = file_entry.and_then(|entry| entry.get("auth")).and_then(toml::Value::as_table); + let _ = writeln!(out, "\n[[iceberg.catalogs]]"); + render_string(out, "name", Some(&catalog.name), file_entry); + match &catalog.kind { + ServerIcebergCatalogKind::Rest(rest) => { + render_string(out, "kind", Some("rest"), file_entry); + render_string(out, "uri", Some(&rest.uri), file_entry); + render_string(out, "warehouse", rest.warehouse.as_deref(), file_entry); + render_string(out, "prefix", rest.prefix.as_deref(), file_entry); + let _ = writeln!(out, "\n[iceberg.catalogs.auth]"); + match &rest.auth { + ServerIcebergRestAuth::None => render_string(out, "kind", Some("none"), file_auth), + ServerIcebergRestAuth::OAuth2 { server_uri, scope, .. } => { + render_string(out, "kind", Some("oauth2"), file_auth); + let _ = writeln!(out, "# credential is set # source: file"); + render_string(out, "server_uri", server_uri.as_deref(), file_auth); + render_string(out, "scope", scope.as_deref(), file_auth); + }, + ServerIcebergRestAuth::Token(_) => { + render_string(out, "kind", Some("token"), file_auth); + let _ = writeln!(out, "# token is set # source: file"); + }, + } + }, + ServerIcebergCatalogKind::Glue(glue) => { + render_string(out, "kind", Some("glue"), file_entry); + render_string(out, "warehouse", Some(&glue.warehouse), file_entry); + render_string(out, "catalog_id", glue.catalog_id.as_deref(), file_entry); + render_string(out, "region", glue.region.as_deref(), file_entry); + render_string(out, "endpoint", glue.endpoint.as_deref(), file_entry); + }, + } + } +} + +fn render_string(out: &mut String, key: &str, value: Option<&str>, file: Option<&toml::Table>) { + let source = if file.is_some_and(|file| file.contains_key(key)) { "file" } else { "default" }; + match value { + Some(value) => { + let _ = writeln!(out, "{key} = {} # source: {source}", render_value(&toml::Value::from(value))); + }, + None => { + let _ = writeln!(out, "# {key} is unset # source: {source}"); + }, + } +} + fn render_device_profiles(out: &mut String, effective: &toml::Table, sources: &ConfigSources) { let profiles = table_lookup(effective, "admission.device_profiles").and_then(toml::Value::as_array); let file_profiles = table_lookup(&sources.file_table, "admission.device_profiles").and_then(toml::Value::as_array); diff --git a/crates/algeon-datafusion/src/server/config/file.rs b/crates/algeon-datafusion/src/server/config/file.rs index 29eeb74d1..08c5eba7e 100644 --- a/crates/algeon-datafusion/src/server/config/file.rs +++ b/crates/algeon-datafusion/src/server/config/file.rs @@ -1,3 +1,5 @@ +use std::fmt; + use serde::Serialize; use toml::de::Error as TomlDeError; @@ -20,6 +22,8 @@ pub(super) struct ServerFile { pub(super) cugraph: ServerCugraphFileConfig, pub(super) models: ServerModelsFileConfig, pub(super) data: ServerDataFileConfig, + pub(super) s3: Option, + pub(super) iceberg: ServerIcebergFileConfig, pub(super) auth: AuthConfig, pub(super) tls: Option, } @@ -61,6 +65,83 @@ pub(super) struct ServerDataFileConfig { pub(super) root: Option, } +/// `[s3]`: the storage every `s3://` source reads through. Its presence, +/// even as an empty table, enables `s3://` locations; unset fields follow the +/// AWS environment. +#[derive(Debug, Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields, default)] +pub(super) struct ServerS3FileConfig { + pub(super) endpoint: Option, + pub(super) region: Option, + pub(super) path_style: Option, + pub(super) unsigned: Option, +} + +/// `[iceberg]`: holds only the `[[iceberg.catalogs]]` array, so it has no +/// environment override form. +#[derive(Debug, Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields, default)] +pub(super) struct ServerIcebergFileConfig { + pub(super) catalogs: Vec, +} + +/// One `[[iceberg.catalogs]]` entry, tagged by `kind`. Each kind accepts only +/// its own keys. +#[derive(Debug, Deserialize, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub(super) enum ServerIcebergCatalogFileConfig { + Rest { + name: String, + uri: String, + warehouse: Option, + prefix: Option, + #[serde(default)] + auth: ServerIcebergRestAuthFileConfig, + }, + Glue { + name: String, + warehouse: String, + catalog_id: Option, + region: Option, + endpoint: Option, + }, +} + +/// `auth` of a REST catalog entry, tagged by `kind`. `None` is an empty +/// struct variant because serde ignores unknown keys beside the tag of a unit +/// variant, even under `deny_unknown_fields`. +#[derive(Debug, Deserialize, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub(super) enum ServerIcebergRestAuthFileConfig { + None {}, + #[serde(rename = "oauth2")] + OAuth2 { + credential: FileSecret, + server_uri: Option, + scope: Option, + }, + Token { + token: FileSecret, + }, +} + +impl Default for ServerIcebergRestAuthFileConfig { + fn default() -> Self { + Self::None {} + } +} + +/// A secret read from the config file, redacted from `Debug`. +#[derive(Deserialize, Serialize)] +#[serde(transparent)] +pub(super) struct FileSecret(pub(super) String); + +impl fmt::Debug for FileSecret { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("") + } +} + #[derive(Debug, Default, Deserialize, Serialize)] #[serde(deny_unknown_fields, default)] pub(super) struct ServerObservabilityFileConfig { @@ -154,6 +235,13 @@ impl ServerFile { }, models: ServerModelsFileConfig { root: Some(PathBuf::new()), max_model_bytes: Some(0) }, data: ServerDataFileConfig { root: Some(PathBuf::new()) }, + s3: Some(ServerS3FileConfig { + endpoint: Some(String::new()), + region: Some(String::new()), + path_style: Some(false), + unsigned: Some(false), + }), + iceberg: ServerIcebergFileConfig { catalogs: Vec::new() }, auth: AuthConfig { mode: AuthMode::BasicBearer { users: Vec::new(), token_ttl_secs: 0 } }, tls: Some(ServerTlsFileConfig { cert_path: PathBuf::new(), diff --git a/crates/algeon-datafusion/src/server/config/iceberg.rs b/crates/algeon-datafusion/src/server/config/iceberg.rs new file mode 100644 index 000000000..4e529c130 --- /dev/null +++ b/crates/algeon-datafusion/src/server/config/iceberg.rs @@ -0,0 +1,195 @@ +use std::{collections::BTreeSet, fmt}; + +use super::{ + ConfigSources, Result, S3SourceStorageProfile, + file::{FileSecret, ServerIcebergCatalogFileConfig, ServerIcebergFileConfig, ServerIcebergRestAuthFileConfig}, +}; + +/// DataFusion's default catalog name, which a server session always holds. +const DEFAULT_DATAFUSION_CATALOG: &str = "datafusion"; + +/// One `[[iceberg.catalogs]]` entry: an Iceberg catalog registered as a +/// DataFusion catalog. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ServerIcebergCatalogConfig { + /// DataFusion catalog name. Loading rejects an empty name, a name used + /// by another entry, and `datafusion`. + pub name: String, + /// Catalog service and its connection settings. + pub kind: ServerIcebergCatalogKind, +} + +/// The catalog service of a [`ServerIcebergCatalogConfig`], from its `kind` key. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ServerIcebergCatalogKind { + /// `kind = "rest"`: an Iceberg REST catalog. + Rest(ServerIcebergRestCatalogConfig), + /// `kind = "glue"`: the AWS Glue Data Catalog. + Glue(ServerIcebergGlueCatalogConfig), +} + +/// Settings of an Iceberg REST catalog entry. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ServerIcebergRestCatalogConfig { + /// Catalog base URI, such as `http://127.0.0.1:8181`. + pub uri: String, + /// Warehouse identifier sent to the catalog's config endpoint. + pub warehouse: Option, + /// Path prefix inserted after the catalog URI's version segment. + pub prefix: Option, + /// Authentication against the catalog service. + pub auth: ServerIcebergRestAuth, +} + +/// Authentication of an Iceberg REST catalog entry, from its `auth` table. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub enum ServerIcebergRestAuth { + /// `kind = "none"`, the default: requests carry no authorization. + #[default] + None, + /// `kind = "oauth2"`: client credentials exchanged for a token. + OAuth2 { + /// `client_id:client_secret`, or a client secret alone. + credential: ServerIcebergSecret, + /// Token endpoint. Unset uses the catalog's own token endpoint. + server_uri: Option, + /// OAuth2 scope requested with the token. + scope: Option, + }, + /// `kind = "token"`: a bearer token sent with every request. + Token(ServerIcebergSecret), +} + +/// Settings of an AWS Glue catalog entry. Credentials come from the default +/// AWS credential chain. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ServerIcebergGlueCatalogConfig { + /// Warehouse location, such as `s3://bucket/warehouse`. + pub warehouse: String, + /// AWS account ID of the Glue catalog. Unset uses the caller's account. + pub catalog_id: Option, + /// Region of the Glue service. Unset follows the AWS environment. + pub region: Option, + /// Glue service endpoint override. Loading requires `[s3]` to name its + /// endpoint too: iceberg-rust's Glue catalog reads table metadata through + /// this endpoint when no S3 endpoint is set. + pub endpoint: Option, +} + +/// A catalog secret from the config file. `Debug` prints ``, and +/// the type has no `Display`; [`Self::expose`] returns the value. +#[derive(Clone, PartialEq, Eq)] +pub struct ServerIcebergSecret(String); + +impl ServerIcebergSecret { + /// Wraps `secret`. + pub fn new(secret: impl Into) -> Self { + Self(secret.into()) + } + + /// The secret value, for the catalog client that sends it. + pub fn expose(&self) -> &str { + &self.0 + } +} + +impl fmt::Debug for ServerIcebergSecret { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("") + } +} + +impl From for ServerIcebergSecret { + fn from(secret: FileSecret) -> Self { + Self(secret.0) + } +} + +/// Validates the `[[iceberg.catalogs]]` entries. Iceberg data files are read +/// through the `[s3]` storage, so any entry requires `[s3]`. +pub(super) fn iceberg_catalogs( + file: ServerIcebergFileConfig, + s3: Option<&S3SourceStorageProfile>, + sources: &ConfigSources, +) -> Result> { + if !file.catalogs.is_empty() && s3.is_none() { + return Err(sources + .invalid( + "iceberg.catalogs", + "needs an `[s3]` table, which the server reads Iceberg data files through; add `[s3]`", + ) + .with_fact("reason", "iceberg_catalogs_require_s3")); + } + let mut names = BTreeSet::new(); + let mut catalogs = Vec::with_capacity(file.catalogs.len()); + for entry in file.catalogs { + let catalog = catalog_config(entry); + if catalog.name.is_empty() { + return Err(sources.invalid("iceberg.catalogs.name", "must not be empty")); + } + if catalog.name == DEFAULT_DATAFUSION_CATALOG { + return Err(sources + .invalid( + "iceberg.catalogs.name", + format!("`{DEFAULT_DATAFUSION_CATALOG}` is the default DataFusion catalog; choose another name"), + ) + .with_fact("reason", "iceberg_catalog_name_reserved")); + } + if !names.insert(catalog.name.clone()) { + return Err(sources + .invalid( + "iceberg.catalogs.name", + format!("`{}` names two entries; give each catalog its own name", catalog.name), + ) + .with_fact("reason", "duplicate_iceberg_catalog")); + } + // iceberg-rust's Glue catalog copies its endpoint into the table's + // FileIO `s3.endpoint` when that property is unset, which sends + // metadata reads to the Glue host and fails the scan's storage check. + if let ServerIcebergCatalogKind::Glue(glue) = &catalog.kind + && glue.endpoint.is_some() + && s3.and_then(|storage| storage.endpoint.as_deref()).is_none_or(|endpoint| endpoint.trim().is_empty()) + { + return Err(sources + .invalid( + "iceberg.catalogs.endpoint", + format!( + "is set for Glue catalog `{}` while `[s3]` names no `endpoint`; set `s3.endpoint`, for example to the regional S3 endpoint, or remove the Glue endpoint", + catalog.name + ), + ) + .with_fact("reason", "iceberg_glue_endpoint_requires_s3_endpoint")); + } + catalogs.push(catalog); + } + Ok(catalogs) +} + +fn catalog_config(entry: ServerIcebergCatalogFileConfig) -> ServerIcebergCatalogConfig { + match entry { + ServerIcebergCatalogFileConfig::Rest { name, uri, warehouse, prefix, auth } => { + let auth = match auth { + ServerIcebergRestAuthFileConfig::None {} => ServerIcebergRestAuth::None, + ServerIcebergRestAuthFileConfig::OAuth2 { credential, server_uri, scope } => { + ServerIcebergRestAuth::OAuth2 { credential: credential.into(), server_uri, scope } + }, + ServerIcebergRestAuthFileConfig::Token { token } => ServerIcebergRestAuth::Token(token.into()), + }; + ServerIcebergCatalogConfig { + name, + kind: ServerIcebergCatalogKind::Rest(ServerIcebergRestCatalogConfig { uri, warehouse, prefix, auth }), + } + }, + ServerIcebergCatalogFileConfig::Glue { name, warehouse, catalog_id, region, endpoint } => { + ServerIcebergCatalogConfig { + name, + kind: ServerIcebergCatalogKind::Glue(ServerIcebergGlueCatalogConfig { + warehouse, + catalog_id, + region, + endpoint, + }), + } + }, + } +} diff --git a/crates/algeon-datafusion/src/server/config/mod.rs b/crates/algeon-datafusion/src/server/config/mod.rs index 5f0f54bd9..c9d1fe8ab 100644 --- a/crates/algeon-datafusion/src/server/config/mod.rs +++ b/crates/algeon-datafusion/src/server/config/mod.rs @@ -20,7 +20,7 @@ use crate::server::{ coordinator::QueryCoordinatorConfig, cugraph_sql::{CugraphGraphConstructionPolicy, CugraphGraphOptions}, cuml_sql::CumlModelStoreConfig, - native_exec::{AlgeonNativeExecutionConfig, SourceChunking}, + native_exec::{AlgeonNativeExecutionConfig, S3SourceStorageProfile, SourceChunking}, planner::AlgeonNativeOptimizerConfig, }; @@ -30,9 +30,11 @@ mod data; mod effective; mod env; mod file; +mod iceberg; mod models; mod native; mod observability; +mod s3; mod tls; use admission::{server_device_profiles, server_queue_policy}; @@ -43,10 +45,17 @@ pub(crate) use effective::non_default_settings; pub use env::{ALGEON_SERVER_CONFIG_FILE_ENV, ALGEON_SERVER_LOG_ENV}; use env::{EnvOverride, ServerEnvironment}; use file::{ServerFile, ServerTlsFileConfig, load_file}; +use iceberg::iceberg_catalogs; +pub use iceberg::{ + ServerIcebergCatalogConfig, ServerIcebergCatalogKind, ServerIcebergGlueCatalogConfig, ServerIcebergRestAuth, + ServerIcebergRestCatalogConfig, ServerIcebergSecret, +}; use models::models_config; use native::native_optimizer_config; pub use observability::ServerObservabilityConfig; use observability::server_observability_config; +use s3::s3_storage; +pub(super) use tls::install_rustls_crypto_provider; use tls::tls_config; const DEFAULT_BIND_ADDR: &str = "127.0.0.1:50051"; @@ -144,6 +153,10 @@ pub struct ServerConfig { /// /// [`FlightSqlServerBuilder::build`]: crate::server::FlightSqlServerBuilder::build pub init_sql: Option, + /// Iceberg catalogs from the `[[iceberg.catalogs]]` array, in file order. + /// Loading requires `[s3]` when any entry is present and rejects + /// duplicate names and the name `datafusion`. + pub iceberg_catalogs: Vec, } impl ServerConfig { @@ -158,6 +171,12 @@ impl ServerConfig { self } + /// The `[s3]` storage every `s3://` source reads through, or `None` + /// without the table. + pub fn s3_storage(&self) -> Option<&S3SourceStorageProfile> { + self.native_optimizer_config.native_execution_config().s3_storage() + } + /// The selected CUDA ordinals: the ordinals of /// [`Self::gpu_device_profiles`], in profile order. pub fn gpu_devices(&self) -> Vec { @@ -203,7 +222,9 @@ impl ServerConfig { let queue_policy = server_queue_policy(&file.admission, sources)?; let (gpu_device_profiles, device_profiles_defaulted) = server_device_profiles(&file.admission)?; let observability = server_observability_config(&file.observability, sources)?; - let native_optimizer_config = native_optimizer_config(&file.native, sources)?; + let s3_storage = s3_storage(file.s3.as_ref(), sources)?; + let iceberg_catalogs = iceberg_catalogs(file.iceberg, s3_storage.as_ref(), sources)?; + let native_optimizer_config = native_optimizer_config(&file.native, s3_storage, sources)?; let cugraph = cugraph_config(&file.cugraph, sources)?; let models = models_config(&file.models, sources)?; let data_root = data_root(&file.data, sources)?; @@ -226,6 +247,7 @@ impl ServerConfig { data_root, tls, init_sql, + iceberg_catalogs, }; config.validate_gpu_device_profiles()?; config.check_admission_within_live_query_capacity(sources)?; diff --git a/crates/algeon-datafusion/src/server/config/native.rs b/crates/algeon-datafusion/src/server/config/native.rs index 158a22f5e..d5214b7f5 100644 --- a/crates/algeon-datafusion/src/server/config/native.rs +++ b/crates/algeon-datafusion/src/server/config/native.rs @@ -4,10 +4,11 @@ use super::{ AlgeonNativeExecutionConfig, AlgeonNativeOptimizerConfig, ConfigSources, Result, SourceChunkPlanningMode, SourceChunking, SourceSchedulerPolicy, StatisticalAggregateExecutionMode, file::ServerNativeFileConfig, }; -use crate::server::planner::AlgeonGpuExecutionMode; +use crate::server::{native_exec::S3SourceStorageProfile, planner::AlgeonGpuExecutionMode}; pub(super) fn native_optimizer_config( file: &ServerNativeFileConfig, + s3_storage: Option, sources: &ConfigSources, ) -> Result { let defaults = SourceChunking::default(); @@ -43,6 +44,9 @@ pub(super) fn native_optimizer_config( execution_config = execution_config .with_statistical_aggregate_execution_mode(parse_statistical_aggregate_execution_mode(sources, raw)?); } + if let Some(storage) = s3_storage { + execution_config = execution_config.with_s3_storage(storage); + } let mut optimizer_config = AlgeonNativeOptimizerConfig::default().with_native_execution_config(execution_config); if let Some(raw) = &file.execution_mode { @@ -63,13 +67,13 @@ fn nonzero( fn parse_source_chunk_planning_mode(sources: &ConfigSources, raw: &str) -> Result { let raw = raw.trim().to_ascii_lowercase(); - [SourceChunkPlanningMode::Conservative, SourceChunkPlanningMode::LocalParquetThroughput] + [SourceChunkPlanningMode::Conservative, SourceChunkPlanningMode::ParquetThroughput] .into_iter() .find(|mode| mode.as_str() == raw) .ok_or_else(|| { sources.invalid( "native.source_chunk_planning_mode", - format!("must be `conservative` or `local_parquet_throughput`, got `{raw}`"), + format!("must be `conservative` or `parquet_throughput`, got `{raw}`"), ) }) } diff --git a/crates/algeon-datafusion/src/server/config/s3.rs b/crates/algeon-datafusion/src/server/config/s3.rs new file mode 100644 index 000000000..8cdfeaebf --- /dev/null +++ b/crates/algeon-datafusion/src/server/config/s3.rs @@ -0,0 +1,38 @@ +use super::{ConfigSources, Result, S3SourceStorageProfile, file::ServerS3FileConfig}; + +/// The storage `[s3]` describes, or `None` without the table. Credentials +/// come from the default AWS chain unless `unsigned` is set. +/// +/// The native Parquet reader addresses a custom endpoint path-style only +/// and rejects virtual-hosted addressing there at query time, so loading +/// rejects that combination first. +pub(super) fn s3_storage( + file: Option<&ServerS3FileConfig>, + sources: &ConfigSources, +) -> Result> { + let Some(file) = file else { + return Ok(None); + }; + if file.endpoint.is_some() && file.path_style == Some(false) { + return Err(sources + .invalid( + "s3.path_style", + "is `false`, and virtual-hosted addressing is unsupported with a custom `s3.endpoint`; set `path_style = true` or remove `endpoint`", + ) + .with_fact("reason", "s3_custom_endpoint_requires_path_style")); + } + let mut storage = S3SourceStorageProfile::default_chain(); + if let Some(endpoint) = &file.endpoint { + storage = storage.with_endpoint(endpoint); + } + if let Some(region) = &file.region { + storage = storage.with_region(region); + } + if let Some(path_style) = file.path_style { + storage = storage.with_path_style_access(path_style); + } + if file.unsigned == Some(true) { + storage = storage.with_unsigned_requests(); + } + Ok(Some(storage)) +} diff --git a/crates/algeon-datafusion/src/server/config/tls.rs b/crates/algeon-datafusion/src/server/config/tls.rs index 7b2ab1988..eb1e70d6a 100644 --- a/crates/algeon-datafusion/src/server/config/tls.rs +++ b/crates/algeon-datafusion/src/server/config/tls.rs @@ -1,7 +1,17 @@ +use rustls::crypto::ring; use tonic::transport::{Certificate, Identity}; use super::{ConfigSources, Path, Result, ServerTlsConfig, file::ServerTlsFileConfig, fs}; +/// Installs ring as rustls's process-wide crypto provider unless the process +/// already installed one. tonic builds its TLS acceptor from that provider, +/// and rustls selects none by itself when the build enables both its `ring` +/// and `aws-lc-rs` backends. +pub(in crate::server) fn install_rustls_crypto_provider() { + // An error means another provider is installed, and tonic uses that one. + let _ = ring::default_provider().install_default(); +} + /// Reads the `[tls]` PEM files into the listener's tonic config. Only file /// access is checked here. [`FlightSqlServerBuilder::build`] parses the PEM and /// matches key to certificate, so a config assembled in Rust takes the same @@ -19,6 +29,7 @@ pub(super) fn tls_config( read_pem(sources, "tls.cert_path", &file.cert_path)?, read_pem(sources, "tls.key_path", &file.key_path)?, ); + install_rustls_crypto_provider(); let mut config = ServerTlsConfig::new().identity(identity); if let Some(path) = &file.client_ca_path { config = config.client_ca_root(Certificate::from_pem(read_pem(sources, "tls.client_ca_path", path)?)); diff --git a/crates/algeon-datafusion/src/server/flight/statement.rs b/crates/algeon-datafusion/src/server/flight/statement.rs index 99d252553..c41a24c3a 100644 --- a/crates/algeon-datafusion/src/server/flight/statement.rs +++ b/crates/algeon-datafusion/src/server/flight/statement.rs @@ -162,7 +162,8 @@ impl DatafusionAlgeonFlightSqlService { let invocation = Invocation::detached(); invocation.set_execution(execution); let mut state = self.ctx.state(); - invocation.install(state.config_mut()); + let runtime_env = Arc::clone(state.runtime_env()); + invocation.install(state.config_mut(), &runtime_env); return Ok(SessionContext::new_with_state(state)); }; attempt.planning_context(&self.ctx, execution) diff --git a/crates/algeon-datafusion/src/server/mod.rs b/crates/algeon-datafusion/src/server/mod.rs index 74663e2bd..f26080521 100644 --- a/crates/algeon-datafusion/src/server/mod.rs +++ b/crates/algeon-datafusion/src/server/mod.rs @@ -63,7 +63,9 @@ mod session; pub use auth::{AuthConfig, AuthMode, PasswordHashError, StaticUserConfig, hash_password, verify_password}; pub use config::{ ALGEON_SERVER_CONFIG_FILE_ENV, ALGEON_SERVER_LOG_ENV, ConfigValueSource, CugraphServerConfig, Http2Keepalive, - LoadedServerConfig, ServerConfig, ServerObservabilityConfig, + LoadedServerConfig, ServerConfig, ServerIcebergCatalogConfig, ServerIcebergCatalogKind, + ServerIcebergGlueCatalogConfig, ServerIcebergRestAuth, ServerIcebergRestCatalogConfig, ServerIcebergSecret, + ServerObservabilityConfig, }; pub use error::{ BLOCKING_KIND_METADATA_KEY, BLOCKING_SUBJECT_METADATA_KEY, ERROR_CODE_METADATA_KEY, ERROR_KIND_METADATA_KEY, @@ -188,7 +190,9 @@ impl FlightSqlServerBuilder { /// Injects the shared base DataFusion session. /// /// An injected backend is required. [`Self::build`] validates that the session - /// is bound to that backend and satisfies the configured SQL-family and catalog contracts. + /// is bound to that backend and, under an execution mode that rejects + /// DataFusion CPU operators, that it plans one target partition. Catalogs + /// and SQL families registered on the session are used as supplied. pub fn session_context(mut self, ctx: Arc) -> Self { self.session_context = Some(ctx); self @@ -420,6 +424,8 @@ fn transport_server(config: &ServerConfig) -> Result { let Some(tls) = &config.tls else { return Ok(server); }; + // A config assembled in Rust skips the loader, which installs the provider. + config::install_rustls_crypto_provider(); server.tls_config(tls.clone()).map_err(|error| { ServerError::invalid_config_with_source( "tls", diff --git a/crates/algeon-datafusion/src/server/tests.rs b/crates/algeon-datafusion/src/server/tests.rs index 58d8425f3..4f79a82cb 100644 --- a/crates/algeon-datafusion/src/server/tests.rs +++ b/crates/algeon-datafusion/src/server/tests.rs @@ -67,6 +67,7 @@ fn base_config() -> ServerConfig { data_root: None, tls: None, init_sql: None, + iceberg_catalogs: Vec::new(), } } diff --git a/crates/algeon-datafusion/tests/server_config_tests_cpu/server_config_tests/iceberg.rs b/crates/algeon-datafusion/tests/server_config_tests_cpu/server_config_tests/iceberg.rs new file mode 100644 index 000000000..2f79e3bce --- /dev/null +++ b/crates/algeon-datafusion/tests/server_config_tests_cpu/server_config_tests/iceberg.rs @@ -0,0 +1,231 @@ +use algeon_datafusion::server::{ + ServerIcebergCatalogConfig, ServerIcebergCatalogKind, ServerIcebergGlueCatalogConfig, ServerIcebergRestAuth, + ServerIcebergRestCatalogConfig, ServerIcebergSecret, +}; + +use super::*; + +const S3: &str = "[s3]\nendpoint = \"http://127.0.0.1:9000\"\npath_style = true\nregion = \"eu-west-2\"\n"; + +fn with_s3(catalogs: &str) -> String { + format!("{S3}\n{catalogs}") +} + +#[test] +fn iceberg_rest_and_glue_catalogs_load_from_the_file() { + let config = load_toml(&with_s3( + r#" +[[iceberg.catalogs]] +name = "lake" +kind = "rest" +uri = "http://127.0.0.1:8181" +warehouse = "s3://warehouse/" +prefix = "tenant" + +[iceberg.catalogs.auth] +kind = "oauth2" +credential = "client:secret" +server_uri = "http://127.0.0.1:8181/v1/oauth/tokens" +scope = "catalog" + +[[iceberg.catalogs]] +name = "tokens" +kind = "rest" +uri = "http://127.0.0.1:8182" +auth = { kind = "token", token = "bearer-value" } + +[[iceberg.catalogs]] +name = "open" +kind = "rest" +uri = "http://127.0.0.1:8183" + +[[iceberg.catalogs]] +name = "glue" +kind = "glue" +warehouse = "s3://bucket/warehouse" +catalog_id = "123456789012" +region = "eu-west-2" +endpoint = "http://127.0.0.1:5000" +"#, + )) + .unwrap(); + + let rest = + |uri: &str, auth| ServerIcebergRestCatalogConfig { uri: uri.to_string(), warehouse: None, prefix: None, auth }; + let expected = vec![ + ServerIcebergCatalogConfig { + name: "lake".to_string(), + kind: ServerIcebergCatalogKind::Rest(ServerIcebergRestCatalogConfig { + warehouse: Some("s3://warehouse/".to_string()), + prefix: Some("tenant".to_string()), + ..rest( + "http://127.0.0.1:8181", + ServerIcebergRestAuth::OAuth2 { + credential: ServerIcebergSecret::new("client:secret"), + server_uri: Some("http://127.0.0.1:8181/v1/oauth/tokens".to_string()), + scope: Some("catalog".to_string()), + }, + ) + }), + }, + ServerIcebergCatalogConfig { + name: "tokens".to_string(), + kind: ServerIcebergCatalogKind::Rest(rest( + "http://127.0.0.1:8182", + ServerIcebergRestAuth::Token(ServerIcebergSecret::new("bearer-value")), + )), + }, + ServerIcebergCatalogConfig { + name: "open".to_string(), + kind: ServerIcebergCatalogKind::Rest(rest("http://127.0.0.1:8183", ServerIcebergRestAuth::None)), + }, + ServerIcebergCatalogConfig { + name: "glue".to_string(), + kind: ServerIcebergCatalogKind::Glue(ServerIcebergGlueCatalogConfig { + warehouse: "s3://bucket/warehouse".to_string(), + catalog_id: Some("123456789012".to_string()), + region: Some("eu-west-2".to_string()), + endpoint: Some("http://127.0.0.1:5000".to_string()), + }), + }, + ]; + assert_eq!(config.iceberg_catalogs, expected); + let ServerIcebergCatalogKind::Rest(ServerIcebergRestCatalogConfig { + auth: ServerIcebergRestAuth::OAuth2 { credential, .. }, + .. + }) = &config.iceberg_catalogs[0].kind + else { + panic!("expected an OAuth2 REST catalog"); + }; + assert_eq!(credential.expose(), "client:secret"); + assert!(config.s3_storage().is_some()); +} + +#[test] +fn iceberg_catalogs_require_the_s3_table() { + let error = load_toml("[[iceberg.catalogs]]\nname = \"lake\"\nkind = \"rest\"\nuri = \"http://127.0.0.1:8181\"\n") + .expect_err("catalogs without [s3]"); + assert_eq!(error.code(), ErrorCode::InvalidConfiguration, "{error}"); + assert_eq!(error.fact_value("config_field"), Some("iceberg.catalogs"), "{error}"); + assert_eq!(error.fact_value("reason"), Some("iceberg_catalogs_require_s3"), "{error}"); + assert!(error.message().contains("add `[s3]`"), "{error}"); +} + +#[test] +fn glue_endpoint_override_requires_an_s3_endpoint() { + let glue = "[[iceberg.catalogs]]\nname = \"lake\"\nkind = \"glue\"\nwarehouse = \"s3://b/w\"\nendpoint = \"http://127.0.0.1:5001\"\n"; + let error = + load_toml(&format!("[s3]\nregion = \"eu-west-2\"\n\n{glue}")).expect_err("Glue endpoint without s3.endpoint"); + assert_eq!(error.code(), ErrorCode::InvalidConfiguration, "{error}"); + assert_eq!(error.fact_value("config_field"), Some("iceberg.catalogs.endpoint"), "{error}"); + assert_eq!(error.fact_value("reason"), Some("iceberg_glue_endpoint_requires_s3_endpoint"), "{error}"); + + let config = load_toml(&format!("[s3]\nendpoint = \"http://127.0.0.1:9000\"\npath_style = true\n\n{glue}")) + .expect("Glue endpoint with s3.endpoint"); + assert_eq!(config.iceberg_catalogs.len(), 1); +} + +#[test] +fn iceberg_catalog_entries_reject_missing_and_foreign_keys() { + for (entry, key) in [ + ("kind = \"rest\"\n", "uri"), + ("kind = \"glue\"\n", "warehouse"), + ("uri = \"http://127.0.0.1:8181\"\n", "kind"), + ("kind = \"hive\"\n", "hive"), + ("kind = \"rest\"\nuri = \"http://h\"\nauth = { kind = \"oauth2\" }\n", "credential"), + ("kind = \"rest\"\nuri = \"http://h\"\nauth = { kind = \"token\" }\n", "token"), + ("kind = \"rest\"\nuri = \"http://h\"\nauth = { kind = \"none\", token = \"t\" }\n", "token"), + ("kind = \"rest\"\nuri = \"http://h\"\nauth = { kind = \"token\", token = \"t\", scope = \"s\" }\n", "scope"), + ("kind = \"glue\"\nwarehouse = \"s3://b/w\"\nuri = \"http://h\"\n", "uri"), + ("kind = \"rest\"\nuri = \"http://h\"\ncatalog_id = \"1\"\n", "catalog_id"), + ("kind = \"rest\"\nuri = \"http://h\"\nproperties = { a = \"b\" }\n", "properties"), + ] { + let toml_body = with_s3(&format!("[[iceberg.catalogs]]\nname = \"lake\"\n{entry}")); + let error = load_toml(&toml_body).expect_err(entry); + assert_eq!(error.code(), ErrorCode::InvalidConfiguration, "{entry}: {error}"); + assert!(error.message().contains(&format!("`{key}`")), "{entry}: {error}"); + } +} + +#[test] +fn iceberg_catalog_names_are_unique_and_leave_the_default_catalog() { + let entry = |name: &str| format!("[[iceberg.catalogs]]\nname = \"{name}\"\nkind = \"rest\"\nuri = \"http://h\"\n"); + for (catalogs, reason) in [ + (format!("{}{}", entry("lake"), entry("lake")), Some("duplicate_iceberg_catalog")), + (entry("datafusion"), Some("iceberg_catalog_name_reserved")), + (entry(""), None), + ] { + let error = load_toml(&with_s3(&catalogs)).expect_err(&catalogs); + assert_eq!(error.code(), ErrorCode::InvalidConfiguration, "{error}"); + assert_eq!(error.fact_value("config_field"), Some("iceberg.catalogs.name"), "{error}"); + assert_eq!(error.fact_value("reason"), reason, "{error}"); + } +} + +#[test] +fn iceberg_secrets_stay_out_of_effective_toml_and_debug() { + let (_dir, env) = env_with_server_toml(&with_s3( + r#" +[[iceberg.catalogs]] +name = "lake" +kind = "rest" +uri = "http://127.0.0.1:8181" +auth = { kind = "oauth2", credential = "client:oauth-secret-value" } + +[[iceberg.catalogs]] +name = "tokens" +kind = "rest" +uri = "http://127.0.0.1:8182" +auth = { kind = "token", token = "bearer-secret-value" } + +[[iceberg.catalogs]] +name = "glue" +kind = "glue" +warehouse = "s3://bucket/warehouse" +"#, + )); + let loaded = LoadedServerConfig::from_vars(env).unwrap(); + let rendered = loaded.effective_toml(); + for line in [ + "\n[[iceberg.catalogs]]\nname = \"lake\" # source: file\nkind = \"rest\" # source: file\n", + "# warehouse is unset # source: default", + "\n[iceberg.catalogs.auth]\nkind = \"oauth2\" # source: file\n# credential is set # source: file\n", + "kind = \"token\" # source: file\n# token is set # source: file\n", + "warehouse = \"s3://bucket/warehouse\" # source: file", + ] { + assert!(rendered.contains(line), "missing `{line}` in\n{rendered}"); + } + let parsed = rendered.parse::().expect("the rendering is valid TOML"); + let catalogs = parsed["iceberg"]["catalogs"].as_array().expect("catalog array"); + assert_eq!(catalogs.len(), 3, "{rendered}"); + + let debug = format!("{:?}", loaded.config); + for (text, secret) in [(&rendered, "oauth-secret-value"), (&debug, "oauth-secret-value")] + .into_iter() + .chain([(&rendered, "bearer-secret-value"), (&debug, "bearer-secret-value")]) + { + assert!(!text.contains(secret), "`{secret}` leaked into\n{text}"); + } + assert!(debug.contains(""), "{debug}"); +} + +#[test] +fn iceberg_catalogs_and_their_secrets_have_no_environment_form() { + let toml_body = with_s3( + "[[iceberg.catalogs]]\nname = \"lake\"\nkind = \"rest\"\nuri = \"http://h\"\nauth = { kind = \"token\", token = \"t\" }\n", + ); + let (_dir, env) = env_with_server_toml(&toml_body); + assert_eq!(ServerConfig::from_vars(env.clone()).unwrap().iceberg_catalogs.len(), 1); + + for name in [ + "ALGEON_SERVER_ICEBERG_CATALOGS", + "ALGEON_SERVER_ICEBERG_CATALOGS_AUTH_TOKEN", + "ALGEON_SERVER_ICEBERG_CATALOGS_AUTH_CREDENTIAL", + ] { + let mut env = env.clone(); + env.insert(name, "from-env".to_string()); + let error = ServerConfig::from_vars(env).expect_err(name); + assert_eq!(error.fact_value("reason"), Some("unknown_environment_variable"), "{error}"); + assert_eq!(error.fact_value("config_env"), Some(name), "{error}"); + } +} diff --git a/crates/algeon-datafusion/tests/server_config_tests_cpu/server_config_tests/mod.rs b/crates/algeon-datafusion/tests/server_config_tests_cpu/server_config_tests/mod.rs index 5542fd620..b3d0673ff 100644 --- a/crates/algeon-datafusion/tests/server_config_tests_cpu/server_config_tests/mod.rs +++ b/crates/algeon-datafusion/tests/server_config_tests_cpu/server_config_tests/mod.rs @@ -18,8 +18,10 @@ mod admission; mod cugraph; mod data; mod feature_gates; +mod iceberg; mod models; mod native_profile; +mod s3; mod startup; mod support; mod tls; diff --git a/crates/algeon-datafusion/tests/server_config_tests_cpu/server_config_tests/s3.rs b/crates/algeon-datafusion/tests/server_config_tests_cpu/server_config_tests/s3.rs new file mode 100644 index 000000000..866d39ea6 --- /dev/null +++ b/crates/algeon-datafusion/tests/server_config_tests_cpu/server_config_tests/s3.rs @@ -0,0 +1,100 @@ +use algeon_datafusion::native_exec::S3SourceStorageProfile; + +use super::*; + +fn s3_storage(config: &ServerConfig) -> Option<&S3SourceStorageProfile> { + config.native_optimizer_config.native_execution_config().s3_storage() +} + +#[test] +fn s3_table_sets_the_storage_for_s3_sources() { + let config = load_toml( + r#" +[s3] +endpoint = "http://127.0.0.1:9000" +region = "eu-west-2" +path_style = true +unsigned = true +"#, + ) + .unwrap(); + let expected = S3SourceStorageProfile::default_chain() + .with_endpoint("http://127.0.0.1:9000") + .with_region("eu-west-2") + .with_path_style_access(true) + .with_unsigned_requests(); + assert_eq!(s3_storage(&config), Some(&expected)); + + let config = load_toml("[s3]\n").unwrap(); + assert_eq!(s3_storage(&config), Some(&S3SourceStorageProfile::default_chain())); + + assert!(s3_storage(&load(BTreeMap::new()).unwrap()).is_none()); +} + +#[test] +fn s3_environment_variables_override_the_table_or_enable_it() { + let (_dir, mut env) = env_with_server_toml("[s3]\nregion = \"eu-west-2\"\nunsigned = true\n"); + env.insert("ALGEON_SERVER_S3_REGION", "us-west-2".to_string()); + env.insert("ALGEON_SERVER_S3_UNSIGNED", "false".to_string()); + let loaded = LoadedServerConfig::from_vars(env).unwrap(); + assert_eq!(s3_storage(&loaded.config), Some(&S3SourceStorageProfile::default_chain().with_region("us-west-2"))); + assert_eq!(loaded.source("s3.region"), ConfigValueSource::Env("ALGEON_SERVER_S3_REGION".to_string())); + + let config = load(BTreeMap::from([ + ("ALGEON_SERVER_S3_ENDPOINT", "http://127.0.0.1:9000".to_string()), + ("ALGEON_SERVER_S3_PATH_STYLE", "true".to_string()), + ])) + .unwrap(); + assert_eq!( + s3_storage(&config), + Some( + &S3SourceStorageProfile::default_chain() + .with_endpoint("http://127.0.0.1:9000") + .with_path_style_access(true) + ) + ); +} + +#[test] +fn s3_custom_endpoint_requires_path_style_addressing() { + let error = load_toml("[s3]\nendpoint = \"http://127.0.0.1:9000\"\npath_style = false\n") + .expect_err("a custom endpoint with virtual-hosted addressing"); + assert_eq!(error.code(), ErrorCode::InvalidConfiguration, "{error}"); + assert_eq!(error.fact_value("config_field"), Some("s3.path_style"), "{error}"); + assert_eq!(error.fact_value("reason"), Some("s3_custom_endpoint_requires_path_style"), "{error}"); + assert!(error.message().contains("set `path_style = true`"), "{error}"); + + let (_dir, mut env) = env_with_server_toml("[s3]\nendpoint = \"http://127.0.0.1:9000\"\n"); + env.insert("ALGEON_SERVER_S3_PATH_STYLE", "false".to_string()); + let error = ServerConfig::from_vars(env).expect_err("virtual-hosted addressing set by the environment"); + assert_eq!(error.fact_value("config_env"), Some("ALGEON_SERVER_S3_PATH_STYLE"), "{error}"); + + // KvikIO and object_store address a custom endpoint path-style when the + // flag is unset. + let config = load_toml("[s3]\nendpoint = \"http://127.0.0.1:9000\"\n").unwrap(); + assert_eq!( + s3_storage(&config), + Some(&S3SourceStorageProfile::default_chain().with_endpoint("http://127.0.0.1:9000")) + ); +} + +#[test] +fn effective_toml_prints_the_s3_table_with_sources() { + let (_dir, mut env) = env_with_server_toml("[s3]\nregion = \"eu-west-2\"\n"); + env.insert("ALGEON_SERVER_S3_PATH_STYLE", "true".to_string()); + let rendered = LoadedServerConfig::from_vars(env).unwrap().effective_toml(); + for line in [ + "\n[s3]\n", + "region = \"eu-west-2\" # source: file", + "path_style = true # source: env ALGEON_SERVER_S3_PATH_STYLE", + "unsigned = false # source: default", + "# endpoint is unset # source: default", + ] { + assert!(rendered.contains(line), "missing `{line}` in\n{rendered}"); + } + let parsed = rendered.parse::().expect("the rendering is valid TOML"); + assert!(parsed.contains_key("s3"), "{rendered}"); + + let defaulted = LoadedServerConfig::from_vars(BTreeMap::<&str, String>::new()).unwrap().effective_toml(); + assert!(defaulted.contains("\n# [s3]\n"), "{defaulted}"); +} diff --git a/crates/algeon-datafusion/tests/server_config_tests_cpu/server_config_tests/startup.rs b/crates/algeon-datafusion/tests/server_config_tests_cpu/server_config_tests/startup.rs index 5155219c8..8da6c396a 100644 --- a/crates/algeon-datafusion/tests/server_config_tests_cpu/server_config_tests/startup.rs +++ b/crates/algeon-datafusion/tests/server_config_tests_cpu/server_config_tests/startup.rs @@ -51,7 +51,9 @@ fn an_unknown_server_variable_fails_and_names_the_closest_known_name() { assert_eq!(error.fact_value("reason"), Some("unknown_environment_variable")); // A name far from every setting gets no guess, so a removed setting is // not mistaken for an unrelated one. - for name in ["ALGEON_SERVER_GPU_DEVICES", "ALGEON_SERVER_S3_REGION", "ALGEON_SERVER_CUGRAPH_DIRECTED"] { + for name in + ["ALGEON_SERVER_GPU_DEVICES", "ALGEON_SERVER_WORKSPACE_BACKING_ALIASES", "ALGEON_SERVER_CUGRAPH_DIRECTED"] + { let error = load(BTreeMap::from([(name, "x".to_string())])).expect_err(name); assert_eq!(error.fact_value("suggestion"), None, "{error}"); assert!(error.message().contains("no known setting has a similar name"), "{error}"); diff --git a/crates/algeon-iceberg/Cargo.toml b/crates/algeon-iceberg/Cargo.toml new file mode 100644 index 000000000..052b22438 --- /dev/null +++ b/crates/algeon-iceberg/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "algeon-iceberg" +description = "Iceberg REST and Glue catalogs whose tables Algeon plans and reads as native Parquet scans." +version = { workspace = true } +edition = "2024" +rust-version = { workspace = true } +license.workspace = true +publish = false +repository.workspace = true +homepage.workspace = true +build = "build.rs" + +[lints] +workspace = true + +[dependencies] +algeon-datafusion = { workspace = true, default-features = false, features = ["server"] } +async-trait = { workspace = true } +# Direct sys edge so build.rs bakes the native runtime path into test binaries. +cudf-sys = { workspace = true } +datafusion = { workspace = true } +futures = { workspace = true } +iceberg = { workspace = true } +iceberg-catalog-glue = { workspace = true } +iceberg-catalog-rest = { workspace = true } +iceberg-storage-opendal = { workspace = true } +# Names the credential source of the `[s3]` storage profile. +query-runtime = { workspace = true } +# Typed cause of Iceberg FileIO errors, read by error classification. +opendal = { version = "0.58", default-features = false } +# iceberg-rust builds reqwest without TLS; REST catalogs behind HTTPS need it. +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } +tokio = { workspace = true, features = ["rt-multi-thread", "sync"] } + +[dev-dependencies] +tempfile = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } + +[build-dependencies] +rapids-build-support = { workspace = true } diff --git a/crates/algeon-iceberg/build.rs b/crates/algeon-iceberg/build.rs new file mode 100644 index 000000000..edd18897b --- /dev/null +++ b/crates/algeon-iceberg/build.rs @@ -0,0 +1,7 @@ +fn main() { + rapids_build_support::configure(rapids_build_support::Config { + link_arg_targets: rapids_build_support::LinkArgTargets::new(true, false, false), + consumers: rapids_build_support::RapidsConsumers::new(false, false, false), + nvtx3interop: false, + }); +} diff --git a/crates/algeon-iceberg/src/binding.rs b/crates/algeon-iceberg/src/binding.rs new file mode 100644 index 000000000..9b2a1b67c --- /dev/null +++ b/crates/algeon-iceberg/src/binding.rs @@ -0,0 +1,249 @@ +//! Checks that reading a data file by column name returns the Iceberg columns. +//! +//! Iceberg binds file columns by field id, falling back to the table's name +//! mapping for files written without ids (PyIceberg `add_files` registers +//! such files). DataFusion's Parquet reader and the native reader bind by +//! name. A file is readable here only when both rules select the same file +//! column for every projected field, level by level through structs, lists +//! and maps. A field absent under both rules is read as null, which is +//! correct only for an optional field whose value is not kept in partition +//! data. + +use std::collections::HashSet; + +use datafusion::{ + arrow::datatypes::{DataType, Field, Fields}, + parquet::arrow::PARQUET_FIELD_ID_META_KEY, +}; +use iceberg::spec::{MappedField, NestedField, PartitionSpec, Schema as IcebergSchema, Transform, Type}; + +use crate::error::UnsupportedReason; + +/// Why one projected field of one file is unreadable, with the Iceberg +/// field it concerns. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct BindingRejection { + pub(crate) reason: UnsupportedReason, + pub(crate) field: String, +} + +/// The binding facts of one data file. +pub(crate) struct FileBinding<'a> { + /// Field ids of identity partition sources, whose values Iceberg keeps + /// in partition data when the file lacks the column. + pub(crate) identity_partition_sources: HashSet, + /// The table's name mapping, consulted for files without field ids. + pub(crate) name_mapping: Option<&'a [MappedField]>, +} + +impl FileBinding<'_> { + /// Checks every field in `projected_field_ids` of `schema` against the + /// columns of the file's Arrow schema `file_fields`, as Parquet's Arrow + /// conversion produced it (with `PARQUET:field_id` metadata). + pub(crate) fn check( + &self, + schema: &IcebergSchema, + projected_field_ids: &[i32], + file_fields: &Fields, + ) -> Result<(), BindingRejection> { + let has_field_ids = fields_have_ids(file_fields); + let checker = Checker { identity_partition_sources: &self.identity_partition_sources, has_field_ids }; + for field_id in projected_field_ids { + let Some(field) = schema.as_struct().field_by_id(*field_id) else { + continue; + }; + let mapping = match (has_field_ids, self.name_mapping) { + (true, _) => Vec::new(), + (false, Some(mapping)) => mapping.iter().collect(), + (false, None) => return Err(checker.rejection(UnsupportedReason::MissingFieldIds, &field.name)), + }; + checker.check_struct_member(field, file_fields, &mapping, &field.name)?; + } + Ok(()) + } +} + +/// Name-mapping entries of one level. Empty for files with field ids. +type Mapping<'a> = [&'a MappedField]; + +struct Checker<'a> { + identity_partition_sources: &'a HashSet, + has_field_ids: bool, +} + +impl Checker<'_> { + /// Checks one struct member, found by name and by id among `file_fields`. + fn check_struct_member( + &self, + field: &NestedField, + file_fields: &Fields, + mapping: &Mapping<'_>, + path: &str, + ) -> Result<(), BindingRejection> { + let by_name = file_fields.iter().position(|file_field| file_field.name() == &field.name); + let mapped = mapping.iter().copied().find(|entry| entry.field_id() == Some(field.id)); + let by_id = if self.has_field_ids { + file_fields.iter().position(|file_field| field_id(file_field) == Some(field.id)) + } else { + mapped.and_then(|entry| { + file_fields.iter().position(|file_field| entry.names().iter().any(|name| name == file_field.name())) + }) + }; + let mismatch = + if self.has_field_ids { UnsupportedReason::FieldIdMismatch } else { UnsupportedReason::MissingFieldIds }; + match (by_name, by_id) { + (None, None) => { + if self.identity_partition_sources.contains(&field.id) { + Err(self.rejection(UnsupportedReason::PartitionColumnAbsent, path)) + } else if field.required { + Err(self.rejection(UnsupportedReason::RequiredColumnAbsent, path)) + } else { + Ok(()) + } + }, + (Some(by_name), Some(by_id)) if by_name == by_id => { + self.check_nested(&field.field_type, &file_fields[by_name], &children(mapped), path) + }, + _ => Err(self.rejection(mismatch, path)), + } + } + + /// Checks the children of a column both rules selected. + fn check_nested( + &self, + field_type: &Type, + file_field: &Field, + mapping: &Mapping<'_>, + path: &str, + ) -> Result<(), BindingRejection> { + match (field_type, file_field.data_type()) { + (Type::Struct(struct_type), DataType::Struct(file_children)) => { + for child in struct_type.fields() { + let child_path = format!("{path}.{}", child.name); + self.check_struct_member(child, file_children, mapping, &child_path)?; + } + Ok(()) + }, + (Type::List(list), DataType::List(element) | DataType::LargeList(element)) => { + self.check_positional(&list.element_field, element, mapping, path) + }, + (Type::Map(map), DataType::Map(entries, _)) => { + let DataType::Struct(entry_fields) = entries.data_type() else { + return Ok(()); + }; + let [key, value] = entry_fields.iter().collect::>()[..] else { + return Ok(()); + }; + self.check_positional(&map.key_field, key, mapping, path)?; + self.check_positional(&map.value_field, value, mapping, path) + }, + _ => Ok(()), + } + } + + /// List elements and map keys and values bind by position, so only + /// their field ids, when the file has them, and their children are + /// compared. + fn check_positional( + &self, + field: &NestedField, + file_field: &Field, + mapping: &Mapping<'_>, + path: &str, + ) -> Result<(), BindingRejection> { + if self.has_field_ids && field_id(file_field).is_some_and(|id| id != field.id) { + return Err(self.rejection(UnsupportedReason::FieldIdMismatch, path)); + } + let mapped = mapping.iter().copied().find(|entry| entry.field_id() == Some(field.id)); + self.check_nested(&field.field_type, file_field, &children(mapped), path) + } + + fn rejection(&self, reason: UnsupportedReason, path: &str) -> BindingRejection { + BindingRejection { reason, field: path.to_string() } + } +} + +fn children(entry: Option<&MappedField>) -> Vec<&MappedField> { + entry.map(|entry| entry.fields().iter().map(AsRef::as_ref).collect()).unwrap_or_default() +} + +/// Field ids of the identity partition sources of a partition spec. +pub(crate) fn identity_partition_sources(spec: &PartitionSpec) -> HashSet { + spec.fields().iter().filter(|field| field.transform == Transform::Identity).map(|field| field.source_id).collect() +} + +fn field_id(field: &Field) -> Option { + field.metadata().get(PARQUET_FIELD_ID_META_KEY).and_then(|id| id.parse().ok()) +} + +/// Whether any column of the file, at any level, carries a field id. +fn fields_have_ids(fields: &Fields) -> bool { + fields.iter().any(|field| field_id(field).is_some() || data_type_has_ids(field.data_type())) +} + +fn data_type_has_ids(data_type: &DataType) -> bool { + match data_type { + DataType::Struct(children) => fields_have_ids(children), + DataType::List(child) | DataType::LargeList(child) | DataType::Map(child, _) => { + field_id(child).is_some() || data_type_has_ids(child.data_type()) + }, + _ => false, + } +} + +/// The first projected column that the file stores with another type than +/// the table schema `table_fields` declares, as after Iceberg promotes `int` +/// to `long` or widens a decimal. DataFusion's reader casts such a column, +/// the native reader decodes the stored type, so the scan rejects the file. +/// String and binary types compare by family, because Iceberg, Parquet +/// writers and DataFusion choose different widths and view types for them; +/// a dictionary compares by its value type. Columns absent from the file are +/// the binding check's concern. +pub(crate) fn column_with_stored_type_mismatch( + schema: &IcebergSchema, + projected_field_ids: &[i32], + table_fields: &Fields, + file_fields: &Fields, +) -> Option { + projected_field_ids.iter().find_map(|field_id| { + let name = &schema.as_struct().field_by_id(*field_id)?.name; + let table_field = table_fields.iter().find(|field| field.name() == name)?; + let file_field = file_fields.iter().find(|field| field.name() == name)?; + (!same_stored_type(table_field.data_type(), file_field.data_type())).then(|| name.clone()) + }) +} + +fn same_stored_type(table: &DataType, file: &DataType) -> bool { + match (table, file) { + (_, DataType::Dictionary(_, value)) => same_stored_type(table, value), + ( + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View, + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View, + ) + | ( + DataType::Binary | DataType::LargeBinary | DataType::BinaryView, + DataType::Binary | DataType::LargeBinary | DataType::BinaryView, + ) => true, + ( + DataType::List(table) + | DataType::LargeList(table) + | DataType::ListView(table) + | DataType::LargeListView(table), + DataType::List(file) | DataType::LargeList(file) | DataType::ListView(file) | DataType::LargeListView(file), + ) => same_stored_type(table.data_type(), file.data_type()), + (DataType::FixedSizeList(table, table_len), DataType::FixedSizeList(file, file_len)) => { + table_len == file_len && same_stored_type(table.data_type(), file.data_type()) + }, + (DataType::Map(table, _), DataType::Map(file, _)) => same_stored_type(table.data_type(), file.data_type()), + (DataType::Struct(table), DataType::Struct(file)) => table.iter().all(|table_child| { + file.iter() + .find(|file_child| file_child.name() == table_child.name()) + .is_none_or(|file_child| same_stored_type(table_child.data_type(), file_child.data_type())) + }), + // Iceberg names UTC `+00:00`; writers also record `UTC`. + (DataType::Timestamp(table_unit, table_zone), DataType::Timestamp(file_unit, file_zone)) => { + table_unit == file_unit && table_zone.is_some() == file_zone.is_some() + }, + (table, file) => table == file, + } +} diff --git a/crates/algeon-iceberg/src/catalog.rs b/crates/algeon-iceberg/src/catalog.rs new file mode 100644 index 000000000..b9db63d07 --- /dev/null +++ b/crates/algeon-iceberg/src/catalog.rs @@ -0,0 +1,210 @@ +//! Iceberg REST and Glue catalogs as DataFusion catalogs. + +use std::{collections::HashMap, fmt, sync::Arc}; + +use algeon_datafusion::{ + error::{ErrorCode, Result}, + native_exec::S3SourceStorageProfile, + server::{ + ServerIcebergCatalogConfig, ServerIcebergCatalogKind, ServerIcebergGlueCatalogConfig, ServerIcebergRestAuth, + ServerIcebergRestCatalogConfig, + }, +}; +use async_trait::async_trait; +use datafusion::{ + catalog::{CatalogProvider, SchemaProvider, TableProvider}, + common::{DataFusionError, Result as DataFusionResult}, +}; +use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, TableIdent}; +use iceberg_catalog_glue::{ + GLUE_CATALOG_PROP_CATALOG_ID, GLUE_CATALOG_PROP_URI, GLUE_CATALOG_PROP_WAREHOUSE, GlueCatalogBuilder, +}; +use iceberg_catalog_rest::{ + REST_CATALOG_PROP_AUTH_TYPE, REST_CATALOG_PROP_URI, REST_CATALOG_PROP_WAREHOUSE, RestCatalogBuilder, +}; +use iceberg_storage_opendal::OpenDalStorageFactory; + +use crate::{error::iceberg_error, storage::file_io_properties, table::IcebergTableProvider}; + +const CONNECT_OPERATION: &str = "algeon_iceberg.catalog.connect"; +const LIST_OPERATION: &str = "algeon_iceberg.catalog.list"; + +/// Connects the Iceberg catalog `options` describes, an +/// `[[iceberg.catalogs]]` entry of the server configuration. +/// +/// Metadata files are read with FileIO properties derived from `storage`. +/// The Glue client is built on the caller's Tokio runtime, which must be +/// current. +pub async fn connect_catalog( + options: &ServerIcebergCatalogConfig, + storage: Option<&S3SourceStorageProfile>, +) -> Result> { + let mut properties = storage.map(file_io_properties).unwrap_or_default(); + let factory = Arc::new(OpenDalStorageFactory::S3 { customized_credential_load: None }); + let connect_error = |source| { + iceberg_error(CONNECT_OPERATION, "failed to connect the Iceberg catalog", source) + .with_fact("catalog", &options.name) + }; + match &options.kind { + ServerIcebergCatalogKind::Rest(rest) => { + properties.extend(rest_properties(rest)); + let catalog = RestCatalogBuilder::default() + .with_storage_factory(factory) + .load(options.name.clone(), properties) + .await + .map_err(connect_error)?; + Ok(Arc::new(catalog)) + }, + ServerIcebergCatalogKind::Glue(glue) => { + properties.extend(glue_properties(glue)); + let catalog = GlueCatalogBuilder::default() + .with_storage_factory(factory) + .load(options.name.clone(), properties) + .await + .map_err(connect_error)?; + Ok(Arc::new(catalog)) + }, + } +} + +/// REST catalog properties. Vended credentials are never requested: data +/// files are read with the server's storage, not the catalog's. +fn rest_properties(rest: &ServerIcebergRestCatalogConfig) -> HashMap { + let mut properties = HashMap::from([(REST_CATALOG_PROP_URI.to_string(), rest.uri.clone())]); + let mut set = |key: &str, value: &Option| { + if let Some(value) = value { + properties.insert(key.to_string(), value.clone()); + } + }; + set(REST_CATALOG_PROP_WAREHOUSE, &rest.warehouse); + set("prefix", &rest.prefix); + match &rest.auth { + ServerIcebergRestAuth::None => { + properties.insert(REST_CATALOG_PROP_AUTH_TYPE.to_string(), "none".to_string()); + }, + ServerIcebergRestAuth::OAuth2 { credential, server_uri, scope } => { + properties.insert(REST_CATALOG_PROP_AUTH_TYPE.to_string(), "oauth2".to_string()); + properties.insert("credential".to_string(), credential.expose().to_string()); + if let Some(server_uri) = server_uri { + properties.insert("oauth2-server-uri".to_string(), server_uri.clone()); + } + if let Some(scope) = scope { + properties.insert("scope".to_string(), scope.clone()); + } + }, + ServerIcebergRestAuth::Token(token) => { + properties.insert(REST_CATALOG_PROP_AUTH_TYPE.to_string(), "oauth2".to_string()); + properties.insert("token".to_string(), token.expose().to_string()); + }, + } + properties +} + +fn glue_properties(glue: &ServerIcebergGlueCatalogConfig) -> HashMap { + let mut properties = HashMap::from([(GLUE_CATALOG_PROP_WAREHOUSE.to_string(), glue.warehouse.clone())]); + for (key, value) in [ + (GLUE_CATALOG_PROP_CATALOG_ID, &glue.catalog_id), + ("region_name", &glue.region), + (GLUE_CATALOG_PROP_URI, &glue.endpoint), + ] { + if let Some(value) = value { + properties.insert(key.to_string(), value.clone()); + } + } + properties +} + +/// A DataFusion catalog over one Iceberg catalog, with one schema per +/// top-level namespace. +/// +/// Namespaces and table names are listed once, when the provider is built; +/// nested namespaces are not listed. [`SchemaProvider::table`] loads the +/// table from the catalog on every call, so a table created later is +/// queryable by name although it is missing from the listing. +#[derive(Debug)] +pub struct IcebergCatalogProvider { + schemas: HashMap>, +} + +impl IcebergCatalogProvider { + /// Connects the catalog `options` describes and lists its namespaces + /// and tables. + pub async fn connect( + options: &ServerIcebergCatalogConfig, + storage: Option<&S3SourceStorageProfile>, + ) -> Result { + let catalog = connect_catalog(options, storage).await?; + Self::try_new(catalog, storage.cloned()).await.map_err(|error| error.with_fact("catalog", &options.name)) + } + + /// Lists the namespaces and tables of a connected `catalog`. `storage` + /// is the S3 storage the session reads data files with. + pub async fn try_new(catalog: Arc, storage: Option) -> Result { + let list_error = |source| iceberg_error(LIST_OPERATION, "failed to list the Iceberg catalog", source); + let namespaces = catalog.list_namespaces(None).await.map_err(list_error)?; + let mut schemas = HashMap::new(); + for namespace in namespaces { + let [name] = namespace.as_ref().as_slice() else { + continue; + }; + let tables = catalog.list_tables(&namespace).await.map_err(list_error)?; + let schema = IcebergSchemaProvider { + catalog: Arc::clone(&catalog), + namespace: namespace.clone(), + table_names: tables.into_iter().map(|table| table.name().to_string()).collect(), + storage: storage.clone(), + }; + schemas.insert(name.clone(), Arc::new(schema)); + } + Ok(Self { schemas }) + } +} + +impl CatalogProvider for IcebergCatalogProvider { + fn schema_names(&self) -> Vec { + let mut names = self.schemas.keys().cloned().collect::>(); + names.sort(); + names + } + + fn schema(&self, name: &str) -> Option> { + self.schemas.get(name).map(|schema| Arc::clone(schema) as Arc) + } +} + +/// One Iceberg namespace as a DataFusion schema. +pub struct IcebergSchemaProvider { + catalog: Arc, + namespace: NamespaceIdent, + table_names: Vec, + storage: Option, +} + +impl fmt::Debug for IcebergSchemaProvider { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("IcebergSchemaProvider") + .field("namespace", &self.namespace) + .field("table_names", &self.table_names) + .finish_non_exhaustive() + } +} + +#[async_trait] +impl SchemaProvider for IcebergSchemaProvider { + fn table_names(&self) -> Vec { + self.table_names.clone() + } + + async fn table(&self, name: &str) -> DataFusionResult>> { + let ident = TableIdent::new(self.namespace.clone(), name.to_string()); + match IcebergTableProvider::try_new(Arc::clone(&self.catalog), ident, self.storage.clone()).await { + Ok(provider) => Ok(Some(Arc::new(provider))), + Err(error) if error.code() == ErrorCode::ObjectNotFound => Ok(None), + Err(error) => Err(DataFusionError::from(error)), + } + } + + fn table_exist(&self, name: &str) -> bool { + self.table_names.iter().any(|table| table == name) + } +} diff --git a/crates/algeon-iceberg/src/error.rs b/crates/algeon-iceberg/src/error.rs new file mode 100644 index 000000000..6c9140823 --- /dev/null +++ b/crates/algeon-iceberg/src/error.rs @@ -0,0 +1,173 @@ +//! Errors for Iceberg scans and catalog calls. +//! +//! A snapshot the scan cannot read correctly fails with +//! [`ErrorCode::UnsupportedTableFeature`] and a log-safe `reason` fact. +//! Catalog and metadata-storage failures map onto the adapter's source codes. + +use std::error::Error as StdError; + +use algeon_datafusion::error::{Error, ErrorCode}; +use iceberg::{Error as IcebergError, ErrorKind as IcebergErrorKind, TableIdent}; + +pub(crate) const SCAN_OPERATION: &str = "algeon_iceberg.scan"; + +/// Maximum source-chain layers searched for typed causes and AWS markers. +const MAX_ERROR_CHAIN_DEPTH: usize = 32; + +/// A table-format feature the scan does not read. The string form is the +/// `reason` fact of the [`ErrorCode::UnsupportedTableFeature`] error. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum UnsupportedReason { + DeleteFiles, + FieldIdMismatch, + MissingFieldIds, + PartitionColumnAbsent, + RequiredColumnAbsent, + EncryptedFile, + NonParquetFile, + MultipleBuckets, + UnsupportedLocationScheme, + StoragePropertiesMismatch, + ColumnTypeMismatch, +} + +impl UnsupportedReason { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::DeleteFiles => "iceberg_delete_files", + Self::FieldIdMismatch => "iceberg_field_id_mismatch", + Self::MissingFieldIds => "iceberg_missing_field_ids", + Self::PartitionColumnAbsent => "iceberg_partition_column_absent", + Self::RequiredColumnAbsent => "iceberg_required_column_absent", + Self::EncryptedFile => "iceberg_encrypted_file", + Self::NonParquetFile => "iceberg_non_parquet_file", + Self::MultipleBuckets => "iceberg_multiple_buckets", + Self::UnsupportedLocationScheme => "iceberg_unsupported_location_scheme", + Self::StoragePropertiesMismatch => "iceberg_storage_properties_mismatch", + Self::ColumnTypeMismatch => "iceberg_column_type_mismatch", + } + } +} + +/// The rejection of a snapshot that uses `reason`. The table and snapshot +/// id are raw facts because they name caller data. +pub(crate) fn unsupported( + reason: UnsupportedReason, + table: &TableIdent, + snapshot_id: Option, + message: impl Into, +) -> Error { + let error = Error::new(ErrorCode::UnsupportedTableFeature, SCAN_OPERATION, message) + .with_log_safe_fact("reason", reason.as_str()) + .with_fact("table", table); + match snapshot_id { + Some(snapshot_id) => error.with_fact("snapshot_id", snapshot_id), + None => error, + } +} + +/// An adapter error for a failed catalog or Iceberg metadata call. +pub(crate) fn iceberg_error(operation: &'static str, message: impl Into, source: IcebergError) -> Error { + let code = iceberg_error_code(&source); + let mut error = Error::new(code, operation, message) + .with_fact("iceberg_error_kind", source.kind().into_static()) + .with_fact("iceberg_retryable", source.retryable()); + if let Some(marker) = aws_marker(&chain_text(&source)) { + error = error.with_fact("aws_error_detail", marker.detail); + } + if let Some(opendal) = find_source::(&source) { + error = error.with_fact("opendal_error_kind", opendal.kind().into_static()); + } + error.with_source(source) +} + +/// The adapter code of an Iceberg error. The typed kind decides first; +/// credential and access markers in the rendered chain decide next, because +/// the Glue catalog flattens AWS SDK errors into text and the REST catalog +/// renders the HTTP status into the error context; a typed OpenDAL cause +/// decides last. +pub(crate) fn iceberg_error_code(source: &IcebergError) -> ErrorCode { + if matches!(source.kind(), IcebergErrorKind::TableNotFound | IcebergErrorKind::NamespaceNotFound) { + return ErrorCode::ObjectNotFound; + } + if let Some(marker) = aws_marker(&chain_text(source)) { + return marker.code; + } + if let Some(opendal) = find_source::(source) { + return opendal_error_code(opendal); + } + if source.retryable() { ErrorCode::SourceTemporaryFailure } else { ErrorCode::SourceDependencyFailure } +} + +fn opendal_error_code(source: &opendal::Error) -> ErrorCode { + match source.kind() { + opendal::ErrorKind::NotFound => ErrorCode::SourceObjectNotFound, + opendal::ErrorKind::PermissionDenied => ErrorCode::SourceAccessDenied, + opendal::ErrorKind::RateLimited => ErrorCode::SourceTemporaryFailure, + _ if source.is_temporary() => ErrorCode::SourceTemporaryFailure, + _ => ErrorCode::SourceDependencyFailure, + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct AwsMarker { + code: ErrorCode, + detail: &'static str, +} + +/// Credential and access markers rendered by the AWS SDK, S3 and HTTP +/// clients, in the order the first match wins. +const AWS_MARKERS: &[(&[&str], ErrorCode, &str)] = &[ + (&["expiredtokenexception", "expiredtoken"], ErrorCode::SourceUnauthenticated, "expired_token"), + (&["invalidaccesskeyid"], ErrorCode::SourceUnauthenticated, "invalid_access_key"), + (&["unrecognizedclientexception"], ErrorCode::SourceUnauthenticated, "invalid_client"), + (&["signaturedoesnotmatch"], ErrorCode::SourceAccessDenied, "signature_mismatch"), + (&["accessdeniedexception", "accessdenied"], ErrorCode::SourceAccessDenied, "access_denied"), + (&["entitynotfoundexception"], ErrorCode::ObjectNotFound, "not_found"), + ( + &["401 unauthorized", "status code: 401", "status: 401", "http 401"], + ErrorCode::SourceUnauthenticated, + "http_401", + ), + (&["403 forbidden", "status code: 403", "status: 403", "http 403"], ErrorCode::SourceAccessDenied, "http_403"), + ( + &["no credentials", "missing credentials", "could not load credentials"], + ErrorCode::SourceUnauthenticated, + "missing_credentials", + ), +]; + +fn aws_marker(text: &str) -> Option { + let lower = text.to_ascii_lowercase(); + AWS_MARKERS + .iter() + .find(|(needles, _, _)| needles.iter().any(|needle| lower.contains(needle))) + .map(|&(_, code, detail)| AwsMarker { code, detail }) +} + +fn chain_text(source: &(dyn StdError + 'static)) -> String { + let mut text = source.to_string(); + let mut current = source.source(); + for _ in 0..MAX_ERROR_CHAIN_DEPTH { + let Some(layer) = current else { break }; + text.push_str(": "); + text.push_str(&layer.to_string()); + current = layer.source(); + } + text +} + +fn find_source<'a, E: StdError + 'static>(source: &'a (dyn StdError + 'static)) -> Option<&'a E> { + let mut current = Some(source); + for _ in 0..MAX_ERROR_CHAIN_DEPTH { + let layer = current?; + if let Some(found) = layer.downcast_ref::() { + return Some(found); + } + current = layer.source(); + } + None +} + +#[cfg(test)] +mod tests; diff --git a/crates/algeon-iceberg/src/error/tests.rs b/crates/algeon-iceberg/src/error/tests.rs new file mode 100644 index 000000000..e761d375b --- /dev/null +++ b/crates/algeon-iceberg/src/error/tests.rs @@ -0,0 +1,37 @@ +use algeon_datafusion::error::ErrorCode; +use iceberg::{Error as IcebergError, ErrorKind as IcebergErrorKind}; + +use super::iceberg_error_code; + +#[test] +fn missing_tables_and_namespaces_are_object_not_found() { + for kind in [IcebergErrorKind::TableNotFound, IcebergErrorKind::NamespaceNotFound] { + assert_eq!(iceberg_error_code(&IcebergError::new(kind, "missing")), ErrorCode::ObjectNotFound); + } +} + +#[test] +fn rest_status_and_aws_markers_classify_credentials_and_access() { + let rest = |status: &str| { + IcebergError::new(IcebergErrorKind::Unexpected, "Received response with unexpected status code") + .with_context("status", status) + }; + assert_eq!(iceberg_error_code(&rest("401 Unauthorized")), ErrorCode::SourceUnauthenticated); + assert_eq!(iceberg_error_code(&rest("403 Forbidden")), ErrorCode::SourceAccessDenied); + let glue = IcebergError::new(IcebergErrorKind::Unexpected, "aws sdk error: ServiceError { ExpiredTokenException }"); + assert_eq!(iceberg_error_code(&glue), ErrorCode::SourceUnauthenticated); +} + +#[test] +fn typed_storage_causes_and_retryability_decide_the_rest() { + let missing_object = IcebergError::new(IcebergErrorKind::Unexpected, "read failed") + .with_source(opendal::Error::new(opendal::ErrorKind::NotFound, "no such key")); + assert_eq!(iceberg_error_code(&missing_object), ErrorCode::SourceObjectNotFound); + let throttled = IcebergError::new(IcebergErrorKind::Unexpected, "read failed") + .with_source(opendal::Error::new(opendal::ErrorKind::RateLimited, "slow down")); + assert_eq!(iceberg_error_code(&throttled), ErrorCode::SourceTemporaryFailure); + let retryable = IcebergError::new(IcebergErrorKind::Unexpected, "connection reset").with_retryable(true); + assert_eq!(iceberg_error_code(&retryable), ErrorCode::SourceTemporaryFailure); + let invalid = IcebergError::new(IcebergErrorKind::DataInvalid, "bad manifest"); + assert_eq!(iceberg_error_code(&invalid), ErrorCode::SourceDependencyFailure); +} diff --git a/crates/algeon-iceberg/src/lib.rs b/crates/algeon-iceberg/src/lib.rs new file mode 100644 index 000000000..210d1d1ef --- /dev/null +++ b/crates/algeon-iceberg/src/lib.rs @@ -0,0 +1,58 @@ +//! Iceberg tables as native Algeon scans. +//! +//! [`register_iceberg_catalogs`] registers one DataFusion catalog per +//! configured Iceberg REST or Glue catalog. Each namespace is a schema and +//! each table an [`IcebergTableProvider`], which plans the current snapshot +//! with Iceberg's manifest pruning and scans its data files as Parquet. The +//! data files are read with the session's `[s3]` storage, never with +//! credentials a catalog returns. +//! +//! Snapshots the Parquet scan would read incorrectly (delete files, columns +//! bound by field id to other names, partition values kept only in +//! metadata) fail with `unsupported_table_feature`; there is no other read +//! path. + +mod binding; +mod catalog; +mod error; +mod predicate; +mod schema; +mod storage; +mod table; + +use std::sync::Arc; + +use algeon_datafusion::{ + error::{Error, ErrorCode, Result}, + server::ServerConfig, +}; +use datafusion::prelude::SessionContext; + +pub use crate::{ + catalog::{IcebergCatalogProvider, IcebergSchemaProvider, connect_catalog}, + table::IcebergTableProvider, +}; + +/// Connects every `[[iceberg.catalogs]]` entry of `config` and registers it +/// on `ctx` under its name. Data files are read with the `[s3]` storage of +/// `config`. +/// +/// Catalogs connect in file order on the caller's Tokio runtime; the first +/// failure is returned and leaves the catalogs registered before it in +/// place. A name the session already holds, such as its default catalog, +/// fails with `invalid_configuration` instead of replacing that catalog. +pub async fn register_iceberg_catalogs(ctx: &SessionContext, config: &ServerConfig) -> Result<()> { + for options in &config.iceberg_catalogs { + if ctx.catalog(&options.name).is_some() { + return Err(Error::new( + ErrorCode::InvalidConfiguration, + "algeon_iceberg.register_catalogs", + format!("iceberg.catalogs name `{}` is already a catalog of the server session", options.name), + ) + .with_fact("config_field", "iceberg.catalogs.name")); + } + let provider = IcebergCatalogProvider::connect(options, config.s3_storage()).await?; + ctx.register_catalog(options.name.clone(), Arc::new(provider)); + } + Ok(()) +} diff --git a/crates/algeon-iceberg/src/predicate.rs b/crates/algeon-iceberg/src/predicate.rs new file mode 100644 index 000000000..21301a20d --- /dev/null +++ b/crates/algeon-iceberg/src/predicate.rs @@ -0,0 +1,116 @@ +//! DataFusion filters as Iceberg predicates for manifest pruning. +//! +//! Every filter stays in the plan above the scan (the provider reports +//! `Inexact` pushdown), so a filter that cannot be converted only loses +//! pruning. Within an `AND`, a convertible side still prunes on its own; +//! under `NOT` and `OR` both sides must convert, since dropping one side +//! there would widen or narrow the result. + +use std::{ops::Not, sync::Arc}; + +use datafusion::{ + logical_expr::{Expr, Operator}, + scalar::ScalarValue, +}; +use iceberg::{ + expr::{Bind, Predicate, Reference}, + spec::{Datum, SchemaRef}, +}; + +/// The conjunction of the filters that convert and bind to `schema`. +/// +/// A filter whose literal does not bind to the column type is dropped, so a +/// type the converter maps imprecisely never fails the scan. +pub(crate) fn filters_to_predicate(filters: &[Expr], schema: &SchemaRef) -> Option { + filters + .iter() + .filter_map(|filter| filter_to_predicate(filter, true)) + .filter(|predicate| predicate.bind(Arc::clone(schema), true).is_ok()) + .reduce(Predicate::and) +} + +fn filter_to_predicate(expr: &Expr, allow_partial_and: bool) -> Option { + match expr { + Expr::BinaryExpr(binary) => match binary.op { + Operator::And => { + let left = filter_to_predicate(&binary.left, allow_partial_and); + let right = filter_to_predicate(&binary.right, allow_partial_and); + match (left, right) { + (Some(left), Some(right)) => Some(left.and(right)), + (Some(side), None) | (None, Some(side)) if allow_partial_and => Some(side), + _ => None, + } + }, + Operator::Or => Some( + filter_to_predicate(&binary.left, allow_partial_and)? + .or(filter_to_predicate(&binary.right, allow_partial_and)?), + ), + Operator::Eq | Operator::NotEq | Operator::Lt | Operator::LtEq | Operator::Gt | Operator::GtEq => { + comparison_to_predicate(&binary.left, binary.op, &binary.right) + .or_else(|| comparison_to_predicate(&binary.right, reverse_operator(binary.op)?, &binary.left)) + }, + _ => None, + }, + Expr::Not(expr) => filter_to_predicate(expr, false).map(Not::not), + Expr::IsNull(expr) => column_name(expr).map(|column| Reference::new(column).is_null()), + Expr::IsNotNull(expr) => column_name(expr).map(|column| Reference::new(column).is_not_null()), + Expr::Column(column) => Some(Reference::new(column.name()).equal_to(Datum::bool(true))), + _ => None, + } +} + +fn comparison_to_predicate(left: &Expr, operator: Operator, right: &Expr) -> Option { + let reference = Reference::new(column_name(left)?); + let Expr::Literal(value, _) = right else { + return None; + }; + let datum = scalar_to_datum(value)?; + Some(match operator { + Operator::Eq => reference.equal_to(datum), + Operator::NotEq => reference.not_equal_to(datum), + Operator::Lt => reference.less_than(datum), + Operator::LtEq => reference.less_than_or_equal_to(datum), + Operator::Gt => reference.greater_than(datum), + Operator::GtEq => reference.greater_than_or_equal_to(datum), + _ => return None, + }) +} + +fn column_name(expr: &Expr) -> Option<&str> { + match expr { + Expr::Column(column) => Some(column.name()), + _ => None, + } +} + +fn scalar_to_datum(value: &ScalarValue) -> Option { + match value { + ScalarValue::Boolean(Some(value)) => Some(Datum::bool(*value)), + ScalarValue::Int8(Some(value)) => Some(Datum::int(i32::from(*value))), + ScalarValue::Int16(Some(value)) => Some(Datum::int(i32::from(*value))), + ScalarValue::Int32(Some(value)) => Some(Datum::int(*value)), + ScalarValue::Int64(Some(value)) => Some(Datum::long(*value)), + ScalarValue::Float32(Some(value)) => Some(Datum::float(*value)), + ScalarValue::Float64(Some(value)) => Some(Datum::double(*value)), + ScalarValue::Utf8(Some(value)) | ScalarValue::LargeUtf8(Some(value)) | ScalarValue::Utf8View(Some(value)) => { + Some(Datum::string(value)) + }, + ScalarValue::Date32(Some(value)) => Some(Datum::date(*value)), + _ => None, + } +} + +fn reverse_operator(operator: Operator) -> Option { + match operator { + Operator::Eq => Some(Operator::Eq), + Operator::NotEq => Some(Operator::NotEq), + Operator::Lt => Some(Operator::Gt), + Operator::LtEq => Some(Operator::GtEq), + Operator::Gt => Some(Operator::Lt), + Operator::GtEq => Some(Operator::LtEq), + _ => None, + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/algeon-iceberg/src/predicate/tests.rs b/crates/algeon-iceberg/src/predicate/tests.rs new file mode 100644 index 000000000..402e3a3fb --- /dev/null +++ b/crates/algeon-iceberg/src/predicate/tests.rs @@ -0,0 +1,52 @@ +use std::sync::Arc; + +use datafusion::{ + logical_expr::{col, lit}, + scalar::ScalarValue, +}; +use iceberg::{ + expr::Reference, + spec::{Datum, NestedField, PrimitiveType, Schema, SchemaRef, Type}, +}; + +use super::filters_to_predicate; + +fn schema() -> SchemaRef { + Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(), + ) +} + +#[test] +fn a_literal_on_the_left_reverses_the_comparison() { + let predicate = filters_to_predicate(&[lit(5_i64).lt(col("id"))], &schema()).unwrap(); + assert_eq!(predicate, Reference::new("id").greater_than(Datum::long(5))); +} + +#[test] +fn an_and_keeps_its_convertible_side_but_an_or_or_not_needs_both() { + let unconvertible = col("name").like(lit("a%")); + let id_filter = col("id").gt_eq(lit(10_i64)); + let and = filters_to_predicate(&[id_filter.clone().and(unconvertible.clone())], &schema()).unwrap(); + assert_eq!(and, Reference::new("id").greater_than_or_equal_to(Datum::long(10))); + assert!(filters_to_predicate(&[id_filter.clone().or(unconvertible.clone())], &schema()).is_none()); + assert!(filters_to_predicate(&[!(id_filter.and(unconvertible))], &schema()).is_none()); +} + +#[test] +fn a_filter_whose_literal_does_not_bind_is_dropped_and_the_rest_are_conjoined() { + let unbindable = col("id").eq(lit(ScalarValue::Utf8(Some("not a number".to_string())))); + let name = col("name").eq(lit(ScalarValue::Utf8View(Some("ann".to_string())))); + let id = col("id").is_not_null(); + let predicate = filters_to_predicate(&[unbindable, name, id], &schema()).unwrap(); + assert_eq!( + predicate, + Reference::new("name").equal_to(Datum::string("ann")).and(Reference::new("id").is_not_null()) + ); +} diff --git a/crates/algeon-iceberg/src/schema.rs b/crates/algeon-iceberg/src/schema.rs new file mode 100644 index 000000000..8da2921cc --- /dev/null +++ b/crates/algeon-iceberg/src/schema.rs @@ -0,0 +1,31 @@ +//! The Arrow schema an Iceberg table exposes to DataFusion. + +use std::sync::Arc; + +use algeon_datafusion::error::{Error, ErrorCode, Result}; +use datafusion::{arrow::datatypes::SchemaRef, datasource::physical_plan::parquet::transform_schema_to_view}; +use iceberg::{ + Error as IcebergError, + arrow::{schema_to_arrow_schema, strip_metadata_from_schema}, + spec::Schema as IcebergSchema, +}; + +use crate::error::SCAN_OPERATION; + +/// The table schema DataFusion plans against. +/// +/// Field-id metadata is removed: DataFusion projections and aliases drop +/// field metadata, and its physical-versus-logical schema check treats the +/// difference as a mismatch. Top-level strings and binaries use the view +/// types, as a DataFusion `ListingTable` over the same files does under the +/// default `schema_force_view_types`, so native lowering and the CPU +/// operators above the scan see the types they see for a listing table. +pub(crate) fn table_arrow_schema(schema: &IcebergSchema) -> Result { + let conversion_error = |error: IcebergError| { + Error::new(ErrorCode::UnsupportedType, SCAN_OPERATION, "the Iceberg schema has no Arrow equivalent") + .with_source(error) + }; + let arrow = schema_to_arrow_schema(schema).map_err(conversion_error)?; + let arrow = strip_metadata_from_schema(&arrow).map_err(conversion_error)?; + Ok(Arc::new(transform_schema_to_view(&arrow))) +} diff --git a/crates/algeon-iceberg/src/storage.rs b/crates/algeon-iceberg/src/storage.rs new file mode 100644 index 000000000..e1dce1ae1 --- /dev/null +++ b/crates/algeon-iceberg/src/storage.rs @@ -0,0 +1,79 @@ +//! Iceberg FileIO properties derived from the server's `[s3]` storage. +//! +//! Iceberg metadata (metadata.json, manifest lists, manifests) is read +//! through the catalog's FileIO, and data files through the DataFusion object +//! store and the native reader, which both use the `[s3]` profile. The FileIO +//! is configured from the same profile, and a loaded table whose FileIO +//! names another endpoint, region or addressing style is rejected, so the +//! two read paths address the same storage. + +use std::collections::HashMap; + +use algeon_datafusion::native_exec::S3SourceStorageProfile; +use iceberg::io::{ + CLIENT_REGION, S3_ACCESS_KEY_ID, S3_ALLOW_ANONYMOUS, S3_ENDPOINT, S3_PATH_STYLE_ACCESS, S3_REGION, + S3_SECRET_ACCESS_KEY, S3_SESSION_TOKEN, +}; +use query_runtime::runtime::S3SourceStorageCredential; + +/// FileIO properties that make Iceberg read metadata with `storage`. +pub(crate) fn file_io_properties(storage: &S3SourceStorageProfile) -> HashMap { + let mut properties = HashMap::new(); + if let Some(endpoint) = &storage.endpoint { + properties.insert(S3_ENDPOINT.to_string(), endpoint.clone()); + } + if let Some(region) = &storage.region { + properties.insert(S3_REGION.to_string(), region.clone()); + } + if let Some(virtual_hosted) = storage.virtual_hosted_style_request { + properties.insert(S3_PATH_STYLE_ACCESS.to_string(), (!virtual_hosted).to_string()); + } + match &storage.credential { + S3SourceStorageCredential::DefaultChain => {}, + S3SourceStorageCredential::Unsigned => { + properties.insert(S3_ALLOW_ANONYMOUS.to_string(), "true".to_string()); + }, + S3SourceStorageCredential::Static { access_key_id, secret_access_key, session_token } => { + properties.insert(S3_ACCESS_KEY_ID.to_string(), access_key_id.clone()); + properties.insert(S3_SECRET_ACCESS_KEY.to_string(), secret_access_key.clone()); + if let Some(session_token) = session_token { + properties.insert(S3_SESSION_TOKEN.to_string(), session_token.clone()); + } + }, + } + properties +} + +/// The first FileIO property of a loaded table that addresses storage other +/// than `storage`, or `None` when they agree. +/// +/// The endpoint must match exactly, since a different endpoint is a +/// different store. Region and addressing style are compared when both +/// sides set them; an unset side takes the client default. +pub(crate) fn mismatched_property( + properties: &HashMap, + storage: &S3SourceStorageProfile, +) -> Option<&'static str> { + let normalized = |endpoint: &str| endpoint.trim().trim_end_matches('/').to_string(); + let file_io_endpoint = properties.get(S3_ENDPOINT).map(|endpoint| normalized(endpoint)); + if file_io_endpoint != storage.endpoint.as_deref().map(normalized) { + return Some(S3_ENDPOINT); + } + let region_key = if properties.contains_key(CLIENT_REGION) { CLIENT_REGION } else { S3_REGION }; + if let (Some(file_io_region), Some(region)) = (properties.get(region_key), &storage.region) + && file_io_region.trim() != region.trim() + { + return Some(region_key); + } + let file_io_path_style = + properties.get(S3_PATH_STYLE_ACCESS).map(|value| value.trim().eq_ignore_ascii_case("true")); + if let (Some(file_io_path_style), Some(virtual_hosted)) = (file_io_path_style, storage.virtual_hosted_style_request) + && file_io_path_style == virtual_hosted + { + return Some(S3_PATH_STYLE_ACCESS); + } + None +} + +#[cfg(test)] +mod tests; diff --git a/crates/algeon-iceberg/src/storage/tests.rs b/crates/algeon-iceberg/src/storage/tests.rs new file mode 100644 index 000000000..0dbc95602 --- /dev/null +++ b/crates/algeon-iceberg/src/storage/tests.rs @@ -0,0 +1,56 @@ +use std::collections::HashMap; + +use algeon_datafusion::native_exec::S3SourceStorageProfile; + +use super::{file_io_properties, mismatched_property}; + +fn properties(pairs: &[(&str, &str)]) -> HashMap { + pairs.iter().map(|(key, value)| (key.to_string(), value.to_string())).collect() +} + +#[test] +fn the_storage_profile_becomes_matching_file_io_properties() { + let storage = S3SourceStorageProfile::default_chain() + .with_endpoint("http://store:9000") + .with_region("eu-west-2") + .with_path_style_access(true) + .with_unsigned_requests(); + let derived = file_io_properties(&storage); + assert_eq!( + derived, + properties(&[ + ("s3.endpoint", "http://store:9000"), + ("s3.region", "eu-west-2"), + ("s3.path-style-access", "true"), + ("s3.allow-anonymous", "true"), + ]) + ); + assert_eq!(mismatched_property(&derived, &storage), None); +} + +#[test] +fn the_endpoint_must_match_while_region_and_addressing_are_compared_when_both_are_set() { + let storage = S3SourceStorageProfile::default_chain().with_region("eu-west-2"); + assert_eq!( + mismatched_property(&properties(&[("s3.endpoint", "http://other:9000")]), &storage), + Some("s3.endpoint") + ); + assert_eq!(mismatched_property(&properties(&[("client.region", "us-east-1")]), &storage), Some("client.region")); + assert_eq!( + mismatched_property(&properties(&[("client.region", "eu-west-2"), ("s3.region", "us-east-1")]), &storage), + None + ); + assert_eq!(mismatched_property(&properties(&[("s3.path-style-access", "true")]), &storage), None); + + let path_style = + S3SourceStorageProfile::default_chain().with_endpoint("http://store:9000/").with_path_style_access(true); + assert_eq!(mismatched_property(&properties(&[("s3.endpoint", "http://store:9000")]), &path_style), None); + assert_eq!( + mismatched_property( + &properties(&[("s3.endpoint", "http://store:9000"), ("s3.path-style-access", "false")]), + &path_style + ), + Some("s3.path-style-access") + ); + assert_eq!(mismatched_property(&HashMap::new(), &path_style), Some("s3.endpoint")); +} diff --git a/crates/algeon-iceberg/src/table.rs b/crates/algeon-iceberg/src/table.rs new file mode 100644 index 000000000..d2f4438c1 --- /dev/null +++ b/crates/algeon-iceberg/src/table.rs @@ -0,0 +1,413 @@ +//! [`IcebergTableProvider`]: an Iceberg table planned as a Parquet scan. + +use std::{fmt, sync::Arc}; + +use algeon_datafusion::{ + error::{Error, ErrorCode, Result}, + native_exec::S3SourceStorageProfile, +}; +use async_trait::async_trait; +use datafusion::{ + arrow::datatypes::{Schema, SchemaRef}, + catalog::{Session, TableProvider}, + common::{DataFusionError, Result as DataFusionResult, Statistics, stats::Precision}, + datasource::{ + TableType, + file_format::{FileFormat, parquet::ParquetFormat}, + listing::PartitionedFile, + physical_plan::{FileGroup, FileScanConfigBuilder, parquet::metadata::DFParquetMetadata}, + table_schema::TableSchema, + }, + execution::{cache::cache_manager::FileMetadataCache, object_store::ObjectStoreUrl}, + logical_expr::{Expr, TableProviderFilterPushDown}, + object_store::{ObjectMeta, ObjectStore, ObjectStoreExt, path::Path as ObjectPath}, + parquet::arrow::parquet_to_arrow_schema, + physical_plan::{ExecutionPlan, empty::EmptyExec}, +}; +use futures::{StreamExt, TryStreamExt, stream}; +use iceberg::{ + Catalog, TableIdent, + scan::FileScanTask, + spec::{DataFileFormat, NameMapping, SchemaRef as IcebergSchemaRef}, + table::Table, +}; + +use crate::{ + binding::{FileBinding, column_with_stored_type_mismatch, identity_partition_sources}, + error::{SCAN_OPERATION, UnsupportedReason, iceberg_error, unsupported}, + predicate::filters_to_predicate, + schema::table_arrow_schema, + storage::mismatched_property, +}; + +/// Data files whose object metadata and footer one scan reads at a time. +const FOOTER_FETCH_CONCURRENCY: usize = 32; + +const LOAD_OPERATION: &str = "algeon_iceberg.load_table"; + +/// An Iceberg table that DataFusion scans as Parquet. +/// +/// Every [`TableProvider::scan`] loads the table again and plans the files +/// of its current snapshot. The scan is a `DataSourceExec` over a stock +/// `ParquetSource` with one file per Iceberg task, which Algeon's native +/// lowering reads like a listing table. A snapshot whose files that reader +/// would read incorrectly fails the scan with +/// [`ErrorCode::UnsupportedTableFeature`] in every execution mode; the +/// `reason` fact names the feature. +pub struct IcebergTableProvider { + catalog: Arc, + ident: TableIdent, + storage: Option, + schema: SchemaRef, +} + +impl fmt::Debug for IcebergTableProvider { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("IcebergTableProvider") + .field("ident", &self.ident) + .field("storage", &self.storage) + .field("schema", &self.schema) + .finish_non_exhaustive() + } +} + +impl IcebergTableProvider { + /// Loads `ident` from `catalog` and takes its schema. + /// + /// `storage` is the S3 storage the session reads data files with. When + /// set, a table whose FileIO properties name another endpoint, region or + /// addressing style is rejected with reason + /// `iceberg_storage_properties_mismatch`. A missing table fails with + /// [`ErrorCode::ObjectNotFound`]. + pub async fn try_new( + catalog: Arc, + ident: TableIdent, + storage: Option, + ) -> Result { + let mut provider = Self { catalog, ident, storage, schema: Arc::new(Schema::empty()) }; + let table = provider.load().await?; + provider.schema = table_arrow_schema(scan_schema(&table)?.as_ref())?; + Ok(provider) + } + + /// The table identifier in its catalog. + pub fn ident(&self) -> &TableIdent { + &self.ident + } + + async fn load(&self) -> Result { + let table = self.catalog.load_table(&self.ident).await.map_err(|source| { + iceberg_error(LOAD_OPERATION, "failed to load the Iceberg table", source).with_fact("table", &self.ident) + })?; + if let Some(storage) = &self.storage + && let Some(property) = mismatched_property(table.file_io().config().props(), storage) + { + return Err(unsupported( + UnsupportedReason::StoragePropertiesMismatch, + &self.ident, + table.metadata().current_snapshot_id(), + format!("the Iceberg table's FileIO property `{property}` differs from the server's [s3] storage"), + ) + .with_fact("file_io_property", property)); + } + Ok(table) + } + + async fn scan_plan( + &self, + state: &dyn Session, + projection: Option<&Vec>, + filters: &[Expr], + limit: Option, + ) -> DataFusionResult> { + let projected_schema = match projection { + Some(indices) => Arc::new(self.schema.project(indices)?), + None => Arc::clone(&self.schema), + }; + let table = self.load().await?; + let Some(snapshot) = table.metadata().current_snapshot() else { + return Ok(Arc::new(EmptyExec::new(projected_schema))); + }; + let snapshot_id = snapshot.snapshot_id(); + let iceberg_schema = scan_schema(&table)?; + if table_arrow_schema(&iceberg_schema)? != self.schema { + return Err(Error::new( + ErrorCode::SourceTemporaryFailure, + SCAN_OPERATION, + "the Iceberg table schema changed after the query resolved the table; run the query again", + ) + .with_fact("table", &self.ident) + .with_fact("snapshot_id", snapshot_id) + .into()); + } + + let mut scan = table.scan().snapshot_id(snapshot_id); + scan = match projection { + Some(indices) => scan.select(indices.iter().map(|index| self.schema.field(*index).name().clone())), + None => scan.select_all(), + }; + if let Some(predicate) = filters_to_predicate(filters, &iceberg_schema) { + scan = scan.with_filter(predicate); + } + let plan_error = |source| iceberg_error(SCAN_OPERATION, "failed to plan the Iceberg scan", source); + let tasks = scan + .build() + .map_err(plan_error)? + .plan_files() + .await + .map_err(plan_error)? + .try_collect::>() + .await + .map_err(plan_error)?; + + let context = ScanContext { ident: &self.ident, snapshot_id, name_mapping: table_name_mapping(&table)? }; + let mut store_url = None; + let mut located = Vec::with_capacity(tasks.len()); + for task in tasks { + let (url, path) = context.check_task(&task)?; + match &store_url { + None => store_url = Some(url), + Some(first) if *first != url => { + return Err(context + .rejection( + UnsupportedReason::MultipleBuckets, + "the snapshot's data files are in more than one bucket", + ) + .into()); + }, + Some(_) => {}, + } + located.push((task, path)); + } + let Some(store_url) = store_url else { + return Ok(Arc::new(EmptyExec::new(projected_schema))); + }; + + let runtime_env = state.runtime_env(); + let store = runtime_env.object_store(&store_url)?; + let metadata_cache = runtime_env.cache_manager.get_file_metadata_cache(); + let metadata_size_hint = state.table_options().parquet.global.metadata_size_hint; + let files = stream::iter(located) + .map(|(task, path)| { + let store = Arc::clone(&store); + let metadata_cache = Arc::clone(&metadata_cache); + let context = &context; + let table_schema = &self.schema; + async move { + context + .partitioned_file(store.as_ref(), metadata_cache, metadata_size_hint, &task, path, table_schema) + .await + } + }) + .buffered(FOOTER_FETCH_CONCURRENCY) + .try_collect::>() + .await?; + + let total_rows = files + .iter() + .map(|file| file.statistics.as_ref().map_or(Precision::Absent, |stats| stats.num_rows)) + .fold(Precision::Exact(0), |total, rows| total.add(&rows)); + let statistics = Statistics { num_rows: total_rows, ..Statistics::new_unknown(&self.schema) }; + let file_groups = FileGroup::new(files).split_files(state.config().target_partitions().max(1)); + let format = ParquetFormat::default().with_options(state.table_options().parquet.clone()); + let source = format.file_source(TableSchema::from(Arc::clone(&self.schema))); + let config = FileScanConfigBuilder::new(store_url, source) + .with_file_groups(file_groups) + .with_statistics(statistics) + .with_projection_indices(projection.cloned())? + .with_limit(limit) + .build(); + format.create_physical_plan(state, config).await + } +} + +/// The schema `TableScan` binds against: the current snapshot's schema, or +/// the table's current schema when it has no snapshot. +fn scan_schema(table: &Table) -> Result { + match table.metadata().current_snapshot() { + Some(snapshot) => snapshot + .schema(table.metadata()) + .map_err(|source| iceberg_error(LOAD_OPERATION, "the Iceberg snapshot names an unknown schema", source)), + None => Ok(Arc::clone(table.metadata().current_schema())), + } +} + +fn table_name_mapping(table: &Table) -> Result> { + table + .metadata() + .table_properties() + .default_name_mapping() + .map_err(|source| iceberg_error(SCAN_OPERATION, "the Iceberg table's name mapping is invalid", source)) +} + +struct ScanContext<'a> { + ident: &'a TableIdent, + snapshot_id: i64, + name_mapping: Option, +} + +impl ScanContext<'_> { + fn rejection(&self, reason: UnsupportedReason, message: impl Into) -> Error { + unsupported(reason, self.ident, Some(self.snapshot_id), message) + } + + /// Rejects a task the Parquet scan cannot read and returns the store + /// URL and in-store path of its data file. + fn check_task(&self, task: &FileScanTask) -> Result<(ObjectStoreUrl, ObjectPath)> { + let location = task.data_file_path(); + let file_rejection = |reason, message: &str| self.rejection(reason, message).with_fact("path", location); + if !task.deletes().is_empty() { + return Err(file_rejection(UnsupportedReason::DeleteFiles, "a data file of the snapshot has delete files") + .with_fact("delete_files", task.deletes().len())); + } + if task.data_file_format() != DataFileFormat::Parquet { + return Err(file_rejection( + UnsupportedReason::NonParquetFile, + "a data file of the snapshot is not Parquet", + ) + .with_fact("file_format", task.data_file_format())); + } + if task.key_metadata().is_some() { + return Err(file_rejection(UnsupportedReason::EncryptedFile, "a data file of the snapshot is encrypted")); + } + split_location(location).ok_or_else(|| { + file_rejection( + UnsupportedReason::UnsupportedLocationScheme, + "a data file of the snapshot is outside s3:// storage", + ) + }) + } + + /// Reads the object metadata and footer of a task's data file, checks + /// its column binding, and returns it as a scan file. The footer goes + /// into the session's file metadata cache, where native lowering reads + /// it. + async fn partitioned_file( + &self, + store: &dyn ObjectStore, + metadata_cache: Arc, + metadata_size_hint: Option, + task: &FileScanTask, + path: ObjectPath, + table_schema: &SchemaRef, + ) -> DataFusionResult { + let object = store.head(&path).await.map_err(|error| DataFusionError::ObjectStore(Box::new(error)))?; + if object.size != task.file_size_in_bytes() { + return Err(Error::new( + ErrorCode::SourceDependencyFailure, + SCAN_OPERATION, + "a data file's size differs from the size its Iceberg manifest records", + ) + .with_fact("table", self.ident) + .with_fact("snapshot_id", self.snapshot_id) + .with_fact("path", task.data_file_path()) + .with_fact("object_size", object.size) + .with_fact("manifest_size", task.file_size_in_bytes()) + .into()); + } + let footer = DFParquetMetadata::new(store, &object) + .with_metadata_size_hint(metadata_size_hint) + .with_file_metadata_cache(Some(metadata_cache)) + .fetch_metadata() + .await?; + let file_metadata = footer.file_metadata(); + let file_schema = parquet_to_arrow_schema(file_metadata.schema_descr(), file_metadata.key_value_metadata())?; + let binding = FileBinding { + identity_partition_sources: task + .partition_spec() + .map(|spec| identity_partition_sources(spec)) + .unwrap_or_default(), + name_mapping: task + .name_mapping() + .map(|mapping| mapping.fields()) + .or_else(|| self.name_mapping.as_ref().map(NameMapping::fields)), + }; + binding.check(task.schema(), task.project_field_ids(), file_schema.fields()).map_err(|rejection| { + let message = match rejection.reason { + UnsupportedReason::FieldIdMismatch => "a data file binds a column by field id to another name", + UnsupportedReason::MissingFieldIds => { + "a data file without field ids has no name mapping that resolves a column to its current name" + }, + UnsupportedReason::PartitionColumnAbsent => { + "a data file lacks an identity partition column, whose values are only in partition data" + }, + _ => "a data file lacks a required column", + }; + self.rejection(rejection.reason, message) + .with_fact("path", task.data_file_path()) + .with_fact("column", rejection.field) + })?; + if let Some(column) = column_with_stored_type_mismatch( + task.schema(), + task.project_field_ids(), + table_schema.fields(), + file_schema.fields(), + ) { + return Err(self + .rejection( + UnsupportedReason::ColumnTypeMismatch, + "a data file stores a column with another type than the table schema declares", + ) + .with_fact("path", task.data_file_path()) + .with_fact("column", column) + .into()); + } + Ok(file_with_row_count(object, task.record_count(), table_schema)) + } +} + +fn file_with_row_count(object: ObjectMeta, record_count: Option, table_schema: &SchemaRef) -> PartitionedFile { + let num_rows = record_count.and_then(|rows| usize::try_from(rows).ok()).map_or(Precision::Absent, Precision::Exact); + let statistics = Statistics { num_rows, ..Statistics::new_unknown(table_schema) }; + PartitionedFile::new_from_meta(object).with_statistics(Arc::new(statistics)) +} + +/// The object store URL and in-store path of a data file location. +/// +/// Iceberg's FileIO treats `s3a://` and `s3n://` as `s3://`; the native +/// reader accepts only `s3://`, so the URL is rewritten. `file://` serves +/// local warehouses. +fn split_location(location: &str) -> Option<(ObjectStoreUrl, ObjectPath)> { + let (scheme, rest) = location.split_once("://")?; + let (url, key) = match scheme { + "s3" | "s3a" | "s3n" => { + let (bucket, key) = rest.split_once('/')?; + if bucket.is_empty() { + return None; + } + (ObjectStoreUrl::parse(format!("s3://{bucket}")).ok()?, key) + }, + "file" => (ObjectStoreUrl::local_filesystem(), rest), + _ => return None, + }; + let path = ObjectPath::parse(key.trim_start_matches('/')).ok()?; + Some((url, path)) +} + +#[async_trait] +impl TableProvider for IcebergTableProvider { + fn schema(&self) -> SchemaRef { + Arc::clone(&self.schema) + } + + fn table_type(&self) -> TableType { + TableType::Base + } + + fn supports_filters_pushdown(&self, filters: &[&Expr]) -> DataFusionResult> { + Ok(vec![TableProviderFilterPushDown::Inexact; filters.len()]) + } + + async fn scan( + &self, + state: &dyn Session, + projection: Option<&Vec>, + filters: &[Expr], + limit: Option, + ) -> DataFusionResult> { + self.scan_plan(state, projection, filters, limit).await + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/algeon-iceberg/src/table/tests.rs b/crates/algeon-iceberg/src/table/tests.rs new file mode 100644 index 000000000..c4fc161c0 --- /dev/null +++ b/crates/algeon-iceberg/src/table/tests.rs @@ -0,0 +1,18 @@ +use datafusion::execution::object_store::ObjectStoreUrl; + +use super::split_location; + +#[test] +fn s3_family_locations_share_the_s3_bucket_url_and_other_schemes_are_refused() { + for scheme in ["s3", "s3a", "s3n"] { + let (url, path) = split_location(&format!("{scheme}://bucket/warehouse/a.parquet")).unwrap(); + assert_eq!(url, ObjectStoreUrl::parse("s3://bucket").unwrap()); + assert_eq!(path.as_ref(), "warehouse/a.parquet"); + } + let (url, path) = split_location("file:///tmp/warehouse/a.parquet").unwrap(); + assert_eq!(url, ObjectStoreUrl::local_filesystem()); + assert_eq!(path.as_ref(), "tmp/warehouse/a.parquet"); + for location in ["gs://bucket/a.parquet", "s3:///a.parquet", "/tmp/a.parquet", "abfss://c@a/x.parquet"] { + assert!(split_location(location).is_none(), "{location}"); + } +} diff --git a/crates/algeon-iceberg/tests/iceberg_tests_cpu/main.rs b/crates/algeon-iceberg/tests/iceberg_tests_cpu/main.rs new file mode 100644 index 000000000..455b1d8db --- /dev/null +++ b/crates/algeon-iceberg/tests/iceberg_tests_cpu/main.rs @@ -0,0 +1,10 @@ +//! CPU tests of Iceberg scan planning: plan shape, pruning, the storage +//! paths, catalog registration, native planning of a remote table, and every +//! rejection of a snapshot the Parquet scan would misread. + +mod native_planning; +mod register; +mod rejections; +mod scan; +#[path = "../support/warehouse.rs"] +mod support; diff --git a/crates/algeon-iceberg/tests/iceberg_tests_cpu/native_planning.rs b/crates/algeon-iceberg/tests/iceberg_tests_cpu/native_planning.rs new file mode 100644 index 000000000..98fb2ecf7 --- /dev/null +++ b/crates/algeon-iceberg/tests/iceberg_tests_cpu/native_planning.rs @@ -0,0 +1,53 @@ +use std::{collections::HashMap, sync::Arc}; + +use algeon_datafusion::{ + planner::{AlgeonGpuExecutionMode, AlgeonNativeOptimizerConfig, plan_with_native_diagnostics}, + session::AlgeonSessionStateBuilderExt, +}; +use datafusion::{ + execution::{SessionStateBuilder, object_store::ObjectStoreUrl}, + object_store::{ObjectStoreExt, PutPayload, memory::InMemory, path::Path as ObjectPath}, + physical_plan::displayable, + prelude::{SessionConfig, SessionContext}, +}; + +use crate::support::{Warehouse, data_file, people_arrow_schema, people_batch, people_schema, write_parquet_bytes}; + +fn native_required_session() -> SessionContext { + let config = SessionConfig::new().with_target_partitions(1); + let optimizer = AlgeonNativeOptimizerConfig::default().with_execution_mode(AlgeonGpuExecutionMode::NativeRequired); + let state = + SessionStateBuilder::new_with_default_features().with_config(config).with_algeon_native(optimizer).build(); + SessionContext::new_with_state(state) +} + +/// Native lowering reads the footers of a multi-file remote scan only from +/// the session's file metadata cache, which the provider fills during +/// `scan`; without them `native_required` planning fails. Planning goes +/// through an invocation, as a server statement's attempt does, because +/// only an invocation opens the footer scope. +#[tokio::test] +async fn a_multi_file_s3_table_plans_as_one_native_exec_under_native_required() { + let warehouse = Warehouse::new().await; + let table = warehouse.create_table("remote", people_schema(), None, HashMap::new()).await; + let store = Arc::new(InMemory::new()); + let mut entries = Vec::new(); + for (index, ids) in [[1_i64, 2], [101, 102]].iter().enumerate() { + let bytes = write_parquet_bytes(&people_batch(&people_arrow_schema(Some([1, 2, 3])), ids)); + let size = bytes.len() as u64; + let key = format!("warehouse/remote/part-{index}.parquet"); + store.put(&ObjectPath::from(key.as_str()), PutPayload::from(bytes)).await.unwrap(); + entries.push(data_file(&format!("s3://bucket/{key}"), size, 2).build().unwrap()); + } + warehouse.append(&table, entries).await; + let ctx = native_required_session(); + ctx.register_object_store(ObjectStoreUrl::parse("s3://bucket").unwrap().as_ref(), store); + warehouse.register(&ctx).await; + + let sql = "SELECT name, count(*) FROM iceberg.ns.remote WHERE id > 1 GROUP BY name"; + let (state, logical) = ctx.sql(sql).await.unwrap().into_parts(); + let (plan, _) = plan_with_native_diagnostics(&state, &logical).await.unwrap(); + let rendered = displayable(plan.as_ref()).indent(false).to_string(); + assert!(rendered.starts_with("AlgeonNativeExec"), "{rendered}"); + assert_eq!(rendered.lines().count(), 1, "{rendered}"); +} diff --git a/crates/algeon-iceberg/tests/iceberg_tests_cpu/register.rs b/crates/algeon-iceberg/tests/iceberg_tests_cpu/register.rs new file mode 100644 index 000000000..7dcf6efbb --- /dev/null +++ b/crates/algeon-iceberg/tests/iceberg_tests_cpu/register.rs @@ -0,0 +1,49 @@ +use std::{fs, net::TcpListener, sync::Arc}; + +use algeon_datafusion::{ + error::ErrorCode, + server::{ALGEON_SERVER_CONFIG_FILE_ENV, LoadedServerConfig, ServerConfig}, +}; +use algeon_iceberg::register_iceberg_catalogs; +use datafusion::{catalog::MemoryCatalogProvider, prelude::SessionContext}; + +fn server_config(catalogs: &str) -> (tempfile::TempDir, ServerConfig) { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("server.toml"); + let toml = format!("[s3]\nendpoint = \"http://127.0.0.1:9\"\npath_style = true\n\n{catalogs}"); + fs::write(&path, toml).unwrap(); + let loaded = + LoadedServerConfig::from_vars([(ALGEON_SERVER_CONFIG_FILE_ENV.to_string(), path.display().to_string())]) + .unwrap(); + (dir, loaded.config) +} + +#[tokio::test] +async fn a_catalog_name_the_session_already_holds_is_refused() { + let (_dir, config) = + server_config("[[iceberg.catalogs]]\nname = \"lake\"\nkind = \"rest\"\nuri = \"http://127.0.0.1:9\"\n"); + let ctx = SessionContext::new(); + ctx.register_catalog("lake", Arc::new(MemoryCatalogProvider::new())); + + let error = register_iceberg_catalogs(&ctx, &config).await.unwrap_err(); + assert_eq!(error.code(), ErrorCode::InvalidConfiguration); + assert_eq!(error.fact_value("config_field"), Some("iceberg.catalogs.name")); +} + +#[tokio::test] +async fn an_unreachable_rest_catalog_fails_registration_with_a_source_error() { + // A bound and closed port refuses the connection. + let port = TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port(); + let (_dir, config) = server_config(&format!( + "[[iceberg.catalogs]]\nname = \"lake\"\nkind = \"rest\"\nuri = \"http://127.0.0.1:{port}\"\n" + )); + let ctx = SessionContext::new(); + + let error = register_iceberg_catalogs(&ctx, &config).await.unwrap_err(); + assert!( + matches!(error.code(), ErrorCode::SourceTemporaryFailure | ErrorCode::SourceDependencyFailure), + "{error:?}" + ); + assert_eq!(error.fact_value("catalog"), Some("lake")); + assert!(ctx.catalog("lake").is_none()); +} diff --git a/crates/algeon-iceberg/tests/iceberg_tests_cpu/rejections.rs b/crates/algeon-iceberg/tests/iceberg_tests_cpu/rejections.rs new file mode 100644 index 000000000..3ac3ee1f6 --- /dev/null +++ b/crates/algeon-iceberg/tests/iceberg_tests_cpu/rejections.rs @@ -0,0 +1,260 @@ +use std::{collections::HashMap, sync::Arc}; + +use algeon_datafusion::{error::ErrorCode, native_exec::S3SourceStorageProfile}; +use algeon_iceberg::IcebergCatalogProvider; +use datafusion::prelude::SessionContext; +use iceberg::spec::{ + DataFile, DataFileFormat, Literal, Struct, Transform, UnboundPartitionField, UnboundPartitionSpec, +}; + +use crate::support::{ + Failure, Warehouse, data_file, people_arrow_schema, people_batch, people_schema, plan_failure, write_parquet, + write_people, +}; + +fn assert_rejected(failure: &Failure, table: &str, reason: &str) { + assert_eq!(failure.code, ErrorCode::UnsupportedTableFeature, "{failure:?}"); + assert_eq!(failure.fact("reason"), Some(reason), "{failure:?}"); + assert_eq!(failure.fact("table"), Some(format!("ns.{table}").as_str()), "{failure:?}"); + assert!(failure.fact("snapshot_id").is_some(), "{failure:?}"); +} + +/// A table holding one hand-written Parquet file whose Arrow schema has +/// the field ids `ids` (none when `None`), with `rows` ids. +async fn table_with_written_file( + warehouse: &Warehouse, + name: &str, + ids: Option<[i32; 3]>, + properties: HashMap, +) { + let table = warehouse.create_table(name, people_schema(), None, properties).await; + let path = warehouse.local_path(&format!("{name}.parquet")); + let size = write_parquet(path.as_ref(), &people_batch(&people_arrow_schema(ids), &[1, 2])); + warehouse.append(&table, vec![data_file(&format!("file://{path}"), size, 2).build().unwrap()]).await; +} + +/// A table holding one manifest entry per location, none of them read. +async fn table_with_entries(warehouse: &Warehouse, name: &str, entries: Vec) { + let table = warehouse.create_table(name, people_schema(), None, HashMap::new()).await; + warehouse.append(&table, entries).await; +} + +#[tokio::test] +async fn a_column_whose_field_id_belongs_to_another_name_is_rejected() { + let warehouse = Warehouse::new().await; + table_with_written_file(&warehouse, "renamed", Some([1, 9, 3]), HashMap::new()).await; + let ctx = warehouse.session().await; + + let failure = plan_failure(&ctx, "SELECT name FROM iceberg.ns.renamed").await; + assert_rejected(&failure, "renamed", "iceberg_field_id_mismatch"); + assert_eq!(failure.fact("column"), Some("name")); + // Columns whose ids agree stay readable. + assert!(ctx.sql("SELECT id FROM iceberg.ns.renamed").await.unwrap().collect().await.is_ok()); +} + +#[tokio::test] +async fn a_file_without_field_ids_and_no_name_mapping_is_rejected() { + let warehouse = Warehouse::new().await; + table_with_written_file(&warehouse, "unmapped", None, HashMap::new()).await; + let ctx = warehouse.session().await; + + let failure = plan_failure(&ctx, "SELECT id FROM iceberg.ns.unmapped").await; + assert_rejected(&failure, "unmapped", "iceberg_missing_field_ids"); +} + +#[tokio::test] +async fn a_name_mapping_that_resolves_a_column_to_another_name_is_rejected() { + let warehouse = Warehouse::new().await; + let mapping = + r#"[{"field-id":1,"names":["id"]},{"field-id":2,"names":["full_name"]},{"field-id":3,"names":["year"]}]"#; + let properties = HashMap::from([("schema.name-mapping.default".to_string(), mapping.to_string())]); + table_with_written_file(&warehouse, "mismapped", None, properties).await; + let ctx = warehouse.session().await; + + let failure = plan_failure(&ctx, "SELECT name FROM iceberg.ns.mismapped").await; + assert_rejected(&failure, "mismapped", "iceberg_missing_field_ids"); +} + +#[tokio::test] +async fn a_required_column_missing_from_a_file_is_rejected() { + let warehouse = Warehouse::new().await; + let table = warehouse.create_table("no_id", people_schema(), None, HashMap::new()).await; + let path = warehouse.local_path("no_id.parquet"); + let batch = people_batch(&people_arrow_schema(Some([1, 2, 3])), &[1, 2]).project(&[1, 2]).unwrap(); + let size = write_parquet(path.as_ref(), &batch); + warehouse.append(&table, vec![data_file(&format!("file://{path}"), size, 2).build().unwrap()]).await; + let ctx = warehouse.session().await; + + let failure = plan_failure(&ctx, "SELECT id FROM iceberg.ns.no_id").await; + assert_rejected(&failure, "no_id", "iceberg_required_column_absent"); + // An optional column absent from the file reads as null. + let batch = people_batch(&people_arrow_schema(Some([1, 2, 3])), &[1, 2]).project(&[0, 1]).unwrap(); + let path = warehouse.local_path("no_year.parquet"); + let size = write_parquet(path.as_ref(), &batch); + let table = warehouse.create_table("no_year", people_schema(), None, HashMap::new()).await; + warehouse.append(&table, vec![data_file(&format!("file://{path}"), size, 2).build().unwrap()]).await; + let ctx = warehouse.session().await; + let batches = ctx.sql("SELECT year FROM iceberg.ns.no_year").await.unwrap().collect().await.unwrap(); + assert_eq!(batches.iter().map(|batch| batch.column(0).null_count()).sum::(), 2); +} + +#[tokio::test] +async fn a_column_stored_with_its_type_before_a_promotion_is_rejected() { + use datafusion::arrow::{ + array::{ArrayRef, Int32Array, StringArray}, + datatypes::{DataType, Schema as ArrowSchema}, + record_batch::RecordBatch, + }; + + use crate::support::field_with_id; + + let warehouse = Warehouse::new().await; + // `id` is `long` in the table; the file keeps the `int` it was written with. + let table = warehouse.create_table("promoted", people_schema(), None, HashMap::new()).await; + let schema = Arc::new(ArrowSchema::new(vec![ + field_with_id("id", DataType::Int32, false, 1), + field_with_id("name", DataType::Utf8, true, 2), + field_with_id("year", DataType::Int32, true, 3), + ])); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + Arc::new(StringArray::from(vec!["a", "b"])) as ArrayRef, + Arc::new(Int32Array::from(vec![2000, 2001])) as ArrayRef, + ], + ) + .unwrap(); + let path = warehouse.local_path("promoted.parquet"); + let size = write_parquet(path.as_ref(), &batch); + warehouse.append(&table, vec![data_file(&format!("file://{path}"), size, 2).build().unwrap()]).await; + let ctx = warehouse.session().await; + + let failure = plan_failure(&ctx, "SELECT id FROM iceberg.ns.promoted").await; + assert_rejected(&failure, "promoted", "iceberg_column_type_mismatch"); + assert_eq!(failure.fact("column"), Some("id"), "{failure:?}"); + // The string and int columns match their table types. + let batches = ctx.sql("SELECT name, year FROM iceberg.ns.promoted").await.unwrap().collect().await.unwrap(); + assert_eq!(batches.iter().map(|batch| batch.num_rows()).sum::(), 2); +} + +#[tokio::test] +async fn an_identity_partition_column_kept_only_in_partition_data_is_rejected() { + let warehouse = Warehouse::new().await; + let spec = UnboundPartitionSpec::builder() + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![3]) + .name("year".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + ) + .unwrap() + .build(); + let table = warehouse.create_table("by_year", people_schema(), Some(spec), HashMap::new()).await; + let path = warehouse.local_path("by_year.parquet"); + let batch = people_batch(&people_arrow_schema(Some([1, 2, 3])), &[1, 2]).project(&[0, 1]).unwrap(); + let size = write_parquet(path.as_ref(), &batch); + let file = data_file(&format!("file://{path}"), size, 2) + .partition(Struct::from_iter([Some(Literal::int(2000))])) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .build() + .unwrap(); + warehouse.append(&table, vec![file]).await; + let ctx = warehouse.session().await; + + let failure = plan_failure(&ctx, "SELECT year FROM iceberg.ns.by_year").await; + assert_rejected(&failure, "by_year", "iceberg_partition_column_absent"); +} + +#[tokio::test] +async fn delete_files_are_rejected() { + let warehouse = Warehouse::new().await; + let table = warehouse.create_table("people", people_schema(), None, HashMap::new()).await; + let files = write_people(&table, &[1, 2, 3]).await; + let target = files[0].file_path().to_string(); + let table = warehouse.append(&table, files).await; + warehouse.commit_position_delete(&table, "with_deletes", &target).await; + let ctx = warehouse.session().await; + + let failure = plan_failure(&ctx, "SELECT id FROM iceberg.ns.with_deletes").await; + assert_rejected(&failure, "with_deletes", "iceberg_delete_files"); +} + +#[tokio::test] +async fn a_data_file_that_is_not_parquet_is_rejected() { + let warehouse = Warehouse::new().await; + let path = format!("file://{}", warehouse.local_path("data.orc")); + let entry = data_file(&path, 100, 1).file_format(DataFileFormat::Orc).build().unwrap(); + table_with_entries(&warehouse, "orc", vec![entry]).await; + let ctx = warehouse.session().await; + + assert_rejected(&plan_failure(&ctx, "SELECT id FROM iceberg.ns.orc").await, "orc", "iceberg_non_parquet_file"); +} + +#[tokio::test] +async fn an_encrypted_data_file_is_rejected() { + let warehouse = Warehouse::new().await; + let path = format!("file://{}", warehouse.local_path("sealed.parquet")); + let entry = data_file(&path, 100, 1).key_metadata(Some(vec![1, 2, 3])).build().unwrap(); + table_with_entries(&warehouse, "sealed", vec![entry]).await; + let ctx = warehouse.session().await; + + assert_rejected(&plan_failure(&ctx, "SELECT id FROM iceberg.ns.sealed").await, "sealed", "iceberg_encrypted_file"); +} + +#[tokio::test] +async fn a_data_file_outside_s3_storage_is_rejected() { + let warehouse = Warehouse::new().await; + let entry = data_file("gs://bucket/data.parquet", 100, 1).build().unwrap(); + table_with_entries(&warehouse, "gcs", vec![entry]).await; + let ctx = warehouse.session().await; + + let failure = plan_failure(&ctx, "SELECT id FROM iceberg.ns.gcs").await; + assert_rejected(&failure, "gcs", "iceberg_unsupported_location_scheme"); +} + +#[tokio::test] +async fn data_files_in_two_buckets_are_rejected() { + let warehouse = Warehouse::new().await; + let entries = vec![ + data_file("s3://first/a.parquet", 100, 1).build().unwrap(), + data_file("s3a://second/b.parquet", 100, 1).build().unwrap(), + ]; + table_with_entries(&warehouse, "split", entries).await; + let ctx = warehouse.session().await; + + assert_rejected(&plan_failure(&ctx, "SELECT id FROM iceberg.ns.split").await, "split", "iceberg_multiple_buckets"); +} + +#[tokio::test] +async fn a_table_whose_file_io_addresses_other_storage_is_rejected() { + let catalog_endpoint = HashMap::from([("s3.endpoint".to_string(), "http://catalog-store:9000".to_string())]); + let warehouse = Warehouse::with_properties(catalog_endpoint).await; + let table = warehouse.create_table("people", people_schema(), None, HashMap::new()).await; + let files = write_people(&table, &[1]).await; + warehouse.append(&table, files).await; + let storage = S3SourceStorageProfile::default_chain().with_endpoint("http://data-store:9000"); + let ctx = SessionContext::new(); + let provider = IcebergCatalogProvider::try_new(Arc::clone(&warehouse.catalog), Some(storage)).await.unwrap(); + ctx.register_catalog("iceberg", Arc::new(provider)); + + let failure = plan_failure(&ctx, "SELECT id FROM iceberg.ns.people").await; + assert_rejected(&failure, "people", "iceberg_storage_properties_mismatch"); + assert_eq!(failure.fact("file_io_property"), Some("s3.endpoint")); +} + +#[tokio::test] +async fn a_data_file_whose_size_differs_from_the_manifest_fails_as_a_source_error() { + let warehouse = Warehouse::new().await; + let table = warehouse.create_table("resized", people_schema(), None, HashMap::new()).await; + let path = warehouse.local_path("resized.parquet"); + let size = write_parquet(path.as_ref(), &people_batch(&people_arrow_schema(Some([1, 2, 3])), &[1])); + warehouse.append(&table, vec![data_file(&format!("file://{path}"), size + 1, 1).build().unwrap()]).await; + let ctx = warehouse.session().await; + + let failure = plan_failure(&ctx, "SELECT id FROM iceberg.ns.resized").await; + assert_eq!(failure.code, ErrorCode::SourceDependencyFailure, "{failure:?}"); + assert_eq!(failure.fact("manifest_size"), Some((size + 1).to_string().as_str())); +} diff --git a/crates/algeon-iceberg/tests/iceberg_tests_cpu/scan.rs b/crates/algeon-iceberg/tests/iceberg_tests_cpu/scan.rs new file mode 100644 index 000000000..0bbeb6fb4 --- /dev/null +++ b/crates/algeon-iceberg/tests/iceberg_tests_cpu/scan.rs @@ -0,0 +1,139 @@ +use std::{collections::HashMap, sync::Arc}; + +use datafusion::{ + arrow::datatypes::DataType, + catalog::TableProvider, + common::stats::Precision, + datasource::{ + file_format::parquet::ParquetFormat, + listing::{ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl}, + }, + execution::object_store::ObjectStoreUrl, + object_store::{ObjectStoreExt, PutPayload, memory::InMemory, path::Path as ObjectPath}, + physical_plan::empty::EmptyExec, +}; +use iceberg::spec::DataFile; + +use crate::support::{ + Warehouse, data_file, file_scan, people_arrow_schema, people_batch, people_schema, plan, sorted_ids, write_parquet, + write_parquet_bytes, write_people, +}; + +#[tokio::test] +async fn each_task_is_one_whole_file_with_manifest_size_and_row_count_and_a_cached_footer() { + let warehouse = Warehouse::new().await; + let table = warehouse.create_table("people", people_schema(), None, HashMap::new()).await; + let mut files = write_people(&table, &[1, 2, 3]).await; + files.extend(write_people(&table, &[101, 102]).await); + let manifest = + files.iter().map(|file| (file.file_path().to_string(), file.file_size_in_bytes())).collect::>(); + warehouse.append(&table, files).await; + let ctx = warehouse.session().await; + + let plan = plan(&ctx, "SELECT id, name FROM iceberg.ns.people").await.unwrap(); + let scan = file_scan(&plan); + assert_eq!(scan.object_store_url, ObjectStoreUrl::local_filesystem()); + let scan_files = scan.file_groups.iter().flat_map(|group| group.files()).collect::>(); + assert_eq!(scan_files.len(), 2); + let cache = ctx.runtime_env().cache_manager.get_file_metadata_cache(); + let mut row_counts = Vec::new(); + for file in scan_files { + assert!(file.range.is_none()); + let location = format!("file:///{}", file.object_meta.location); + assert_eq!(Some(&file.object_meta.size), manifest.get(&location), "{location}"); + let rows = file.statistics.as_ref().unwrap().num_rows; + row_counts.push(rows); + assert!(cache.get(&file.object_meta.location).is_some(), "the footer of {location} must be cached"); + } + row_counts.sort_by_key(|rows| rows.get_value().copied()); + assert_eq!(row_counts, [Precision::Exact(2), Precision::Exact(3)]); + assert_eq!(sorted_ids(&ctx, "SELECT id FROM iceberg.ns.people").await, [1, 2, 3, 101, 102]); +} + +#[tokio::test] +async fn a_filter_prunes_data_files_by_manifest_bounds() { + let warehouse = Warehouse::new().await; + let table = warehouse.create_table("people", people_schema(), None, HashMap::new()).await; + let mut files = write_people(&table, &[1, 2, 3]).await; + files.extend(write_people(&table, &[101, 102]).await); + warehouse.append(&table, files).await; + let ctx = warehouse.session().await; + + let sql = "SELECT id FROM iceberg.ns.people WHERE id >= 100"; + let scan = file_scan(&plan(&ctx, sql).await.unwrap()); + assert_eq!(scan.file_groups.iter().map(|group| group.len()).sum::(), 1); + assert_eq!(sorted_ids(&ctx, sql).await, [101, 102]); +} + +#[tokio::test] +async fn a_table_without_a_snapshot_scans_no_rows() { + let warehouse = Warehouse::new().await; + warehouse.create_table("empty", people_schema(), None, HashMap::new()).await; + let ctx = warehouse.session().await; + + let plan = plan(&ctx, "SELECT id, name FROM iceberg.ns.empty").await.unwrap(); + assert!( + plan.downcast_ref::().is_some() + || plan.children().iter().any(|child| child.downcast_ref::().is_some()) + ); + assert!(sorted_ids(&ctx, "SELECT id FROM iceberg.ns.empty").await.is_empty()); +} + +#[tokio::test] +async fn s3a_locations_read_through_the_s3_bucket_store() { + let warehouse = Warehouse::new().await; + let table = warehouse.create_table("remote", people_schema(), None, HashMap::new()).await; + let store = Arc::new(InMemory::new()); + let bytes = write_parquet_bytes(&people_batch(&people_arrow_schema(Some([1, 2, 3])), &[7, 8])); + let size = bytes.len() as u64; + store.put(&ObjectPath::from("data/remote.parquet"), PutPayload::from(bytes)).await.unwrap(); + let file: DataFile = data_file("s3a://bucket/data/remote.parquet", size, 2).build().unwrap(); + warehouse.append(&table, vec![file]).await; + let ctx = warehouse.session().await; + ctx.register_object_store(ObjectStoreUrl::parse("s3://bucket").unwrap().as_ref(), store); + + let plan = plan(&ctx, "SELECT id FROM iceberg.ns.remote").await.unwrap(); + let scan = file_scan(&plan); + assert_eq!(scan.object_store_url, ObjectStoreUrl::parse("s3://bucket").unwrap()); + let location = &scan.file_groups[0].files()[0].object_meta.location; + assert_eq!(location.as_ref(), "data/remote.parquet"); + assert!(ctx.runtime_env().cache_manager.get_file_metadata_cache().get(location).is_some()); + assert_eq!(sorted_ids(&ctx, "SELECT id FROM iceberg.ns.remote").await, [7, 8]); +} + +#[tokio::test] +async fn files_without_field_ids_bind_through_the_table_name_mapping() { + let warehouse = Warehouse::new().await; + let mapping = r#"[{"field-id":1,"names":["id"]},{"field-id":2,"names":["name"]},{"field-id":3,"names":["year"]}]"#; + let properties = HashMap::from([("schema.name-mapping.default".to_string(), mapping.to_string())]); + let table = warehouse.create_table("added", people_schema(), None, properties).await; + let path = warehouse.local_path("added.parquet"); + let size = write_parquet(path.as_ref(), &people_batch(&people_arrow_schema(None), &[4, 5, 6])); + warehouse.append(&table, vec![data_file(&format!("file://{path}"), size, 3).build().unwrap()]).await; + let ctx = warehouse.session().await; + + assert_eq!(sorted_ids(&ctx, "SELECT id FROM iceberg.ns.added WHERE name <> 'person-5'").await, [4, 6]); +} + +/// String columns use the type a DataFusion listing table infers for the +/// same files under the session's default `schema_force_view_types`. +#[tokio::test] +async fn string_columns_have_the_listing_table_type() { + let warehouse = Warehouse::new().await; + let table = warehouse.create_table("people", people_schema(), None, HashMap::new()).await; + let files = write_people(&table, &[1, 2]).await; + let location = files[0].file_path().to_string(); + warehouse.append(&table, files).await; + let ctx = warehouse.session().await; + + let url = ListingTableUrl::parse(&location).unwrap(); + let options = ListingOptions::new(Arc::new(ParquetFormat::default())); + let config = ListingTableConfig::new(url).with_listing_options(options).infer_schema(&ctx.state()).await.unwrap(); + let listing = ListingTable::try_new(config).unwrap(); + let listing_type = listing.schema().field_with_name("name").unwrap().data_type().clone(); + + let iceberg = ctx.table("iceberg.ns.people").await.unwrap(); + let iceberg_type = iceberg.schema().field_with_name(None, "name").unwrap().data_type().clone(); + assert_eq!(listing_type, DataType::Utf8View); + assert_eq!(iceberg_type, listing_type); +} diff --git a/crates/algeon-iceberg/tests/iceberg_tests_gpu.rs b/crates/algeon-iceberg/tests/iceberg_tests_gpu.rs new file mode 100644 index 000000000..8354ed5a3 --- /dev/null +++ b/crates/algeon-iceberg/tests/iceberg_tests_gpu.rs @@ -0,0 +1,80 @@ +//! An Iceberg table executed natively on the GPU returns the rows a +//! DataFusion listing table returns over the same data files. + +#[path = "support/warehouse.rs"] +mod support; + +use std::{collections::HashMap, sync::Arc, time::Duration}; + +use algeon_datafusion::{ + AttemptOptions, + backend::{AlgeonGpuBackend, AlgeonGpuDeviceConfig, AlgeonGpuDeviceProfile, GpuMemoryOwnership}, + planner::AlgeonGpuExecutionMode, +}; +use datafusion::{ + arrow::{array::RecordBatch, util::pretty::pretty_format_batches}, + datasource::{ + file_format::parquet::ParquetFormat, + listing::{ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl}, + }, + execution::SessionStateBuilder, + physical_plan::displayable, + prelude::{SessionConfig, SessionContext}, +}; +use futures::TryStreamExt; + +use crate::support::{Warehouse, people_schema, write_people}; + +const SQL: &str = "SELECT name, year, SUM(id) AS total FROM {table} WHERE id > 1 GROUP BY name, year ORDER BY name"; + +fn sorted_text(batches: &[RecordBatch]) -> String { + pretty_format_batches(batches).unwrap().to_string() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_local_warehouse_table_executes_natively_with_listing_table_results() { + let warehouse = Warehouse::new().await; + let table = warehouse.create_table("people", people_schema(), None, HashMap::new()).await; + let mut files = write_people(&table, &[1, 2, 3, 250]).await; + files.extend(write_people(&table, &[101, 102, 103]).await); + let data_dir = files[0].file_path().rsplit_once('/').unwrap().0.to_string(); + warehouse.append(&table, files).await; + + let device = AlgeonGpuDeviceConfig::new(AlgeonGpuDeviceProfile::new(0), GpuMemoryOwnership::WholeDeviceExclusive); + let backend = AlgeonGpuBackend::builder() + .devices([device]) + .with_execution_mode(AlgeonGpuExecutionMode::NativeRequired) + .build() + .expect("the Iceberg GPU test requires CUDA device 0"); + let state = backend + .install_on( + SessionStateBuilder::new_with_default_features() + .with_config(SessionConfig::new().with_target_partitions(1)), + ) + .unwrap() + .build(); + let ctx = SessionContext::new_with_state(state); + warehouse.register(&ctx).await; + + let attempt = backend.begin_attempt(AttemptOptions::default()).unwrap(); + let plan = attempt.plan_sql(&ctx, &SQL.replace("{table}", "iceberg.ns.people")).await.unwrap(); + let rendered = displayable(plan.as_ref()).indent(false).to_string(); + let operators = rendered.lines().map(|line| line.trim().split(':').next().unwrap()).collect::>(); + assert_eq!(operators, ["NativeAdmissionExec", "AlgeonNativeExec"], "{rendered}"); + let native = attempt.execute_physical_plan(plan, &ctx).unwrap().try_collect::>().await.unwrap(); + attempt.completion().await; + + let cpu = SessionContext::new(); + let options = ListingOptions::new(Arc::new(ParquetFormat::default())).with_file_extension(".parquet"); + let config = ListingTableConfig::new(ListingTableUrl::parse(format!("{data_dir}/")).unwrap()) + .with_listing_options(options) + .infer_schema(&cpu.state()) + .await + .unwrap(); + cpu.register_table("listing", Arc::new(ListingTable::try_new(config).unwrap())).unwrap(); + let expected = cpu.sql(&SQL.replace("{table}", "listing")).await.unwrap().collect().await.unwrap(); + + assert_eq!(native.iter().map(RecordBatch::num_rows).sum::(), 6); + assert_eq!(sorted_text(&native), sorted_text(&expected)); + backend.close_and_wait(Duration::from_secs(10)).unwrap(); +} diff --git a/crates/algeon-iceberg/tests/support/warehouse.rs b/crates/algeon-iceberg/tests/support/warehouse.rs new file mode 100644 index 000000000..372e09a2e --- /dev/null +++ b/crates/algeon-iceberg/tests/support/warehouse.rs @@ -0,0 +1,356 @@ +#![allow(dead_code, reason = "shared test helpers used across independent integration test targets")] + +//! In-process Iceberg tables: a memory catalog over a local warehouse, data +//! files written by Iceberg's Parquet writer or by hand, and snapshots +//! committed with `fast_append` or assembled from hand-written manifests. + +use std::{ + collections::HashMap, + fs::{self, File}, + path::Path, + str::FromStr, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, +}; + +use algeon_datafusion::error::{DataFusionErrorExt, Error as AlgeonError, ErrorCode}; +use algeon_iceberg::IcebergCatalogProvider; +use datafusion::{ + arrow::{ + array::{ArrayRef, Int32Array, Int64Array, RecordBatch, StringArray}, + datatypes::{DataType, Field, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef}, + }, + common::{DataFusionError, Result as DataFusionResult}, + datasource::{physical_plan::FileScanConfig, source::DataSourceExec}, + parquet::arrow::{ArrowWriter, PARQUET_FIELD_ID_META_KEY}, + physical_plan::ExecutionPlan, + prelude::SessionContext, +}; +use iceberg::{ + Catalog, CatalogBuilder, MetadataLocation, NamespaceIdent, TableCreation, TableIdent, + io::LocalFsStorageFactory, + memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}, + spec::{ + DataContentType, DataFile, DataFileBuilder, DataFileFormat, MAIN_BRANCH, ManifestListWriter, + ManifestWriterBuilder, NestedField, Operation, PrimitiveType, Schema, Snapshot, Struct, Summary, + TableMetadataBuilder, Type, UnboundPartitionSpec, + }, + table::Table, + transaction::{ApplyTransactionAction, Transaction}, + writer::{ + IcebergWriter, IcebergWriterBuilder, + base_writer::data_file_writer::DataFileWriterBuilder, + file_writer::{ + ParquetWriterBuilder, + location_generator::{DefaultFileNameGenerator, DefaultLocationGenerator}, + rolling_writer::RollingFileWriterBuilder, + }, + }, +}; +use tempfile::TempDir; + +pub(crate) const NAMESPACE: &str = "ns"; + +static FILE_PREFIX: AtomicUsize = AtomicUsize::new(0); + +/// A memory catalog whose warehouse is a temporary local directory. +pub(crate) struct Warehouse { + pub(crate) dir: TempDir, + pub(crate) catalog: Arc, +} + +impl Warehouse { + pub(crate) async fn new() -> Self { + Self::with_properties(HashMap::new()).await + } + + /// A warehouse whose catalog FileIO also receives `properties`. + pub(crate) async fn with_properties(properties: HashMap) -> Self { + let dir = tempfile::tempdir().unwrap(); + let mut catalog_properties = properties; + catalog_properties.insert(MEMORY_CATALOG_WAREHOUSE.to_string(), format!("file://{}", dir.path().display())); + let catalog = MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load("test", catalog_properties) + .await + .unwrap(); + catalog.create_namespace(&NamespaceIdent::new(NAMESPACE.to_string()), HashMap::new()).await.unwrap(); + Self { dir, catalog: Arc::new(catalog) } + } + + pub(crate) fn ident(name: &str) -> TableIdent { + TableIdent::new(NamespaceIdent::new(NAMESPACE.to_string()), name.to_string()) + } + + pub(crate) async fn create_table( + &self, + name: &str, + schema: Schema, + partition_spec: Option, + properties: HashMap, + ) -> Table { + let creation = TableCreation::builder() + .name(name.to_string()) + .schema(schema) + .partition_spec_opt(partition_spec) + .properties(properties) + .build(); + self.catalog.create_table(&NamespaceIdent::new(NAMESPACE.to_string()), creation).await.unwrap() + } + + pub(crate) async fn append(&self, table: &Table, files: Vec) -> Table { + let transaction = Transaction::new(table); + let transaction = transaction.fast_append().add_data_files(files).apply(transaction).unwrap(); + transaction.commit(self.catalog.as_ref()).await.unwrap() + } + + /// Registers table `name`: `table` plus a snapshot whose delete manifest + /// holds one position delete for the data file at `target`. Iceberg's + /// transactions cannot commit delete files, so the manifest, manifest + /// list and metadata are written by hand. + pub(crate) async fn commit_position_delete(&self, table: &Table, name: &str, target: &str) -> Table { + let metadata = table.metadata(); + let parent = metadata.current_snapshot().unwrap(); + let snapshot_id = parent.snapshot_id() + 1; + let sequence_number = metadata.last_sequence_number() + 1; + let file_io = table.file_io(); + let metadata_dir = format!("{}/metadata", metadata.location()); + + let mut delete_file = DataFileBuilder::default(); + delete_file + .content(DataContentType::PositionDeletes) + .file_path(format!("{metadata_dir}/../data/deletes.parquet")) + .file_format(DataFileFormat::Parquet) + .partition(Struct::empty()) + .record_count(1) + .file_size_in_bytes(64) + .referenced_data_file(Some(target.to_string())); + let manifest_output = file_io.new_output(format!("{metadata_dir}/deletes-{snapshot_id}.avro")).unwrap(); + let mut manifest = ManifestWriterBuilder::new( + manifest_output, + Some(snapshot_id), + Arc::clone(metadata.current_schema()), + metadata.default_partition_spec().as_ref().clone(), + ) + .build_v2_deletes(); + manifest.add_file(delete_file.build().unwrap(), sequence_number).unwrap(); + let delete_manifest = manifest.write_manifest_file().await.unwrap(); + + let existing = table.manifest_list_reader(parent).load().await.unwrap().consume_entries(); + let manifest_list = format!("{metadata_dir}/snap-{snapshot_id}.avro"); + let mut list = ManifestListWriter::v2( + file_io.new_output(&manifest_list).unwrap().writer().await.unwrap(), + snapshot_id, + Some(parent.snapshot_id()), + sequence_number, + ); + list.add_manifests(existing.into_iter().chain([delete_manifest])).unwrap(); + list.close().await.unwrap(); + + let snapshot = Snapshot::builder() + .with_snapshot_id(snapshot_id) + .with_parent_snapshot_id(Some(parent.snapshot_id())) + .with_sequence_number(sequence_number) + .with_timestamp_ms(parent.timestamp_ms() + 1) + .with_manifest_list(manifest_list) + .with_summary(Summary { operation: Operation::Delete, additional_properties: HashMap::new() }) + .with_schema_id(metadata.current_schema_id()) + .build(); + let new_metadata = TableMetadataBuilder::new_from_metadata(metadata.clone(), None) + .set_branch_snapshot(snapshot, MAIN_BRANCH) + .unwrap() + .build() + .unwrap() + .metadata; + let location = MetadataLocation::from_str(table.metadata_location().unwrap()) + .unwrap() + .with_next_version() + .try_with_new_metadata(&new_metadata) + .unwrap(); + new_metadata.write_to(file_io, &location).await.unwrap(); + self.catalog.register_table(&Self::ident(name), location.to_string()).await.unwrap() + } + + /// A path under the warehouse directory for a hand-written file. + pub(crate) fn local_path(&self, name: &str) -> String { + self.dir.path().join(name).display().to_string() + } + + /// A session with every top-level namespace of the catalog as a schema + /// of the `iceberg` catalog. + pub(crate) async fn session(&self) -> SessionContext { + let ctx = SessionContext::new(); + self.register(&ctx).await; + ctx + } + + pub(crate) async fn register(&self, ctx: &SessionContext) { + let provider = IcebergCatalogProvider::try_new(Arc::clone(&self.catalog), None).await.unwrap(); + ctx.register_catalog("iceberg", Arc::new(provider)); + } +} + +/// `id long required, name string optional, year int optional`. +pub(crate) fn people_schema() -> Schema { + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(3, "year", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap() +} + +/// A batch of `people_schema` rows with ids `ids`; `year` is the id +/// divided by 100 plus 2000. +pub(crate) fn people_batch(schema: &ArrowSchemaRef, ids: &[i64]) -> RecordBatch { + let names = ids.iter().map(|id| format!("person-{id}")).collect::>(); + let years = ids.iter().map(|id| 2000 + i32::try_from(id / 100).unwrap()).collect::>(); + RecordBatch::try_new( + Arc::clone(schema), + vec![ + Arc::new(Int64Array::from(ids.to_vec())) as ArrayRef, + Arc::new(StringArray::from(names)) as ArrayRef, + Arc::new(Int32Array::from(years)) as ArrayRef, + ], + ) + .unwrap() +} + +/// Writes `ids` through Iceberg's Parquet writer, which records field ids +/// and column bounds. +pub(crate) async fn write_people(table: &Table, ids: &[i64]) -> Vec { + let iceberg_schema = Arc::clone(table.metadata().current_schema()); + let arrow_schema: ArrowSchemaRef = Arc::new(iceberg_schema.as_ref().try_into().unwrap()); + let rolling = RollingFileWriterBuilder::new_with_default_file_size( + ParquetWriterBuilder::new(Default::default(), iceberg_schema), + table.file_io().clone(), + DefaultLocationGenerator::new(table.metadata()).unwrap(), + // A generator numbers files from zero, so each call needs its own prefix. + DefaultFileNameGenerator::new( + format!("data-{}", FILE_PREFIX.fetch_add(1, Ordering::Relaxed)), + None, + DataFileFormat::Parquet, + ), + ); + let mut writer = DataFileWriterBuilder::new(rolling).build(None).await.unwrap(); + writer.write(people_batch(&arrow_schema, ids)).await.unwrap(); + writer.close().await.unwrap() +} + +/// An Arrow field with a Parquet field id. +pub(crate) fn field_with_id(name: &str, data_type: DataType, nullable: bool, id: i32) -> Field { + Field::new(name, data_type, nullable) + .with_metadata(HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), id.to_string())])) +} + +/// Writes `batch` as a Parquet file at `path` with Arrow's writer, which +/// keeps whatever field ids the batch schema carries. +pub(crate) fn write_parquet_bytes(batch: &RecordBatch) -> Vec { + let mut bytes = Vec::new(); + let mut writer = ArrowWriter::try_new(&mut bytes, batch.schema(), None).unwrap(); + writer.write(batch).unwrap(); + writer.close().unwrap(); + bytes +} + +pub(crate) fn write_parquet(path: &Path, batch: &RecordBatch) -> u64 { + let file = File::create(path).unwrap(); + let mut writer = ArrowWriter::try_new(file, batch.schema(), None).unwrap(); + writer.write(batch).unwrap(); + writer.close().unwrap(); + fs::metadata(path).unwrap().len() +} + +/// A manifest entry for a hand-written data file at `location`. +pub(crate) fn data_file(location: &str, size: u64, rows: u64) -> DataFileBuilder { + let mut builder = DataFileBuilder::default(); + builder + .content(DataContentType::Data) + .file_path(location.to_string()) + .file_format(DataFileFormat::Parquet) + .partition(Struct::empty()) + .record_count(rows) + .file_size_in_bytes(size); + builder +} + +/// `people_schema` columns as an Arrow schema whose fields carry the field +/// ids in `ids`, or none when `ids` is `None`. +pub(crate) fn people_arrow_schema(ids: Option<[i32; 3]>) -> ArrowSchemaRef { + let fields = [("id", DataType::Int64, false), ("name", DataType::Utf8, true), ("year", DataType::Int32, true)]; + Arc::new(ArrowSchema::new( + fields + .into_iter() + .enumerate() + .map(|(index, (name, data_type, nullable))| match ids { + Some(ids) => field_with_id(name, data_type, nullable, ids[index]), + None => Field::new(name, data_type, nullable), + }) + .collect::>(), + )) +} + +/// The file scan of a plan's only `DataSourceExec`. +pub(crate) fn file_scan(plan: &Arc) -> FileScanConfig { + fn find(plan: &Arc, found: &mut Vec) { + if let Some(source) = plan.downcast_ref::() + && let Some(config) = source.data_source().downcast_ref::() + { + found.push(config.clone()); + } + for child in plan.children() { + find(child, found); + } + } + let mut found = Vec::new(); + find(plan, &mut found); + assert_eq!(found.len(), 1, "the plan must hold one file scan"); + found.pop().unwrap() +} + +pub(crate) async fn plan(ctx: &SessionContext, sql: &str) -> DataFusionResult> { + ctx.sql(sql).await?.create_physical_plan().await +} + +/// The code and facts of an Algeon error. +#[derive(Debug)] +pub(crate) struct Failure { + pub(crate) code: ErrorCode, + pub(crate) facts: HashMap, +} + +impl Failure { + pub(crate) fn fact(&self, key: &str) -> Option<&str> { + self.facts.get(key).map(String::as_str) + } +} + +pub(crate) async fn plan_failure(ctx: &SessionContext, sql: &str) -> Failure { + let error = plan(ctx, sql).await.expect_err("planning must fail"); + failure(&error) +} + +pub(crate) fn failure(error: &DataFusionError) -> Failure { + let error: &AlgeonError = error.find_algeon_error().unwrap_or_else(|| panic!("not an Algeon error: {error}")); + Failure { + code: error.code(), + facts: error.facts().iter().map(|(key, value)| (key.to_string(), value.to_string())).collect(), + } +} + +pub(crate) async fn sorted_ids(ctx: &SessionContext, sql: &str) -> Vec { + let batches = ctx.sql(sql).await.unwrap().collect().await.unwrap(); + let mut ids = batches + .iter() + .flat_map(|batch| { + let column = batch.column(0).as_any().downcast_ref::().unwrap(); + column.iter().map(Option::unwrap).collect::>() + }) + .collect::>(); + ids.sort_unstable(); + ids +} diff --git a/crates/algeon-query-engine/src/chunk/tests.rs b/crates/algeon-query-engine/src/chunk/tests.rs index 6b995acf1..10dcc722d 100644 --- a/crates/algeon-query-engine/src/chunk/tests.rs +++ b/crates/algeon-query-engine/src/chunk/tests.rs @@ -14,7 +14,7 @@ use parquet::{arrow::ArrowWriter, file::properties::WriterProperties}; use query_planning::{ backend::BackendCapabilities, chunk::{ - LOCAL_PARQUET_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK, SourceChunkPlanningOptions, SourceChunkPlanningPolicy, + PARQUET_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK, SourceChunkPlanningOptions, SourceChunkPlanningPolicy, SourceTaskMemoryModel, decimal::validate_supplied_reader_decimal_types_match_footer, }, error::QueryErrorCode, @@ -174,7 +174,7 @@ fn source_chunks_bound_decoded_dictionary_strings_by_the_byte_target() { let plan = plan_local_parquet_source_chunks( NodeId::from_index_for_test(3), &source, - SourceChunkPlanningPolicy::new(Some(target), LOCAL_PARQUET_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK), + SourceChunkPlanningPolicy::new(Some(target), PARQUET_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK), ) .unwrap(); @@ -357,37 +357,37 @@ fn local_throughput_source_chunk_planning_raises_row_group_cap_for_large_local_k let policy = SourceChunkPlanningPolicy::from_memory_policy_with_mode( &source, bounded_policy(64 * 1024 * 1024 * 1024, 64 * 1024 * 1024, 1), - SourceChunkPlanningMode::LocalParquetThroughput, + SourceChunkPlanningMode::ParquetThroughput, ); - assert_eq!(policy.max_row_groups_per_chunk, LOCAL_PARQUET_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK); + assert_eq!(policy.max_row_groups_per_chunk, PARQUET_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK); assert_eq!(policy.max_source_chunk_bytes, Some(64 * 1024 * 1024)); } #[test] fn local_throughput_source_chunk_planning_uses_configured_expert_options() { let source = local_source(); - let options = SourceChunkPlanningOptions::default().with_local_parquet_throughput_max_row_groups_per_chunk(16); + let options = SourceChunkPlanningOptions::default().with_parquet_throughput_max_row_groups_per_chunk(16); let policy = SourceChunkPlanningPolicy::from_memory_policy_with_mode_and_options( &source, bounded_policy(8 * 1024 * 1024 * 1024, 64 * 1024 * 1024, 1), - SourceChunkPlanningMode::LocalParquetThroughput, + SourceChunkPlanningMode::ParquetThroughput, options, ); assert_eq!(policy.max_row_groups_per_chunk, 16); - assert_eq!(policy.source_chunk_planning_reason(), "local_parquet_throughput_admitted"); + assert_eq!(policy.source_chunk_planning_reason(), "parquet_throughput_admitted"); } #[test] fn source_chunk_planning_options_reject_zero_thresholds() { let error = SourceChunkPlanningOptions::default() - .with_local_parquet_throughput_max_row_groups_per_chunk(0) + .with_parquet_throughput_max_row_groups_per_chunk(0) .validate() .unwrap_err(); assert_eq!(error.code(), QueryErrorCode::InvalidConfiguration); - assert_eq!(error.fact_value("field"), Some("local_parquet_throughput_max_row_groups_per_chunk")); + assert_eq!(error.fact_value("field"), Some("parquet_throughput_max_row_groups_per_chunk")); assert_eq!(error.fact_value("reason"), Some("zero")); } @@ -416,11 +416,11 @@ fn local_throughput_source_chunk_planning_accepts_local_cache_reader_path() { let policy = SourceChunkPlanningPolicy::from_memory_policy_with_mode( &source, bounded_policy(64 * 1024 * 1024 * 1024, 64 * 1024 * 1024, 1), - SourceChunkPlanningMode::LocalParquetThroughput, + SourceChunkPlanningMode::ParquetThroughput, ); - assert_eq!(policy.max_row_groups_per_chunk, LOCAL_PARQUET_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK); - assert_eq!(policy.source_chunk_planning_reason(), "local_parquet_throughput_admitted"); + assert_eq!(policy.max_row_groups_per_chunk, PARQUET_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK); + assert_eq!(policy.source_chunk_planning_reason(), "parquet_throughput_admitted"); fs::remove_file(path).unwrap(); } @@ -560,11 +560,11 @@ fn local_throughput_source_chunk_planning_widens_chunks_under_a_small_grant() { let policy = SourceChunkPlanningPolicy::from_memory_policy_with_mode( &source, bounded_policy(24 * 1024 * 1024 * 1024, MemoryControlPolicy::DEFAULT_MAX_SOURCE_CHUNK_BYTES, 1), - SourceChunkPlanningMode::LocalParquetThroughput, + SourceChunkPlanningMode::ParquetThroughput, ); - assert_eq!(policy.source_chunk_planning_reason(), "local_parquet_throughput_admitted"); - assert_eq!(policy.max_row_groups_per_chunk, LOCAL_PARQUET_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK); + assert_eq!(policy.source_chunk_planning_reason(), "parquet_throughput_admitted"); + assert_eq!(policy.max_row_groups_per_chunk, PARQUET_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK); } #[test] @@ -573,7 +573,7 @@ fn local_throughput_source_chunk_planning_requires_represented_source_read_bytes let policy = SourceChunkPlanningPolicy::from_memory_policy_with_mode( &source, bounded_policy(64 * 1024 * 1024 * 1024, 64 * 1024 * 1024, 1), - SourceChunkPlanningMode::LocalParquetThroughput, + SourceChunkPlanningMode::ParquetThroughput, ); assert_eq!(policy.max_row_groups_per_chunk, 1); @@ -593,7 +593,7 @@ fn local_throughput_source_chunk_planning_rejects_object_store_sources() { SourceChunkPlanningPolicy::from_memory_policy_with_mode( &object_store, policy, - SourceChunkPlanningMode::LocalParquetThroughput, + SourceChunkPlanningMode::ParquetThroughput, ) .max_row_groups_per_chunk, 1 @@ -710,7 +710,7 @@ fn local_throughput_source_chunk_planning_rejects_partitioned_and_scan_limit_sou SourceChunkPlanningPolicy::from_memory_policy_with_mode( &partitioned, policy, - SourceChunkPlanningMode::LocalParquetThroughput, + SourceChunkPlanningMode::ParquetThroughput, ) .max_row_groups_per_chunk, 1 @@ -719,7 +719,7 @@ fn local_throughput_source_chunk_planning_rejects_partitioned_and_scan_limit_sou SourceChunkPlanningPolicy::from_memory_policy_with_mode( &scan_limited, policy, - SourceChunkPlanningMode::LocalParquetThroughput, + SourceChunkPlanningMode::ParquetThroughput, ) .max_row_groups_per_chunk, 1 diff --git a/crates/algeon-query-engine/src/exec/bounded/aggregate.rs b/crates/algeon-query-engine/src/exec/bounded/aggregate.rs index 1e4bcf8b8..32f387c84 100644 --- a/crates/algeon-query-engine/src/exec/bounded/aggregate.rs +++ b/crates/algeon-query-engine/src/exec/bounded/aggregate.rs @@ -478,10 +478,10 @@ mod tests { fn throughput_like_policy() -> SourceChunkPlanningPolicy { let mut policy = SourceChunkPlanningPolicy::new(Some(32 * 1024 * 1024 * 1024), 1024); - // Mirror LocalParquetThroughput widening without needing a real source. + // Mirror ParquetThroughput widening without needing a real source. policy.max_row_groups_per_chunk = 1024; - policy.mark_downstream_cap("local_parquet_throughput_admitted"); - let _ = SourceChunkPlanningMode::LocalParquetThroughput; + policy.mark_downstream_cap("parquet_throughput_admitted"); + let _ = SourceChunkPlanningMode::ParquetThroughput; policy } diff --git a/crates/algeon-query-engine/tests/native_execution_tests_gpu/aggregate_execution/grouped.rs b/crates/algeon-query-engine/tests/native_execution_tests_gpu/aggregate_execution/grouped.rs index 34a2446d5..e84b29a0a 100644 --- a/crates/algeon-query-engine/tests/native_execution_tests_gpu/aggregate_execution/grouped.rs +++ b/crates/algeon-query-engine/tests/native_execution_tests_gpu/aggregate_execution/grouped.rs @@ -2211,7 +2211,7 @@ fn bounded_native_query_sizes_throughput_chunks_by_decoded_dictionary_strings() max_retry_attempts: MemoryControlPolicy::DEFAULT_MAX_RETRY_ATTEMPTS, min_retry_chunk_row_groups: MemoryControlPolicy::DEFAULT_MIN_RETRY_CHUNK_ROW_GROUPS, }) - .with_source_chunk_planning_mode(SourceChunkPlanningMode::LocalParquetThroughput); + .with_source_chunk_planning_mode(SourceChunkPlanningMode::ParquetThroughput); let result = execute_query_plan(&context, &plan).unwrap(); let metrics = result.metrics(); assert!(metrics.source_chunks_planned > 2, "decoded sizing must split each file: {metrics:?}"); diff --git a/crates/algeon-query-planning/src/chunk/mod.rs b/crates/algeon-query-planning/src/chunk/mod.rs index b2f3604fc..92240675d 100644 --- a/crates/algeon-query-planning/src/chunk/mod.rs +++ b/crates/algeon-query-planning/src/chunk/mod.rs @@ -38,9 +38,7 @@ pub use model::{SourceChunk, SourceChunkBatch, SourceChunkPlan}; pub use planner::{ annotate_source_chunk_batching_reasons, plan_source_chunk_batches, plan_source_chunk_batches_with_row_cap, }; -pub use policy::{ - LOCAL_PARQUET_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK, SourceChunkPlanningOptions, SourceChunkPlanningPolicy, -}; +pub use policy::{PARQUET_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK, SourceChunkPlanningOptions, SourceChunkPlanningPolicy}; /// What the execution backend contributes to exact source-read sizing. /// diff --git a/crates/algeon-query-planning/src/chunk/policy.rs b/crates/algeon-query-planning/src/chunk/policy.rs index 11129476c..0c16b0bc0 100644 --- a/crates/algeon-query-planning/src/chunk/policy.rs +++ b/crates/algeon-query-planning/src/chunk/policy.rs @@ -5,53 +5,50 @@ use crate::{ source::{LocalParquetFile, LocalParquetSource, ParquetReadStrategy}, }; -/// Row-group cap used by local-throughput source planning when the configured +/// Row-group cap used by throughput source planning when the configured /// policy is the conservative one-row-group shape. It bounds how much source /// one chunk retry reads again; the source-byte target sizes the chunk. -pub const LOCAL_PARQUET_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK: usize = - SourceChunkPlanningOptions::DEFAULT_LOCAL_PARQUET_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK; +pub const PARQUET_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK: usize = + SourceChunkPlanningOptions::DEFAULT_PARQUET_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK; -/// Expert policy for local Parquet source chunk planning heuristics. +/// Expert policy for Parquet source chunk planning heuristics. /// -/// This controls when [`SourceChunkPlanningMode::LocalParquetThroughput`] may +/// This controls when [`SourceChunkPlanningMode::ParquetThroughput`] may /// widen conservative one-row-group chunks. It is intentionally a Rust-only /// policy until production evidence shows a stable tuning model. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[non_exhaustive] pub struct SourceChunkPlanningOptions { - local_parquet_throughput_max_row_groups_per_chunk: usize, + parquet_throughput_max_row_groups_per_chunk: usize, } impl SourceChunkPlanningOptions { - pub(crate) const DEFAULT_LOCAL_PARQUET_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK: usize = 1024; + pub(crate) const DEFAULT_PARQUET_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK: usize = 1024; - pub(crate) fn local_parquet_throughput_max_row_groups_per_chunk(&self) -> usize { - self.local_parquet_throughput_max_row_groups_per_chunk + pub(crate) fn parquet_throughput_max_row_groups_per_chunk(&self) -> usize { + self.parquet_throughput_max_row_groups_per_chunk } - /// Sets the row-group cap available when local-throughput planning widens a + /// Sets the row-group cap available when throughput planning widens a /// conservative chunk. The value must be nonzero; call [`Self::validate`] /// after changing options. - pub fn with_local_parquet_throughput_max_row_groups_per_chunk(mut self, row_groups: usize) -> Self { - self.local_parquet_throughput_max_row_groups_per_chunk = row_groups; + pub fn with_parquet_throughput_max_row_groups_per_chunk(mut self, row_groups: usize) -> Self { + self.parquet_throughput_max_row_groups_per_chunk = row_groups; self } /// Rejects a zero row-group limit with [`QueryErrorCode::InvalidConfiguration`]. pub fn validate(&self) -> QueryResult<()> { validate_source_chunk_planning_nonzero_usize( - "local_parquet_throughput_max_row_groups_per_chunk", - self.local_parquet_throughput_max_row_groups_per_chunk, + "parquet_throughput_max_row_groups_per_chunk", + self.parquet_throughput_max_row_groups_per_chunk, ) } } impl Default for SourceChunkPlanningOptions { fn default() -> Self { - Self { - local_parquet_throughput_max_row_groups_per_chunk: - Self::DEFAULT_LOCAL_PARQUET_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK, - } + Self { parquet_throughput_max_row_groups_per_chunk: Self::DEFAULT_PARQUET_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK } } } @@ -138,7 +135,7 @@ impl SourceChunkPlanningPolicy { policy } - /// Derives a source-task policy with explicit local-throughput planning options. + /// Derives a source-task policy with explicit throughput planning options. pub fn for_source_task_with_mode_and_options( source: &LocalParquetSource, memory_policy: MemoryControlPolicy, @@ -163,7 +160,7 @@ impl SourceChunkPlanningPolicy { ) } - /// Derives a source policy with explicit local-throughput planning options. + /// Derives a source policy with explicit throughput planning options. pub fn from_memory_policy_with_mode_and_options( source: &LocalParquetSource, memory_policy: MemoryControlPolicy, @@ -183,17 +180,17 @@ impl SourceChunkPlanningPolicy { mode: SourceChunkPlanningMode, options: SourceChunkPlanningOptions, ) -> Self { - if mode != SourceChunkPlanningMode::LocalParquetThroughput { + if mode != SourceChunkPlanningMode::ParquetThroughput { return self; } self.source_chunk_planning_mode = mode; - if !local_parquet_throughput_planning_admitted(source, self) { - self.source_chunk_planning_reason = "local_parquet_throughput_rejected"; + if !parquet_throughput_planning_admitted(source, self) { + self.source_chunk_planning_reason = "parquet_throughput_rejected"; return self; } self.max_row_groups_per_chunk = - self.max_row_groups_per_chunk.max(options.local_parquet_throughput_max_row_groups_per_chunk()); - self.source_chunk_planning_reason = "local_parquet_throughput_admitted"; + self.max_row_groups_per_chunk.max(options.parquet_throughput_max_row_groups_per_chunk()); + self.source_chunk_planning_reason = "parquet_throughput_admitted"; self } } @@ -231,7 +228,7 @@ impl From for SourceChunkPlanningPolicy { /// Whether throughput planning may raise a one-row-group cap for `source`. /// The grant does not enter here: the source-byte target already scales the /// widened chunks with it. -fn local_parquet_throughput_planning_admitted(source: &LocalParquetSource, policy: SourceChunkPlanningPolicy) -> bool { +fn parquet_throughput_planning_admitted(source: &LocalParquetSource, policy: SourceChunkPlanningPolicy) -> bool { if policy.max_row_groups_per_chunk != 1 || policy.max_source_chunk_bytes.is_none() { return false; } @@ -244,25 +241,24 @@ fn local_parquet_throughput_planning_admitted(source: &LocalParquetSource, polic return false; } source.files.iter().all(|file| { - local_parquet_throughput_file_uses_supported_reader_path(file) + parquet_throughput_file_uses_supported_reader_path(file) && file.partition_values.is_empty() - && matches!( - file.resolved_source.read_strategy(), - ParquetReadStrategy::LocalKvikio | ParquetReadStrategy::LocalCacheKvikio - ) - && local_parquet_throughput_file_has_source_read_bytes(source, file) + && parquet_throughput_file_has_source_read_bytes(source, file) }) } -fn local_parquet_throughput_file_uses_supported_reader_path(file: &LocalParquetFile) -> bool { +/// A remote file qualifies like a local one: one chunk is one reader call +/// over its row groups, and the reader fetches their byte ranges in parallel. +/// Both need the row-group byte metadata that planning reads from the footer. +fn parquet_throughput_file_uses_supported_reader_path(file: &LocalParquetFile) -> bool { match file.resolved_source.read_strategy() { ParquetReadStrategy::LocalKvikio => file.location.is_local_path(), ParquetReadStrategy::LocalCacheKvikio => file.path.is_file(), - ParquetReadStrategy::RemoteDirect => false, + ParquetReadStrategy::RemoteDirect => !file.location.is_local_path(), } } -fn local_parquet_throughput_file_has_source_read_bytes(source: &LocalParquetSource, file: &LocalParquetFile) -> bool { +fn parquet_throughput_file_has_source_read_bytes(source: &LocalParquetSource, file: &LocalParquetFile) -> bool { let selected_row_groups = match &file.row_groups { Some(row_groups) if !row_groups.is_empty() => row_groups.clone(), Some(_) => return false, @@ -285,3 +281,51 @@ fn local_parquet_throughput_file_has_source_read_bytes(source: &LocalParquetSour || file.pure_filter_row_group_uncompressed_bytes.as_ref().is_some_and(|bytes| max_row_group < bytes.len()); projected_covers_selection && pure_filter_covers_selection } + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow::datatypes::{DataType, Field, Schema}; + + use super::*; + use crate::source::{CredentialMode, ResolvedParquetSource}; + + fn one_row_group_policy() -> MemoryControlPolicy { + MemoryControlPolicy { + device_budget_bytes: 64 * 1024 * 1024 * 1024, + max_source_chunk_bytes: 64 * 1024 * 1024, + max_row_groups_per_chunk: 1, + max_retry_attempts: 3, + min_retry_chunk_row_groups: 1, + } + } + + fn remote_source(file: LocalParquetFile) -> LocalParquetSource { + let schema = Arc::new(Schema::new(vec![Field::new("c0", DataType::Int64, true)])); + LocalParquetSource::from_files(vec![file], schema, vec!["c0".to_string()]) + } + + fn remote_file() -> LocalParquetFile { + let resolved = + ResolvedParquetSource::remote_kvikio("s3://bucket/table/part-0.parquet", CredentialMode::SharedProcessEnv); + LocalParquetFile::from_resolved_parquet_source(resolved).unwrap().with_stable_id("part-0") + } + + #[test] + fn throughput_planning_widens_remote_chunks_when_the_footer_supplied_row_group_bytes() { + let file = remote_file() + .with_row_group_bytes(vec![1024; 7]) + .with_projected_row_group_bytes(vec![512; 7], vec![1024; 7]) + .with_row_group_rows(vec![1; 7]); + + let policy = SourceChunkPlanningPolicy::from_memory_policy_with_mode( + &remote_source(file), + one_row_group_policy(), + SourceChunkPlanningMode::ParquetThroughput, + ); + + assert_eq!(policy.source_chunk_planning_reason(), "parquet_throughput_admitted"); + assert_eq!(policy.max_row_groups_per_chunk, PARQUET_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK); + } +} diff --git a/crates/algeon-query-planning/src/execution_contracts.rs b/crates/algeon-query-planning/src/execution_contracts.rs index f178a2e91..71b9500ff 100644 --- a/crates/algeon-query-planning/src/execution_contracts.rs +++ b/crates/algeon-query-planning/src/execution_contracts.rs @@ -68,12 +68,12 @@ pub enum SourceChunkPlanningMode { /// Each chunk is then the smallest retry unit, but per-chunk work such as /// partial aggregation grows with the table's row-group count. Conservative, - /// Raises a one-row-group cap so a chunk of a local Parquet source can hold - /// many row groups up to the source-byte target; `crate::chunk` owns the - /// row-group cap and the source conditions. A source that does not qualify - /// keeps conservative chunks. + /// Raises a one-row-group cap so a chunk of a Parquet source, local or + /// remote, can hold many row groups up to the source-byte target; + /// `crate::chunk` owns the row-group cap and the source conditions. A + /// source that does not qualify keeps conservative chunks. #[default] - LocalParquetThroughput, + ParquetThroughput, } impl SourceChunkPlanningMode { @@ -81,7 +81,7 @@ impl SourceChunkPlanningMode { pub const fn as_str(self) -> &'static str { match self { Self::Conservative => "conservative", - Self::LocalParquetThroughput => "local_parquet_throughput", + Self::ParquetThroughput => "parquet_throughput", } } } diff --git a/crates/algeon-query-planning/src/source/diagnostics.rs b/crates/algeon-query-planning/src/source/diagnostics.rs index e65ecf2c2..ca1727b40 100644 --- a/crates/algeon-query-planning/src/source/diagnostics.rs +++ b/crates/algeon-query-planning/src/source/diagnostics.rs @@ -110,6 +110,18 @@ pub struct SourceDiagnostics { } impl SourceDiagnostics { + /// Sets the read strategy, credential mode, and byte-cache labels to the + /// ones a source over the resolved `files` reports, and direct device read + /// to whether those files prove it and no feature was rejected. + pub fn set_resolved_read_labels(&mut self, files: &[LocalParquetFile]) { + self.source_read_strategy = summarize_source_read_strategy(files); + self.credential_mode = summarize_credential_mode(files); + self.lakehouse_cache_status = summarize_lakehouse_cache_status(files); + self.lakehouse_cache_read_mode = summarize_lakehouse_cache_read_mode(files); + self.lakehouse_cache_fill_mode = summarize_lakehouse_cache_fill_mode(files); + self.direct_device_read = self.rejected_features.is_empty() && files_have_proven_direct_device_read(files); + } + pub(super) fn for_local_parquet_files( files: &[LocalParquetFile], columns: LocalParquetDiagnosticColumns<'_>, @@ -622,6 +634,8 @@ pub enum RejectedSourceFeature { ZeroColumnCount, /// Logical-to-source column mapping is invalid. SchemaMapping, + /// A remote object's footer was not available to planning. + RemoteFooterUnavailable, } impl RejectedSourceFeature { @@ -643,6 +657,7 @@ impl RejectedSourceFeature { Self::RowSelection => "row_selection", Self::ZeroColumnCount => "zero_column_count", Self::SchemaMapping => "schema_mapping", + Self::RemoteFooterUnavailable => "remote_footer_unavailable", } } } diff --git a/crates/algeon-query-runtime/src/runtime/context/mod.rs b/crates/algeon-query-runtime/src/runtime/context/mod.rs index daa6c8dc3..459976a8e 100644 --- a/crates/algeon-query-runtime/src/runtime/context/mod.rs +++ b/crates/algeon-query-runtime/src/runtime/context/mod.rs @@ -714,7 +714,7 @@ impl ExecutionContext { self.device.planning_policy } - /// Returns expert local-throughput source chunk planning policy. + /// Returns expert throughput source chunk planning policy. pub fn source_chunk_planning_options(&self) -> SourceChunkPlanningOptions { self.device.source_chunk_planning_options } diff --git a/crates/algeon-query-runtime/src/runtime/context/standalone.rs b/crates/algeon-query-runtime/src/runtime/context/standalone.rs index 9f0a1d124..be4b0582c 100644 --- a/crates/algeon-query-runtime/src/runtime/context/standalone.rs +++ b/crates/algeon-query-runtime/src/runtime/context/standalone.rs @@ -65,7 +65,7 @@ pub struct StandaloneConfig { pub memory_control_policy: MemoryControlPolicy, pub source_scheduler_policy: SourceSchedulerPolicy, pub source_chunk_planning_mode: SourceChunkPlanningMode, - /// Expert policy for local-throughput source chunk planning. + /// Expert policy for throughput source chunk planning. pub source_chunk_planning_options: SourceChunkPlanningOptions, /// Object-store profiles used by remote reads and cache fills. pub source_storage_profiles: Vec, diff --git a/crates/algeon-query-runtime/src/runtime/query_service/config.rs b/crates/algeon-query-runtime/src/runtime/query_service/config.rs index 76ecc4c24..1eae7348f 100644 --- a/crates/algeon-query-runtime/src/runtime/query_service/config.rs +++ b/crates/algeon-query-runtime/src/runtime/query_service/config.rs @@ -83,7 +83,7 @@ pub struct QueryExecutionConfig { pub planning_policy: PlanningPolicy, /// Grouped aggregate strategy execution policy copied into admitted contexts. pub aggregate_strategy_execution_options: AggregateStrategyExecutionOptions, - /// Expert local-throughput source chunk planning policy copied into admitted contexts. + /// Expert throughput source chunk planning policy copied into admitted contexts. pub source_chunk_planning_options: SourceChunkPlanningOptions, /// Source chunk byte target copied into admitted bounded contexts. /// `u64::MAX` derives a finite target from the immutable grant. diff --git a/crates/algeon-query-runtime/src/runtime/query_service/tests/config_resources.rs b/crates/algeon-query-runtime/src/runtime/query_service/tests/config_resources.rs index e93f62364..cb1a2efd8 100644 --- a/crates/algeon-query-runtime/src/runtime/query_service/tests/config_resources.rs +++ b/crates/algeon-query-runtime/src/runtime/query_service/tests/config_resources.rs @@ -26,7 +26,7 @@ fn query_service_handle_exposes_configured_execution_policies() { let aggregate_strategy_execution_options = AggregateStrategyExecutionOptions::default().with_encoded_key_partial_min_input_rows(22); let source_chunk_planning_options = - SourceChunkPlanningOptions::default().with_local_parquet_throughput_max_row_groups_per_chunk(33); + SourceChunkPlanningOptions::default().with_parquet_throughput_max_row_groups_per_chunk(33); let config = QueryExecutionConfig { source_scheduler_policy: SourceSchedulerPolicy::new(4), source_chunk_planning_mode: SourceChunkPlanningMode::Conservative, @@ -104,14 +104,14 @@ fn query_service_rejects_invalid_aggregate_strategy_execution_policy() { fn query_service_rejects_invalid_source_chunk_planning_policy() { let config = QueryExecutionConfig { source_chunk_planning_options: SourceChunkPlanningOptions::default() - .with_local_parquet_throughput_max_row_groups_per_chunk(0), + .with_parquet_throughput_max_row_groups_per_chunk(0), ..QueryExecutionConfig::default() }; let error = config.validate().unwrap_err(); assert_eq!(error.code(), QueryErrorCode::InvalidConfiguration); - assert_eq!(error.fact_value("field"), Some("local_parquet_throughput_max_row_groups_per_chunk")); + assert_eq!(error.fact_value("field"), Some("parquet_throughput_max_row_groups_per_chunk")); assert_eq!(error.fact_value("reason"), Some("zero")); } #[test] diff --git a/crates/algeon-query-runtime/src/runtime/tests.rs b/crates/algeon-query-runtime/src/runtime/tests.rs index 4379ee3a8..72af77a58 100644 --- a/crates/algeon-query-runtime/src/runtime/tests.rs +++ b/crates/algeon-query-runtime/src/runtime/tests.rs @@ -353,13 +353,13 @@ fn cache_fill_byte_budget_rejects_oversized_fetches() { fn execution_context_rejects_invalid_source_chunk_planning_options() { let error = ExecutionContext::standalone(StandaloneConfig { source_chunk_planning_options: SourceChunkPlanningOptions::default() - .with_local_parquet_throughput_max_row_groups_per_chunk(0), + .with_parquet_throughput_max_row_groups_per_chunk(0), ..StandaloneConfig::default() }) .unwrap_err(); assert_eq!(error.code(), QueryErrorCode::InvalidConfiguration); - assert_eq!(error.fact_value("field"), Some("local_parquet_throughput_max_row_groups_per_chunk")); + assert_eq!(error.fact_value("field"), Some("parquet_throughput_max_row_groups_per_chunk")); } #[test] diff --git a/crates/algeon-server/Cargo.toml b/crates/algeon-server/Cargo.toml index d3f9cca60..f42b8bc3e 100644 --- a/crates/algeon-server/Cargo.toml +++ b/crates/algeon-server/Cargo.toml @@ -16,6 +16,7 @@ default = [] cugraph = ["algeon-datafusion/cugraph", "dep:cugraph-sys"] cuvs = ["algeon-datafusion/cuvs", "dep:cuvs-sys"] cuml = ["algeon-datafusion/cuml", "dep:cuml-sys"] +iceberg = ["dep:algeon-iceberg"] nvml = ["algeon-datafusion/nvml"] [lints] @@ -23,6 +24,7 @@ workspace = true [dependencies] algeon-datafusion = { workspace = true, default-features = false, features = ["server"] } +algeon-iceberg = { workspace = true, optional = true } # Keep these direct sys edges so build.rs receives runtime link metadata. cudf-sys = { workspace = true } cugraph-sys = { workspace = true, optional = true } diff --git a/crates/algeon-server/src/bin/algeon_server.rs b/crates/algeon-server/src/bin/algeon_server.rs index a39864763..c7d4a9c17 100644 --- a/crates/algeon-server/src/bin/algeon_server.rs +++ b/crates/algeon-server/src/bin/algeon_server.rs @@ -1,5 +1,7 @@ //! Standalone Arrow Flight SQL server binary for `algeon-datafusion`. +#[cfg(feature = "iceberg")] +use std::sync::Arc; use std::{ env, error::Error as StdError, @@ -7,6 +9,8 @@ use std::{ process::ExitCode, }; +#[cfg(feature = "iceberg")] +use algeon_datafusion::server::{backend_from_server_config, build_session_context_with_backend}; use algeon_datafusion::{ backend::GpuMemoryOwnership, build_info::build_info, @@ -113,9 +117,17 @@ fn main() -> ExitCode { } } +/// Optional features of this binary beyond the library's engine features. +const SERVER_FEATURES: &[&str] = &[ + #[cfg(feature = "iceberg")] + "iceberg", +]; + fn print_version() { let build = build_info(); - let features = if build.features.is_empty() { "none".to_string() } else { build.features.join(",") }; + let mut features = build.features.iter().chain(SERVER_FEATURES).copied().collect::>(); + features.sort_unstable(); + let features = if features.is_empty() { "none".to_string() } else { features.join(",") }; println!("algeon_server {}", build.version); println!("commit: {}", build.git_commit.unwrap_or("unknown")); println!("features: {features}"); @@ -213,7 +225,26 @@ async fn serve(mut loaded: LoadedServerConfig) -> Result<(), ServerError> { .map(|ordinal| (ordinal, GpuMemoryOwnership::WholeDeviceExclusive)) .collect::>(); loaded.config = loaded.config.with_gpu_memory_ownership(ownership); - FlightSqlServer::builder().loaded_config(loaded).build().await?.serve().await + let builder = FlightSqlServer::builder(); + #[cfg(feature = "iceberg")] + let builder = { + // Iceberg catalogs register on the shared session, which each + // statement session clones, so the launcher builds that session. + let backend = backend_from_server_config(&loaded.config)?; + let ctx = build_session_context_with_backend(&loaded.config, &backend).await?; + algeon_iceberg::register_iceberg_catalogs(&ctx, &loaded.config) + .await + .map_err(|error| ServerError::session("failed to register the Iceberg catalogs", error))?; + builder.backend(backend).session_context(Arc::new(ctx)) + }; + #[cfg(not(feature = "iceberg"))] + if !loaded.config.iceberg_catalogs.is_empty() { + return Err(ServerError::invalid_config( + "iceberg.catalogs", + "needs a server built with the `iceberg` feature; this binary was built without it", + )); + } + builder.loaded_config(loaded).build().await?.serve().await } /// Configuration error fields emitted as dedicated tracing attributes rather diff --git a/crates/bench/src/clickbench_benchmark/mod.rs b/crates/bench/src/clickbench_benchmark/mod.rs index a265ac8b5..ee113a309 100644 --- a/crates/bench/src/clickbench_benchmark/mod.rs +++ b/crates/bench/src/clickbench_benchmark/mod.rs @@ -1020,7 +1020,7 @@ mod tests { assert!(tsv.contains("# clickbench_benchmark_native_profile_source=env_profile\n")); assert!(tsv.contains("# clickbench_benchmark_native_max_source_chunk_bytes=34359738368\n")); assert!(tsv.contains("# clickbench_benchmark_native_max_row_groups_per_chunk=1024\n")); - assert!(tsv.contains("# clickbench_benchmark_native_source_chunk_planning_mode=local_parquet_throughput\n")); + assert!(tsv.contains("# clickbench_benchmark_native_source_chunk_planning_mode=parquet_throughput\n")); assert!(tsv.contains( "# clickbench_benchmark_kvikio_compat_mode=on\n# clickbench_benchmark_kvikio_threads=64\n# clickbench_benchmark_kvikio_task_size_bytes=4194304\n" )); @@ -1090,7 +1090,7 @@ mod tests { max_row_groups_per_chunk: CLICKBENCH_GPU_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK, max_retry_attempts: SourceChunking::DEFAULT_MAX_RETRY_ATTEMPTS.get(), min_retry_chunk_row_groups: SourceChunking::DEFAULT_MIN_RETRY_CHUNK_ROW_GROUPS.get(), - source_chunk_planning_mode: SourceChunkPlanningMode::LocalParquetThroughput, + source_chunk_planning_mode: SourceChunkPlanningMode::ParquetThroughput, }, progress: false, dump_rows_dir: None, diff --git a/crates/bench/src/report_workload/clickbench.rs b/crates/bench/src/report_workload/clickbench.rs index 5a83ca4dd..97a93b2db 100644 --- a/crates/bench/src/report_workload/clickbench.rs +++ b/crates/bench/src/report_workload/clickbench.rs @@ -46,7 +46,7 @@ pub const ALGEON_CLICKBENCH_NATIVE_MAX_RETRY_ATTEMPTS_ENV: &str = "ALGEON_CLICKB pub const ALGEON_CLICKBENCH_NATIVE_MIN_RETRY_CHUNK_ROW_GROUPS_ENV: &str = "ALGEON_CLICKBENCH_NATIVE_MIN_RETRY_CHUNK_ROW_GROUPS"; /// Source chunk planning mode used by bounded native ClickBench execution. -/// Accepts `conservative` or `local_parquet_throughput`; defaults to the +/// Accepts `conservative` or `parquet_throughput`; defaults to the /// selected profile. pub const ALGEON_CLICKBENCH_NATIVE_SOURCE_CHUNK_PLANNING_MODE_ENV: &str = "ALGEON_CLICKBENCH_NATIVE_SOURCE_CHUNK_PLANNING_MODE"; @@ -291,7 +291,7 @@ fn native_profile_base(profile: Option<&str>) -> NativeProfileBase { max_row_groups_per_chunk: CLICKBENCH_GPU_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK, max_retry_attempts: SourceChunking::DEFAULT_MAX_RETRY_ATTEMPTS.get(), min_retry_chunk_row_groups: SourceChunking::DEFAULT_MIN_RETRY_CHUNK_ROW_GROUPS.get(), - source_chunk_planning_mode: SourceChunkPlanningMode::LocalParquetThroughput, + source_chunk_planning_mode: SourceChunkPlanningMode::ParquetThroughput, }, other => panic!( "unsupported {ALGEON_CLICKBENCH_NATIVE_PROFILE_ENV}={other}; expected {CLICKBENCH_NATIVE_PROFILE_CONSERVATIVE} or {CLICKBENCH_NATIVE_PROFILE_GPU_THROUGHPUT}" @@ -323,9 +323,9 @@ fn parse_env_usize(name: &'static str, value: String) -> usize { fn parse_env_source_chunk_planning_mode(value: String) -> SourceChunkPlanningMode { match value.trim().to_ascii_lowercase().as_str() { "conservative" => SourceChunkPlanningMode::Conservative, - "local_parquet_throughput" | "local_throughput" => SourceChunkPlanningMode::LocalParquetThroughput, + "parquet_throughput" => SourceChunkPlanningMode::ParquetThroughput, other => panic!( - "unsupported {ALGEON_CLICKBENCH_NATIVE_SOURCE_CHUNK_PLANNING_MODE_ENV}={other}; expected conservative or local_parquet_throughput" + "unsupported {ALGEON_CLICKBENCH_NATIVE_SOURCE_CHUNK_PLANNING_MODE_ENV}={other}; expected conservative or parquet_throughput" ), } } diff --git a/crates/bench/src/report_workload/tpcds.rs b/crates/bench/src/report_workload/tpcds.rs index 8ab9c25a6..ff4ed3f5d 100644 --- a/crates/bench/src/report_workload/tpcds.rs +++ b/crates/bench/src/report_workload/tpcds.rs @@ -54,7 +54,7 @@ pub const ALGEON_TPCDS_NATIVE_MAX_RETRY_ATTEMPTS_ENV: &str = "ALGEON_TPCDS_NATIV /// Minimum row-group granularity used when retrying with smaller chunks. pub const ALGEON_TPCDS_NATIVE_MIN_RETRY_CHUNK_ROW_GROUPS_ENV: &str = "ALGEON_TPCDS_NATIVE_MIN_RETRY_CHUNK_ROW_GROUPS"; /// Source chunk planning mode used by bounded native TPC-DS execution. Accepts -/// `conservative` or `local_parquet_throughput`; defaults to the selected +/// `conservative` or `parquet_throughput`; defaults to the selected /// profile. pub const ALGEON_TPCDS_NATIVE_SOURCE_CHUNK_PLANNING_MODE_ENV: &str = "ALGEON_TPCDS_NATIVE_SOURCE_CHUNK_PLANNING_MODE"; /// Execution contract for native floating statistical aggregates. Accepts @@ -389,7 +389,7 @@ fn native_profile_base(profile: Option<&str>) -> NativeProfileBase { max_row_groups_per_chunk: TPCDS_SF100_GPU_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK, max_retry_attempts: SourceChunking::DEFAULT_MAX_RETRY_ATTEMPTS.get(), min_retry_chunk_row_groups: SourceChunking::DEFAULT_MIN_RETRY_CHUNK_ROW_GROUPS.get(), - source_chunk_planning_mode: SourceChunkPlanningMode::LocalParquetThroughput, + source_chunk_planning_mode: SourceChunkPlanningMode::ParquetThroughput, }, other => panic!( "unsupported {ALGEON_TPCDS_NATIVE_PROFILE_ENV}={other}; expected {TPCDS_NATIVE_PROFILE_CONSERVATIVE} or {TPCDS_NATIVE_PROFILE_SF100_GPU_THROUGHPUT}" @@ -421,9 +421,9 @@ fn parse_env_usize(name: &'static str, value: String) -> usize { fn parse_env_source_chunk_planning_mode(value: String) -> SourceChunkPlanningMode { match value.trim().to_ascii_lowercase().as_str() { "conservative" => SourceChunkPlanningMode::Conservative, - "local_parquet_throughput" | "local_throughput" => SourceChunkPlanningMode::LocalParquetThroughput, + "parquet_throughput" => SourceChunkPlanningMode::ParquetThroughput, other => panic!( - "unsupported {ALGEON_TPCDS_NATIVE_SOURCE_CHUNK_PLANNING_MODE_ENV}={other}; expected conservative or local_parquet_throughput" + "unsupported {ALGEON_TPCDS_NATIVE_SOURCE_CHUNK_PLANNING_MODE_ENV}={other}; expected conservative or parquet_throughput" ), } } diff --git a/crates/bench/src/tpch_benchmark/config.rs b/crates/bench/src/tpch_benchmark/config.rs index a63719758..af9a7a88c 100644 --- a/crates/bench/src/tpch_benchmark/config.rs +++ b/crates/bench/src/tpch_benchmark/config.rs @@ -527,7 +527,7 @@ fn native_tpch_profile_base(profile: Option<&str>) -> DataFusionResult Err(plan_error(format!( "invalid {ALGEON_TPCH_BENCH_NATIVE_PROFILE_ENV}: {other}; expected {TPCH_NATIVE_PROFILE_CONSERVATIVE} or {TPCH_NATIVE_PROFILE_SF100_GPU_THROUGHPUT}" @@ -617,9 +617,9 @@ fn parse_env_u64_value(name: &'static str, value: String) -> DataFusionResult DataFusionResult { match value.trim().to_ascii_lowercase().as_str() { "conservative" => Ok(SourceChunkPlanningMode::Conservative), - "local_parquet_throughput" | "local_throughput" => Ok(SourceChunkPlanningMode::LocalParquetThroughput), + "parquet_throughput" => Ok(SourceChunkPlanningMode::ParquetThroughput), other => Err(plan_error(format!( - "invalid {ALGEON_TPCH_BENCH_NATIVE_SOURCE_CHUNK_PLANNING_MODE_ENV}: {other}; expected conservative or local_parquet_throughput" + "invalid {ALGEON_TPCH_BENCH_NATIVE_SOURCE_CHUNK_PLANNING_MODE_ENV}: {other}; expected conservative or parquet_throughput" ))), } } diff --git a/crates/bench/src/tpch_benchmark/tests/config_dataset.rs b/crates/bench/src/tpch_benchmark/tests/config_dataset.rs index 4469992ab..901aee92c 100644 --- a/crates/bench/src/tpch_benchmark/tests/config_dataset.rs +++ b/crates/bench/src/tpch_benchmark/tests/config_dataset.rs @@ -36,7 +36,7 @@ fn native_profile_source_chunk_planning_reaches_execution_config() { let execution = config.native_execution_config_with_resources(&resources); - assert_eq!(execution.source_chunk_planning_mode(), SourceChunkPlanningMode::LocalParquetThroughput); + assert_eq!(execution.source_chunk_planning_mode(), SourceChunkPlanningMode::ParquetThroughput); } #[test] @@ -46,7 +46,7 @@ fn native_profile_exact_overrides_resolve_to_named_profile() { (ALGEON_TPCH_BENCH_NATIVE_MAX_ROW_GROUPS_PER_CHUNK_ENV, "1024"), (ALGEON_TPCH_BENCH_NATIVE_MAX_RETRY_ATTEMPTS_ENV, "4"), (ALGEON_TPCH_BENCH_NATIVE_MIN_RETRY_CHUNK_ROW_GROUPS_ENV, "1"), - (ALGEON_TPCH_BENCH_NATIVE_SOURCE_CHUNK_PLANNING_MODE_ENV, "local_parquet_throughput"), + (ALGEON_TPCH_BENCH_NATIVE_SOURCE_CHUNK_PLANNING_MODE_ENV, "parquet_throughput"), ]) .unwrap(); @@ -70,7 +70,7 @@ fn explicit_device_budget_override_is_custom_and_explicit() { assert_eq!(profile.max_row_groups_per_chunk, TPCH_SF100_GPU_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK); assert_eq!(profile.max_retry_attempts, DEFAULT_NATIVE_MAX_RETRY_ATTEMPTS); assert_eq!(profile.min_retry_chunk_row_groups, DEFAULT_NATIVE_MIN_RETRY_CHUNK_ROW_GROUPS); - assert_eq!(profile.source_chunk_planning_mode, SourceChunkPlanningMode::LocalParquetThroughput); + assert_eq!(profile.source_chunk_planning_mode, SourceChunkPlanningMode::ParquetThroughput); assert_eq!(NativeTpchDeviceBudget::Automatic.memory_limit(), GpuDeviceMemoryLimit::default()); } @@ -113,7 +113,7 @@ fn native_profile_rejects_invalid_profile_and_source_mode() { let error = tpch_native_profile_settings_from([(ALGEON_TPCH_BENCH_NATIVE_SOURCE_CHUNK_PLANNING_MODE_ENV, "wide")]) .expect_err("unknown source chunk planning modes must be rejected"); - assert!(error.to_string().contains("expected conservative or local_parquet_throughput")); + assert!(error.to_string().contains("expected conservative or parquet_throughput")); } #[test] diff --git a/crates/bench/src/tpch_benchmark/tests/large_keyed_join_gate.rs b/crates/bench/src/tpch_benchmark/tests/large_keyed_join_gate.rs index 04e49c70e..2ded2111c 100644 --- a/crates/bench/src/tpch_benchmark/tests/large_keyed_join_gate.rs +++ b/crates/bench/src/tpch_benchmark/tests/large_keyed_join_gate.rs @@ -121,7 +121,7 @@ fn sf100_bounded_memory_policy() -> NativeTpchMemoryPolicy { 1024, DEFAULT_NATIVE_MAX_RETRY_ATTEMPTS, DEFAULT_NATIVE_MIN_RETRY_CHUNK_ROW_GROUPS, - SourceChunkPlanningMode::LocalParquetThroughput, + SourceChunkPlanningMode::ParquetThroughput, ) } diff --git a/crates/bench/src/tpch_benchmark/tests/rejected_rows.rs b/crates/bench/src/tpch_benchmark/tests/rejected_rows.rs index 731611c97..c4f220a3f 100644 --- a/crates/bench/src/tpch_benchmark/tests/rejected_rows.rs +++ b/crates/bench/src/tpch_benchmark/tests/rejected_rows.rs @@ -76,7 +76,7 @@ fn benchmark_report_metadata_identifies_named_sf100_profile() { assert!(text.contains("# tpch_benchmark_native_device_budget_bytes=automatic\n")); assert!(text.contains("# tpch_benchmark_native_max_source_chunk_bytes=34359738368\n")); assert!(text.contains("# tpch_benchmark_native_max_row_groups_per_chunk=1024\n")); - assert!(text.contains("# tpch_benchmark_native_source_chunk_planning_mode=local_parquet_throughput\n")); + assert!(text.contains("# tpch_benchmark_native_source_chunk_planning_mode=parquet_throughput\n")); } #[test] diff --git a/crates/bench/tests/clickbench_workload_tests_cpu.rs b/crates/bench/tests/clickbench_workload_tests_cpu.rs index e23055f5d..9d7e81b1f 100644 --- a/crates/bench/tests/clickbench_workload_tests_cpu.rs +++ b/crates/bench/tests/clickbench_workload_tests_cpu.rs @@ -39,12 +39,12 @@ fn clickbench_native_profile_numeric_overrides_become_custom() { fn clickbench_native_profile_source_chunk_planning_mode_override_becomes_custom() { let profile = clickbench_native_profile_settings_from([ (ALGEON_CLICKBENCH_NATIVE_PROFILE_ENV, "conservative"), - (ALGEON_CLICKBENCH_NATIVE_SOURCE_CHUNK_PLANNING_MODE_ENV, "local_parquet_throughput"), + (ALGEON_CLICKBENCH_NATIVE_SOURCE_CHUNK_PLANNING_MODE_ENV, "parquet_throughput"), ]); assert_eq!(profile.profile_name, CLICKBENCH_NATIVE_PROFILE_CUSTOM); assert_eq!(profile.profile_source, "custom_overrides"); - assert_eq!(profile.source_chunk_planning_mode, SourceChunkPlanningMode::LocalParquetThroughput); + assert_eq!(profile.source_chunk_planning_mode, SourceChunkPlanningMode::ParquetThroughput); } #[test] @@ -60,7 +60,7 @@ fn clickbench_native_profile_exact_overrides_resolve_to_named_profile() { ALGEON_CLICKBENCH_NATIVE_MAX_ROW_GROUPS_PER_CHUNK_ENV, &CLICKBENCH_GPU_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK.to_string(), ), - (ALGEON_CLICKBENCH_NATIVE_SOURCE_CHUNK_PLANNING_MODE_ENV, "local_parquet_throughput"), + (ALGEON_CLICKBENCH_NATIVE_SOURCE_CHUNK_PLANNING_MODE_ENV, "parquet_throughput"), ]); assert_eq!(profile.profile_name, CLICKBENCH_NATIVE_PROFILE_GPU_THROUGHPUT); @@ -86,7 +86,7 @@ fn clickbench_native_optimizer_config_uses_resolved_profile_knobs() { let config = clickbench_native_optimizer_config_with_profile_settings(&profile); let execution = config.native_execution_config(); - assert_eq!(execution.source_chunk_planning_mode(), SourceChunkPlanningMode::LocalParquetThroughput); + assert_eq!(execution.source_chunk_planning_mode(), SourceChunkPlanningMode::ParquetThroughput); assert_eq!( execution.source_chunking().max_source_chunk_bytes.get(), CLICKBENCH_GPU_THROUGHPUT_MAX_SOURCE_CHUNK_BYTES diff --git a/crates/bench/tests/tools/test_tpcds_native_profile_sweep.py b/crates/bench/tests/tools/test_tpcds_native_profile_sweep.py index 7074dd7d7..7adc960ee 100644 --- a/crates/bench/tests/tools/test_tpcds_native_profile_sweep.py +++ b/crates/bench/tests/tools/test_tpcds_native_profile_sweep.py @@ -86,7 +86,7 @@ def test_candidate_env_clears_ambient_profile_for_auto_budget(self) -> None: try: os.environ["ALGEON_TPCDS_NATIVE_PROFILE"] = "sf100_gpu_throughput" os.environ["ALGEON_TPCDS_NATIVE_DEVICE_BUDGET_BYTES"] = "102005473280" - os.environ["ALGEON_TPCDS_NATIVE_SOURCE_CHUNK_PLANNING_MODE"] = "local_parquet_throughput" + os.environ["ALGEON_TPCDS_NATIVE_SOURCE_CHUNK_PLANNING_MODE"] = "parquet_throughput" env = module.candidate_env(args, candidate) finally: restore_env("ALGEON_TPCDS_NATIVE_PROFILE", old_profile) diff --git a/crates/bench/tests/tools/test_tpcds_profile_metadata.py b/crates/bench/tests/tools/test_tpcds_profile_metadata.py index 769949cdb..8c0984580 100644 --- a/crates/bench/tests/tools/test_tpcds_profile_metadata.py +++ b/crates/bench/tests/tools/test_tpcds_profile_metadata.py @@ -34,7 +34,7 @@ def test_cpu_gpu_compare_reads_native_profile_comments(self) -> None: "# tpcds_benchmark_native_device_budget_bytes=102005473280", "# tpcds_benchmark_native_max_source_chunk_bytes=34359738368", "# tpcds_benchmark_native_max_row_groups_per_chunk=1024", - "# tpcds_benchmark_native_source_chunk_planning_mode=local_parquet_throughput", + "# tpcds_benchmark_native_source_chunk_planning_mode=parquet_throughput", "engine\tquery_id\titeration\twarmup\twall_ms\toutcome\toutput_rows\tnative_runtime_started", "native_gpu\tq44\t0\tfalse\t640.0\tsucceeded\t1\ttrue", ] @@ -49,7 +49,7 @@ def test_cpu_gpu_compare_reads_native_profile_comments(self) -> None: self.assertEqual(rows["q44"]["measured"], [640.0]) self.assertEqual(profile["name"], "sf100_gpu_throughput") self.assertEqual(profile["row_groups"], "1024") - self.assertEqual(profile["source_chunk_planning_mode"], "local_parquet_throughput") + self.assertEqual(profile["source_chunk_planning_mode"], "parquet_throughput") self.assertTrue(profile["known"]) if __name__ == "__main__": diff --git a/crates/tools/Cargo.toml b/crates/tools/Cargo.toml index 4e8f387fa..378e67785 100644 --- a/crates/tools/Cargo.toml +++ b/crates/tools/Cargo.toml @@ -64,6 +64,16 @@ flight-sql-stress-check = ["dep:algeon-datafusion", "dep:cudf-sys", "algeon-data # constant, to validate retained artifacts. server-runtime-observability-e2e = ["dep:query-runtime"] embedded-backend-e2e = ["dep:reqwest"] +# `s3://` Parquet scans through one S3 configuration for DataFusion and native +# reads, for a library and a server-configured backend, against real S3 or an +# S3-compatible service. +s3-parquet-e2e = ["dep:cudf-sys", "dep:datafusion", "dep:algeon-datafusion", "algeon-datafusion/server"] +# Iceberg REST and Glue catalogs served by an in-process Flight SQL server, +# against the local services of fixtures/iceberg-local. +iceberg-local-e2e = [ + "dep:cudf-sys", "dep:datafusion", "dep:algeon-datafusion", "algeon-datafusion/server", + "dep:algeon-iceberg", "dep:iceberg", "dep:tokio-util", +] embedded-backend-fixture = [ "dep:algeon-datafusion", "algeon-datafusion/server", "algeon-datafusion/cugraph", "dep:cudf-sys", "dep:cugraph-sys", "dep:datafusion", "dep:anyhow", @@ -123,12 +133,14 @@ cuvs-sys = { workspace = true, optional = true } cuml-sys = { workspace = true, optional = true } datafusion = { workspace = true, optional = true } algeon-datafusion = { workspace = true, optional = true } +algeon-iceberg = { workspace = true, optional = true } bench = { path = "../bench", optional = true } workloads = { workspace = true, optional = true } query-engine = { workspace = true, optional = true } query-planning = { workspace = true, optional = true } query-runtime = { workspace = true, optional = true } futures = { workspace = true } +iceberg = { workspace = true, optional = true } parquet = { workspace = true, features = ["arrow"] } opentelemetry = { workspace = true, optional = true } opentelemetry-proto = { workspace = true, optional = true } diff --git a/crates/tools/README.md b/crates/tools/README.md index 154c545f4..dc9726ebe 100644 --- a/crates/tools/README.md +++ b/crates/tools/README.md @@ -22,6 +22,27 @@ to `127.0.0.1:8080`; the Flight frontend consumes the normal `ALGEON_SERVER_*` configuration. The HTTP routes remain `/healthz`, `/api/pipelines`, `/api/pipelines/{name}`, `/api/pipelines/{name}/plan` and `/api/gpu`. +## S3 and Iceberg acceptance runs + +[`s3-parquet-e2e`](src/e2e/s3_parquet_e2e.rs) scans generated Parquet objects +through the `[s3]` storage against real S3 or the RustFS service in +[`fixtures/s3-local`](fixtures/s3-local/compose.yaml). +[`iceberg-local-e2e`](src/e2e/iceberg_local_e2e.rs) writes Iceberg tables +through a REST and a Glue catalog and queries them over an in-process Flight +SQL server, against the services in +[`fixtures/iceberg-local`](fixtures/iceberg-local/compose.yaml). + +```bash +docker compose -f crates/tools/fixtures/iceberg-local/compose.yaml up -d --wait +AWS_ACCESS_KEY_ID=algeon-e2e AWS_SECRET_ACCESS_KEY=algeon-e2e-secret \ + flock /tmp/cudf-gpu.lock cargo run -p tools --features iceberg-local-e2e -- iceberg-local-e2e +docker compose -f crates/tools/fixtures/iceberg-local/compose.yaml down -v +``` + +`algeon_tools --help` prints each run's environment, and +[Tests](../../website/docs/build/tests.md#acceptance-runs-outside-the-gate) +lists what each run checks. + ## Attempt profiling probe [`attempt_lifecycle`](src/bin/attempt_lifecycle.rs) exercises CPU aggregation over diff --git a/crates/tools/fixtures/iceberg-local/compose.yaml b/crates/tools/fixtures/iceberg-local/compose.yaml new file mode 100644 index 000000000..dec95c2e3 --- /dev/null +++ b/crates/tools/fixtures/iceberg-local/compose.yaml @@ -0,0 +1,51 @@ +# Local Iceberg REST and Glue catalogs over RustFS for the explicit +# `iceberg-local-e2e` tool. +# +# docker compose -f crates/tools/fixtures/iceberg-local/compose.yaml up -d --wait +# docker compose -f crates/tools/fixtures/iceberg-local/compose.yaml down -v +# +# RustFS and its bucket come from ../s3-local/compose.yaml: http://127.0.0.1:9000 +# (path-style), bucket `algeon-e2e`, access key `algeon-e2e` and secret key +# `algeon-e2e-secret`. Both stacks bind port 9000, so run one at a time. +# REST catalog: http://127.0.0.1:8181, warehouse `s3://algeon-e2e/warehouse/`. +# Glue (moto): http://127.0.0.1:5001. Catalog state lives in the containers. + +name: algeon-iceberg-local + +include: + - ../s3-local/compose.yaml + +services: + # The fixture turns CATALOG_* variables into catalog properties: `__` + # becomes `-` and `_` becomes `.`, so CATALOG_S3_PATH__STYLE__ACCESS is + # `s3.path-style-access`. Its JDBC catalog is a SQLite file in the container. + rest: + image: apache/iceberg-rest-fixture:1.10.1 + depends_on: + s3-init: + condition: service_completed_successfully + environment: + - CATALOG_WAREHOUSE=s3://algeon-e2e/warehouse/ + - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO + - CATALOG_S3_ENDPOINT=http://s3:9000 + - CATALOG_S3_PATH__STYLE__ACCESS=true + - AWS_ACCESS_KEY_ID=algeon-e2e + - AWS_SECRET_ACCESS_KEY=algeon-e2e-secret + - AWS_REGION=us-east-1 + ports: + - "127.0.0.1:8181:8181" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8181/v1/config"] + interval: 2s + timeout: 5s + retries: 30 + + glue: + image: motoserver/moto:5.1.14 + ports: + - "127.0.0.1:5001:5000" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:5000/moto-api/"] + interval: 2s + timeout: 5s + retries: 30 diff --git a/crates/tools/fixtures/s3-local/compose.yaml b/crates/tools/fixtures/s3-local/compose.yaml new file mode 100644 index 000000000..3fa78fa3a --- /dev/null +++ b/crates/tools/fixtures/s3-local/compose.yaml @@ -0,0 +1,35 @@ +# Local S3-compatible service for the explicit S3 E2E tools. +# +# docker compose -f crates/tools/fixtures/s3-local/compose.yaml up -d --wait +# docker compose -f crates/tools/fixtures/s3-local/compose.yaml down -v +# +# Endpoint http://127.0.0.1:9000 (path-style), bucket `algeon-e2e`, access key +# `algeon-e2e` and secret key `algeon-e2e-secret`. + +name: algeon-s3-local + +services: + s3: + image: rustfs/rustfs:1.0.0 + environment: + - RUSTFS_ACCESS_KEY=algeon-e2e + - RUSTFS_SECRET_KEY=algeon-e2e-secret + ports: + - "127.0.0.1:9000:9000" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/health/ready"] + interval: 2s + timeout: 5s + retries: 30 + + # One-shot job. Creating an existing bucket succeeds, so reruns are safe. + s3-init: + image: rustfs/rustfs:1.0.0 + depends_on: + s3: + condition: service_healthy + restart: "no" + entrypoint: + - /bin/sh + - -c + - curl -fsS --aws-sigv4 aws:amz:us-east-1:s3 -u algeon-e2e:algeon-e2e-secret -X PUT http://s3:9000/algeon-e2e diff --git a/crates/tools/src/e2e/iceberg_local_e2e.rs b/crates/tools/src/e2e/iceberg_local_e2e.rs new file mode 100644 index 000000000..99ac2fbf8 --- /dev/null +++ b/crates/tools/src/e2e/iceberg_local_e2e.rs @@ -0,0 +1,563 @@ +//! Explicit E2E for Iceberg REST and Glue catalogs served over Flight SQL. +//! +//! The runner needs the local services of +//! `crates/tools/fixtures/iceberg-local/compose.yaml`: RustFS, the Iceberg +//! REST fixture and moto as Glue. It writes tables with iceberg-rust through +//! the same catalog connections the server makes, starts the Flight SQL +//! server in process the way the `algeon_server` launcher does with the +//! `iceberg` feature, and checks the catalogs through a Flight SQL client: +//! the catalog, namespace and table listings, a native aggregate over a +//! REST table and a Glue table, and the rejection of a snapshot with a +//! position-delete file. Every namespace, table and object it wrote is +//! removed at the end. + +use std::{ + collections::{BTreeMap, HashMap}, + env, + error::Error, + fs, + net::{SocketAddr, TcpListener}, + path::Path, + str::FromStr, + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use algeon_datafusion::{ + backend::GpuMemoryOwnership, + server::{ + ALGEON_SERVER_CONFIG_FILE_ENV, ERROR_CODE_METADATA_KEY, FlightSqlServer, REASON_METADATA_KEY, ServerConfig, + ServerShutdown, backend_from_server_config, build_session_context_with_backend, + }, +}; +use algeon_iceberg::{connect_catalog, register_iceberg_catalogs}; +use arrow::{ + array::{Array, ArrayRef, Int64Array, RecordBatch, StringArray}, + compute::cast, + datatypes::{DataType, Field, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef}, +}; +use arrow_flight::{ + FlightInfo, + error::FlightError, + sql::{CommandGetDbSchemas, CommandGetTables, client::FlightSqlServiceClient}, +}; +use futures::TryStreamExt; +use iceberg::{ + Catalog, MetadataLocation, NamespaceIdent, TableCreation, TableIdent, + io::FileIO, + spec::{ + DataContentType, DataFile, DataFileBuilder, DataFileFormat, MAIN_BRANCH, ManifestListWriter, + ManifestWriterBuilder, NestedField, Operation, PrimitiveType, Schema, Snapshot, Struct, Summary, + TableMetadataBuilder, Type, + }, + table::Table, + transaction::{ApplyTransactionAction, Transaction}, + writer::{ + IcebergWriter, IcebergWriterBuilder, + base_writer::data_file_writer::DataFileWriterBuilder, + file_writer::{ + ParquetWriterBuilder, + location_generator::{DefaultFileNameGenerator, DefaultLocationGenerator}, + rolling_writer::RollingFileWriterBuilder, + }, + }, +}; +use parquet::arrow::{ArrowWriter, PARQUET_FIELD_ID_META_KEY}; +use tokio_util::sync::CancellationToken; +use tonic::{Code, Status, transport::Channel}; + +use crate::connection_retry::{ConnectionRetryConfig, connect_with_retry}; + +type E2eResult = Result>; + +const S3_ENDPOINT: &str = "http://127.0.0.1:9000"; +const REGION: &str = "us-east-1"; +const REST_URI: &str = "http://127.0.0.1:8181"; +const REST_WAREHOUSE: &str = "s3://algeon-e2e/warehouse/"; +const GLUE_ENDPOINT: &str = "http://127.0.0.1:5001"; +const GLUE_WAREHOUSE: &str = "s3://algeon-e2e/glue-warehouse"; +const REST_CATALOG: &str = "rest"; +const GLUE_CATALOG: &str = "glue"; +const TABLE: &str = "events"; +const DELETES_TABLE: &str = "events_with_deletes"; +const GROUPS: i64 = 7; +/// Ids of the two data files of the REST table. +const REST_FILES: [std::ops::Range; 2] = [0..1_000, 1_000..2_500]; +const GLUE_IDS: std::ops::Range = 0..700; +/// Iceberg's reserved field ids of the position-delete file columns. +const DELETE_FILE_PATH_FIELD_ID: i32 = 2_147_483_546; +const DELETE_POS_FIELD_ID: i32 = 2_147_483_545; +const CONNECT_BUDGET: Duration = Duration::from_secs(30); +const CONNECT_BACKOFF: Duration = Duration::from_millis(100); + +/// Catalog objects the run created, removed in reverse order at the end. +#[derive(Default)] +struct Written { + tables: Vec<(Arc, TableIdent)>, + namespaces: Vec<(Arc, NamespaceIdent)>, + locations: Vec<(FileIO, String)>, +} + +pub async fn run(args: Vec) -> E2eResult<()> { + if let Some(arg) = args.first() { + if arg == "-h" || arg == "--help" { + print_usage(); + return Ok(()); + } + return Err(format!("unknown iceberg-local-e2e argument: {arg}").into()); + } + let secs = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); + let namespace = format!("e2e_{}_{secs}", std::process::id()); + let config_dir = env::temp_dir().join(format!("algeon-iceberg-local-e2e-{namespace}")); + let config = server_config(&config_dir)?; + let rest = connect_catalog(&config.iceberg_catalogs[0], config.s3_storage()).await?; + let glue = connect_catalog(&config.iceberg_catalogs[1], config.s3_storage()).await?; + + let mut written = Written::default(); + let result = async { + write_tables(&rest, &glue, &namespace, &mut written).await?; + serve_and_check(config, &namespace).await + } + .await; + let cleanup = remove(written).await; + fs::remove_dir_all(&config_dir)?; + result?; + cleanup?; + println!("cleanup ok"); + println!("iceberg_local_e2e ok"); + Ok(()) +} + +fn print_usage() { + println!( + "Usage: iceberg-local-e2e\n\n\ + Writes Iceberg tables through a REST catalog and a Glue catalog, serves both from an in-process\n\ + Flight SQL server in native_required mode, checks the listings, two native aggregates and the\n\ + rejection of a snapshot with a position-delete file, and removes what it wrote. It needs the\n\ + services of crates/tools/fixtures/iceberg-local/compose.yaml: RustFS at {S3_ENDPOINT}, the REST\n\ + catalog at {REST_URI} and moto as Glue at {GLUE_ENDPOINT}. S3 and Glue credentials come from the\n\ + default AWS chain.\n\n\ + docker compose -f crates/tools/fixtures/iceberg-local/compose.yaml up -d --wait\n\ + AWS_ACCESS_KEY_ID=algeon-e2e AWS_SECRET_ACCESS_KEY=algeon-e2e-secret \\\n\ + flock /tmp/cudf-gpu.lock cargo run -p tools --features iceberg-local-e2e -- iceberg-local-e2e\n\ + docker compose -f crates/tools/fixtures/iceberg-local/compose.yaml down -v" + ); +} + +/// The server configuration, loaded from a TOML file like the launcher's. +/// The REST entry comes first and the Glue entry second. +fn server_config(dir: &Path) -> E2eResult { + fs::create_dir_all(dir)?; + let path = dir.join("server.toml"); + let toml = format!( + "[native]\nexecution_mode = \"native_required\"\n\n\ + [s3]\nendpoint = \"{S3_ENDPOINT}\"\nregion = \"{REGION}\"\npath_style = true\n\n\ + [[iceberg.catalogs]]\nname = \"{REST_CATALOG}\"\nkind = \"rest\"\nuri = \"{REST_URI}\"\nwarehouse = \"{REST_WAREHOUSE}\"\n\n\ + [[iceberg.catalogs]]\nname = \"{GLUE_CATALOG}\"\nkind = \"glue\"\nwarehouse = \"{GLUE_WAREHOUSE}\"\n\ + region = \"{REGION}\"\nendpoint = \"{GLUE_ENDPOINT}\"\n\n\ + [[admission.device_profiles]]\ndevice_ordinal = 0\n" + ); + fs::write(&path, toml)?; + let config = ServerConfig::from_vars([(ALGEON_SERVER_CONFIG_FILE_ENV, path.to_string_lossy().into_owned())])?; + Ok(config.with_gpu_memory_ownership([(0, GpuMemoryOwnership::WholeDeviceExclusive)])) +} + +/// `id`, `grp` and `val`, all required longs. +fn events_schema() -> E2eResult { + Ok(Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::required(2, "grp", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::required(3, "val", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build()?) +} + +/// REST: `events` with two data files from one `fast_append`, and +/// `events_with_deletes`, a snapshot of `events` plus a position delete. +/// Glue: `events` with one data file. +async fn write_tables( + rest: &Arc, + glue: &Arc, + namespace: &str, + written: &mut Written, +) -> E2eResult<()> { + let rest_events = create_table(rest, namespace, written).await?; + let mut files = Vec::new(); + for (index, ids) in REST_FILES.iter().enumerate() { + files.extend(write_events(&rest_events, &format!("part-{index}"), ids.clone()).await?); + } + if files.len() != REST_FILES.len() { + return Err(format!("expected {} REST data files, wrote {}", REST_FILES.len(), files.len()).into()); + } + let target = files[0].file_path().to_string(); + let rest_events = append(rest, &rest_events, files).await?; + let deletes_ident = TableIdent::new(NamespaceIdent::new(namespace.to_string()), DELETES_TABLE.to_string()); + commit_position_delete(rest, &rest_events, &deletes_ident, &target).await?; + written.tables.push((Arc::clone(rest), deletes_ident)); + println!("rest tables written: {TABLE} ({} data files), {DELETES_TABLE} (1 position delete)", REST_FILES.len()); + + let glue_events = create_table(glue, namespace, written).await?; + let files = write_events(&glue_events, "part-0", GLUE_IDS).await?; + append(glue, &glue_events, files).await?; + println!("glue table written: {TABLE} (1 data file)"); + Ok(()) +} + +async fn create_table(catalog: &Arc, namespace: &str, written: &mut Written) -> E2eResult
{ + let namespace = NamespaceIdent::new(namespace.to_string()); + catalog.create_namespace(&namespace, HashMap::new()).await?; + written.namespaces.push((Arc::clone(catalog), namespace.clone())); + let creation = TableCreation::builder().name(TABLE.to_string()).schema(events_schema()?).build(); + let table = catalog.create_table(&namespace, creation).await?; + written.tables.push((Arc::clone(catalog), table.identifier().clone())); + written.locations.push((table.file_io().clone(), table.metadata().location().to_string())); + Ok(table) +} + +fn events_batch(schema: &ArrowSchemaRef, ids: std::ops::Range) -> E2eResult { + let ids = ids.collect::>(); + Ok(RecordBatch::try_new( + Arc::clone(schema), + vec![ + Arc::new(Int64Array::from(ids.clone())) as ArrayRef, + Arc::new(Int64Array::from(ids.iter().map(|id| id % GROUPS).collect::>())), + Arc::new(Int64Array::from(ids.iter().map(|id| id * 3).collect::>())), + ], + )?) +} + +/// Writes `ids` with Iceberg's Parquet writer, which records field ids and +/// column bounds, under the table's data directory. +async fn write_events(table: &Table, prefix: &str, ids: std::ops::Range) -> E2eResult> { + let iceberg_schema = Arc::clone(table.metadata().current_schema()); + let arrow_schema: ArrowSchemaRef = Arc::new(iceberg_schema.as_ref().try_into()?); + let rolling = RollingFileWriterBuilder::new_with_default_file_size( + ParquetWriterBuilder::new(Default::default(), iceberg_schema), + table.file_io().clone(), + DefaultLocationGenerator::new(table.metadata())?, + DefaultFileNameGenerator::new(prefix.to_string(), None, DataFileFormat::Parquet), + ); + let mut writer = DataFileWriterBuilder::new(rolling).build(None).await?; + writer.write(events_batch(&arrow_schema, ids)?).await?; + Ok(writer.close().await?) +} + +async fn append(catalog: &Arc, table: &Table, files: Vec) -> E2eResult
{ + let transaction = Transaction::new(table); + let transaction = transaction.fast_append().add_data_files(files).apply(transaction)?; + Ok(transaction.commit(catalog.as_ref()).await?) +} + +/// Registers `ident` as `table` plus a snapshot whose delete manifest holds +/// one position delete for the data file at `target`. Iceberg's +/// transactions cannot commit delete files, so the delete file, manifest, +/// manifest list and metadata are written by hand. +async fn commit_position_delete( + catalog: &Arc, + table: &Table, + ident: &TableIdent, + target: &str, +) -> E2eResult<()> { + let metadata = table.metadata(); + let parent = metadata.current_snapshot().ok_or("the table has no snapshot to delete from")?; + let snapshot_id = parent.snapshot_id() + 1; + let sequence_number = metadata.last_sequence_number() + 1; + let file_io = table.file_io(); + let metadata_dir = format!("{}/metadata", metadata.location()); + + let delete_path = format!("{}/data/deletes-{snapshot_id}.parquet", metadata.location()); + let delete_bytes = position_delete_parquet(target)?; + let delete_size = u64::try_from(delete_bytes.len())?; + file_io.new_output(&delete_path)?.write(delete_bytes.into()).await?; + let mut delete_file = DataFileBuilder::default(); + delete_file + .content(DataContentType::PositionDeletes) + .file_path(delete_path) + .file_format(DataFileFormat::Parquet) + .partition(Struct::empty()) + .record_count(1) + .file_size_in_bytes(delete_size) + .referenced_data_file(Some(target.to_string())); + let manifest_output = file_io.new_output(format!("{metadata_dir}/deletes-{snapshot_id}.avro"))?; + let mut manifest = ManifestWriterBuilder::new( + manifest_output, + Some(snapshot_id), + Arc::clone(metadata.current_schema()), + metadata.default_partition_spec().as_ref().clone(), + ) + .build_v2_deletes(); + manifest.add_file(delete_file.build()?, sequence_number)?; + let delete_manifest = manifest.write_manifest_file().await?; + + let existing = table.manifest_list_reader(parent).load().await?.consume_entries(); + let manifest_list = format!("{metadata_dir}/snap-{snapshot_id}.avro"); + let mut list = ManifestListWriter::v2( + file_io.new_output(&manifest_list)?.writer().await?, + snapshot_id, + Some(parent.snapshot_id()), + sequence_number, + ); + list.add_manifests(existing.into_iter().chain([delete_manifest]))?; + list.close().await?; + + let snapshot = Snapshot::builder() + .with_snapshot_id(snapshot_id) + .with_parent_snapshot_id(Some(parent.snapshot_id())) + .with_sequence_number(sequence_number) + .with_timestamp_ms(parent.timestamp_ms() + 1) + .with_manifest_list(manifest_list) + .with_summary(Summary { operation: Operation::Delete, additional_properties: HashMap::new() }) + .with_schema_id(metadata.current_schema_id()) + .build(); + let new_metadata = TableMetadataBuilder::new_from_metadata(metadata.clone(), None) + .set_branch_snapshot(snapshot, MAIN_BRANCH)? + .build()? + .metadata; + // The next version gets a new file name; the current file stays the + // metadata of `table`. + let location = MetadataLocation::from_str(table.metadata_location().ok_or("the table has no metadata location")?)? + .with_next_version() + .try_with_new_metadata(&new_metadata)?; + new_metadata.write_to(file_io, &location).await?; + catalog.register_table(ident, location.to_string()).await?; + Ok(()) +} + +/// A position-delete file that deletes row 0 of `target`. +fn position_delete_parquet(target: &str) -> E2eResult> { + let field = |name: &str, data_type, id: i32| { + Field::new(name, data_type, false) + .with_metadata(HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), id.to_string())])) + }; + let schema = Arc::new(ArrowSchema::new(vec![ + field("file_path", DataType::Utf8, DELETE_FILE_PATH_FIELD_ID), + field("pos", DataType::Int64, DELETE_POS_FIELD_ID), + ])); + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(StringArray::from(vec![target])) as ArrayRef, Arc::new(Int64Array::from(vec![0_i64]))], + )?; + let mut bytes = Vec::new(); + let mut writer = ArrowWriter::try_new(&mut bytes, schema, None)?; + writer.write(&batch)?; + writer.close()?; + Ok(bytes) +} + +/// Drops the tables and namespaces and deletes every object under the table +/// locations. Every step runs; the first failure is returned. +async fn remove(written: Written) -> E2eResult<()> { + let mut first_error: Option> = None; + let mut record = |result: E2eResult<()>| { + if let Err(error) = result + && first_error.is_none() + { + first_error = Some(error); + } + }; + for (catalog, ident) in written.tables.iter().rev() { + record(catalog.drop_table(ident).await.map_err(Into::into)); + } + for (catalog, namespace) in written.namespaces.iter().rev() { + record(catalog.drop_namespace(namespace).await.map_err(Into::into)); + } + for (file_io, location) in &written.locations { + record(file_io.delete_prefix(location).await.map_err(Into::into)); + } + first_error.map_or(Ok(()), Err) +} + +/// Builds the server as the launcher does: the backend and shared session +/// from `config`, the Iceberg catalogs registered on that session, then the +/// Flight SQL server around both. +async fn serve_and_check(config: ServerConfig, namespace: &str) -> E2eResult<()> { + let backend = backend_from_server_config(&config)?; + let ctx = build_session_context_with_backend(&config, &backend).await?; + register_iceberg_catalogs(&ctx, &config).await?; + let listener = TcpListener::bind("127.0.0.1:0")?; + let addr = listener.local_addr()?; + let shutdown = CancellationToken::new(); + let server = FlightSqlServer::builder() + .config(config) + .backend(backend) + .session_context(Arc::new(ctx)) + .listener(listener) + .shutdown(ServerShutdown::Token(shutdown.clone())) + .build() + .await?; + let serving = tokio::spawn(server.serve()); + let checks = check_server(addr, namespace).await; + shutdown.cancel(); + let served = serving.await?; + checks?; + served?; + Ok(()) +} + +async fn check_server(addr: SocketAddr, namespace: &str) -> E2eResult<()> { + let url = format!("http://{addr}"); + let mut client = connect_with_retry( + &url, + ConnectionRetryConfig::time_budget(CONNECT_BUDGET, CONNECT_BACKOFF), + FlightSqlServiceClient::new, + |error| format!("no Flight SQL server accepted connections at {url}: {error}"), + ) + .await?; + + let info = client.get_catalogs().await?; + let catalogs = string_column(&fetch(&mut client, info).await?, "catalog_name")?; + for name in [REST_CATALOG, GLUE_CATALOG] { + if !catalogs.iter().any(|catalog| catalog == name) { + return Err(format!("GetCatalogs returned {catalogs:?}, missing {name}").into()); + } + } + println!("catalogs listed ok: {REST_CATALOG}, {GLUE_CATALOG}"); + + for catalog in [REST_CATALOG, GLUE_CATALOG] { + let info = client + .get_db_schemas(CommandGetDbSchemas { catalog: Some(catalog.into()), db_schema_filter_pattern: None }) + .await?; + let schemas = string_column(&fetch(&mut client, info).await?, "db_schema_name")?; + if !schemas.iter().any(|schema| schema == namespace) { + return Err(format!("GetDbSchemas for {catalog} returned {schemas:?}, missing {namespace}").into()); + } + println!("{catalog} namespace listed ok: {namespace}"); + } + + for (catalog, expected) in [(REST_CATALOG, vec![TABLE, DELETES_TABLE]), (GLUE_CATALOG, vec![TABLE])] { + let info = client + .get_tables(CommandGetTables { + catalog: Some(catalog.into()), + db_schema_filter_pattern: Some(namespace.into()), + table_name_filter_pattern: None, + table_types: vec![], + include_schema: false, + }) + .await?; + let mut tables = string_column(&fetch(&mut client, info).await?, "table_name")?; + tables.sort(); + let mut expected = expected.into_iter().map(str::to_string).collect::>(); + expected.sort(); + if tables != expected { + return Err( + format!("GetTables for {catalog}.{namespace} returned {tables:?}, expected {expected:?}").into() + ); + } + println!("{catalog} tables listed ok: {}", tables.join(", ")); + } + + let rest_ids = REST_FILES.iter().flat_map(Clone::clone).collect::>(); + for (catalog, ids) in [(REST_CATALOG, rest_ids), (GLUE_CATALOG, GLUE_IDS.collect())] { + let query = format!( + "SELECT grp, COUNT(*) AS n, SUM(val) AS total FROM {catalog}.{namespace}.{TABLE} GROUP BY grp ORDER BY grp" + ); + let (gpu_path, reason_code, detail) = explain_gpu(&mut client, &query).await?; + if gpu_path != "native" { + return Err(format!( + "EXPLAIN GPU over {catalog}.{namespace}.{TABLE} reports gpu_path {gpu_path}, reason {reason_code}: {detail}" + ) + .into()); + } + let info = client.execute(query, None).await?; + let rows = aggregate_rows(&fetch(&mut client, info).await?)?; + let expected = expected_rows(&ids); + if rows != expected { + return Err(format!("{catalog}.{namespace}.{TABLE} returned {rows:?}, expected {expected:?}").into()); + } + println!("{catalog} aggregate ok: native plan, {} rows over {} ids", rows.len(), ids.len()); + } + + let query = format!("SELECT COUNT(*) FROM {REST_CATALOG}.{namespace}.{DELETES_TABLE}"); + let status = match execute(&mut client, &query).await { + Ok(_) => return Err(format!("{query} succeeded; the snapshot has a position-delete file").into()), + Err(FlightError::Tonic(status)) => *status, + Err(other) => return Err(format!("{query} failed without a gRPC status: {other}").into()), + }; + let code = metadata_value(&status, ERROR_CODE_METADATA_KEY); + let reason = metadata_value(&status, REASON_METADATA_KEY); + if status.code() != Code::Unimplemented + || code != Some("unsupported_table_feature") + || reason != Some("iceberg_delete_files") + { + return Err(format!( + "{query} failed with {:?}, {ERROR_CODE_METADATA_KEY}={code:?}, {REASON_METADATA_KEY}={reason:?}: {}", + status.code(), + status.message() + ) + .into()); + } + println!("rest delete-file table rejected ok: UNIMPLEMENTED, unsupported_table_feature, iceberg_delete_files"); + Ok(()) +} + +/// Runs `sql` and collects its batches. A planning or execution failure +/// keeps its gRPC status, whether GetFlightInfo or DoGet returns it. +async fn execute(client: &mut FlightSqlServiceClient, sql: &str) -> Result, FlightError> { + let info = client.execute(sql.to_string(), None).await?; + fetch(client, info).await +} + +async fn fetch( + client: &mut FlightSqlServiceClient, + info: FlightInfo, +) -> Result, FlightError> { + let mut batches = Vec::new(); + for endpoint in info.endpoint { + let Some(ticket) = endpoint.ticket else { + continue; + }; + batches.extend(client.do_get(ticket).await?.try_collect::>().await?); + } + Ok(batches) +} + +/// The `gpu_path`, `reason_code` and `detail_json` of the summary row +/// `EXPLAIN GPU` returns for `query`. +async fn explain_gpu(client: &mut FlightSqlServiceClient, query: &str) -> E2eResult<(String, String, String)> { + let batches = execute(client, &format!("EXPLAIN GPU {query}")).await?; + let row_kinds = string_column(&batches, "row_kind")?; + let gpu_paths = string_column(&batches, "gpu_path")?; + let reason_codes = string_column(&batches, "reason_code")?; + let details = string_column(&batches, "detail_json")?; + let summary = row_kinds.iter().position(|kind| kind == "summary").ok_or("EXPLAIN GPU returned no summary row")?; + Ok((gpu_paths[summary].clone(), reason_codes[summary].clone(), details[summary].clone())) +} + +fn string_column(batches: &[RecordBatch], name: &str) -> E2eResult> { + let mut values = Vec::new(); + for batch in batches { + let column = cast(batch.column(batch.schema().index_of(name)?), &DataType::Utf8)?; + let column = column.as_any().downcast_ref::().ok_or("cast to Utf8 returned another array")?; + values.extend((0..column.len()).map(|row| column.value(row).to_string())); + } + Ok(values) +} + +fn aggregate_rows(batches: &[RecordBatch]) -> E2eResult> { + let mut rows = Vec::new(); + for batch in batches { + let column = |index: usize| -> E2eResult<&Int64Array> { + Ok(batch.column(index).as_any().downcast_ref::().ok_or("result column is not Int64")?) + }; + let (grp, count, total) = (column(0)?, column(1)?, column(2)?); + for row in 0..batch.num_rows() { + rows.push((grp.value(row), count.value(row), total.value(row))); + } + } + Ok(rows) +} + +fn expected_rows(ids: &[i64]) -> Vec<(i64, i64, i64)> { + let mut groups = BTreeMap::::new(); + for id in ids { + let entry = groups.entry(id % GROUPS).or_default(); + entry.0 += 1; + entry.1 += id * 3; + } + groups.into_iter().map(|(grp, (count, total))| (grp, count, total)).collect() +} + +fn metadata_value<'a>(status: &'a Status, key: &str) -> Option<&'a str> { + status.metadata().get(key).and_then(|value| value.to_str().ok()) +} diff --git a/crates/tools/src/e2e/mod.rs b/crates/tools/src/e2e/mod.rs index 4e7073031..c5e5a0b47 100644 --- a/crates/tools/src/e2e/mod.rs +++ b/crates/tools/src/e2e/mod.rs @@ -4,6 +4,10 @@ pub(super) mod cugraph_bfs_predicate_fixture; pub(super) mod embedded_backend_e2e; #[cfg(not(test))] pub(super) mod flight_sql_cugraph_smoke; +#[cfg(all(feature = "iceberg-local-e2e", not(test)))] +pub(super) mod iceberg_local_e2e; +#[cfg(all(feature = "s3-parquet-e2e", not(test)))] +pub(super) mod s3_parquet_e2e; #[cfg(feature = "server-runtime-observability-e2e")] #[cfg_attr(test, allow(dead_code))] pub(super) mod server_runtime_observability; diff --git a/crates/tools/src/e2e/s3_parquet_e2e.rs b/crates/tools/src/e2e/s3_parquet_e2e.rs new file mode 100644 index 000000000..2517951dd --- /dev/null +++ b/crates/tools/src/e2e/s3_parquet_e2e.rs @@ -0,0 +1,406 @@ +//! Explicit E2E for `s3://` Parquet scans through one S3 configuration. +//! +//! The runner needs real S3 or an S3-compatible service and an existing +//! bucket, so it stays out of Cargo test targets. It uploads generated Parquet +//! objects through the session's `s3://` object store, then runs one aggregate +//! over three layouts: one object, a prefix of three objects and one object +//! that DataFusion splits into byte ranges. Each layout runs in a +//! native-required and a functions-only session from a library backend built +//! with the S3 storage, and in a native-required session built from server +//! configuration. Every check plans and executes through an attempt, because +//! only an attempt's planning invocation carries the remote footers native +//! lowering reads. Each final plan must be the disposition its mode requires +//! and each result must equal the generated data. + +use std::{ + env, + error::Error, + fmt::Write as _, + fs, + ops::Range, + path::Path, + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use algeon_datafusion::{ + attempt::{AlgeonAttempt, AttemptOptions}, + backend::{AlgeonGpuBackend, AlgeonGpuDeviceConfig, AlgeonGpuDeviceProfile, GpuMemoryOwnership}, + native_exec::{AlgeonNativeExecutionConfig, S3SourceStorageProfile}, + planner::{AlgeonGpuExecutionMode, AlgeonNativeOptimizerConfig, FinalPlanDisposition, classify_final_plan}, + server::{ + ALGEON_SERVER_CONFIG_FILE_ENV, ServerConfig, backend_from_server_config, build_session_context_with_backend, + }, +}; +use datafusion::{ + arrow::{ + array::{Array, Int64Array}, + datatypes::{DataType, Field, Schema}, + record_batch::RecordBatch, + }, + datasource::{physical_plan::ParquetSource, source::DataSourceExec}, + execution::{SessionStateBuilder, object_store::ObjectStoreUrl}, + object_store::{ObjectStore, ObjectStoreExt, path::Path as ObjectStorePath}, + parquet::{arrow::ArrowWriter, file::properties::WriterProperties}, + physical_plan::ExecutionPlan, + prelude::{ParquetReadOptions, SessionContext}, +}; +use futures::TryStreamExt; + +type E2eResult = Result>; + +const ENDPOINT_ENV: &str = "ALGEON_S3_PARQUET_E2E_ENDPOINT"; +const BUCKET_ENV: &str = "ALGEON_S3_PARQUET_E2E_BUCKET"; +const REGION_ENV: &str = "ALGEON_S3_PARQUET_E2E_REGION"; +const DEFAULT_REGION: &str = "us-east-1"; +const GROUPS: i64 = 7; +const SMALL_OBJECT_ROWS: i64 = 10_000; +const MULTI_OBJECT_COUNT: i64 = 3; +/// Large enough that one object spans many row groups and a few megabytes, so +/// a byte-range split lands inside it. +const RANGED_OBJECT_ROWS: i64 = 400_000; +const ROW_GROUP_ROWS: usize = 25_000; +const RANGE_TARGET_PARTITIONS: usize = 4; +const QUERY: &str = "SELECT grp, COUNT(*) AS n, SUM(val) AS total FROM t GROUP BY grp ORDER BY grp"; +const CLOSE_TIMEOUT: Duration = Duration::from_secs(60); + +struct Settings { + endpoint: Option, + bucket: String, + region: String, + storage: S3SourceStorageProfile, +} + +struct Object { + key: String, + ids: Range, +} + +struct Case { + name: &'static str, + location: String, + objects: Vec, + /// The case splits its one object into byte ranges. + ranged: bool, +} + +pub async fn run(args: Vec) -> E2eResult<()> { + if let Some(arg) = args.first() { + if arg == "-h" || arg == "--help" { + print_usage(); + return Ok(()); + } + return Err(format!("unknown s3-parquet-e2e argument: {arg}").into()); + } + let settings = Settings::from_env()?; + let nanos = SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos(); + let run_id = format!("{}-{nanos}", std::process::id()); + let cases = cases(&settings.bucket, &run_id); + + let backend = library_backend(&settings.storage, AlgeonGpuExecutionMode::NativeRequired)?; + let ctx = library_session(&backend)?; + let store = ctx.runtime_env().object_store(ObjectStoreUrl::parse(format!("s3://{}", settings.bucket))?)?; + close(ctx, backend)?; + + let mut uploaded = Vec::new(); + let result = async { + upload(store.as_ref(), &cases, &mut uploaded).await?; + run_scenarios(&settings, &cases, &run_id).await + } + .await; + let cleanup = delete(store.as_ref(), &uploaded).await; + result?; + cleanup?; + println!("s3_parquet_e2e ok"); + Ok(()) +} + +async fn run_scenarios(settings: &Settings, cases: &[Case], run_id: &str) -> E2eResult<()> { + for (label, mode, disposition) in [ + ("library native_required", AlgeonGpuExecutionMode::NativeRequired, FinalPlanDisposition::Native), + ("library functions_only", AlgeonGpuExecutionMode::FunctionsOnly, FinalPlanDisposition::DataFusion), + ] { + let backend = library_backend(&settings.storage, mode)?; + let ctx = library_session(&backend)?; + check_cases(label, &backend, &ctx, cases, disposition).await?; + close(ctx, backend)?; + println!("{label} ok"); + } + + let config_dir = env::temp_dir().join(format!("algeon-s3-parquet-e2e-{run_id}")); + let result = async { + let config = server_config(&config_dir, settings)?; + let backend = backend_from_server_config(&config)?; + let ctx = build_session_context_with_backend(&config, &backend).await?; + check_cases("server native_required", &backend, &ctx, cases, FinalPlanDisposition::Native).await?; + close(ctx, backend) + } + .await; + fs::remove_dir_all(&config_dir)?; + result?; + println!("server native_required ok"); + Ok(()) +} + +impl Settings { + fn from_env() -> E2eResult { + let bucket = required_env(BUCKET_ENV)?; + let endpoint = optional_env(ENDPOINT_ENV); + let region = optional_env(REGION_ENV) + .or_else(|| optional_env("AWS_REGION")) + .unwrap_or_else(|| DEFAULT_REGION.to_owned()); + let mut storage = S3SourceStorageProfile::default_chain().with_region(region.clone()); + // A custom endpoint is addressed path-style. Real S3 keeps the + // default virtual-hosted addressing. + if let Some(endpoint) = &endpoint { + storage = storage.with_endpoint(endpoint.clone()).with_path_style_access(true); + } + Ok(Self { endpoint, bucket, region, storage }) + } +} + +fn print_usage() { + println!( + "Usage: s3-parquet-e2e\n\n\ + Uploads generated Parquet objects under a unique key prefix, checks one object, a prefix of three\n\ + objects and one object split into byte ranges in library and server-configured sessions, and\n\ + deletes the objects. {BUCKET_ENV} names an existing bucket. {ENDPOINT_ENV} selects an\n\ + S3-compatible service addressed path-style; without it the run uses real S3 with virtual-hosted\n\ + addressing. {REGION_ENV} defaults to AWS_REGION, then {DEFAULT_REGION}. Credentials come from the\n\ + default AWS chain.\n\n\ + Local RustFS (compose file: crates/tools/fixtures/s3-local/compose.yaml):\n\ + docker compose -f crates/tools/fixtures/s3-local/compose.yaml up -d --wait\n\ + ALGEON_S3_PARQUET_E2E_ENDPOINT=http://127.0.0.1:9000 ALGEON_S3_PARQUET_E2E_BUCKET=algeon-e2e \\\n\ + AWS_ACCESS_KEY_ID=algeon-e2e AWS_SECRET_ACCESS_KEY=algeon-e2e-secret \\\n\ + flock /tmp/cudf-gpu.lock cargo run -p tools --features s3-parquet-e2e -- s3-parquet-e2e\n\ + docker compose -f crates/tools/fixtures/s3-local/compose.yaml down -v\n\n\ + Real S3 (credentials from the default chain):\n\ + ALGEON_S3_PARQUET_E2E_BUCKET= ALGEON_S3_PARQUET_E2E_REGION= \\\n\ + flock /tmp/cudf-gpu.lock cargo run -p tools --features s3-parquet-e2e -- s3-parquet-e2e" + ); +} + +fn required_env(name: &str) -> E2eResult { + optional_env(name).ok_or_else(|| format!("{name} must be set").into()) +} + +fn optional_env(name: &str) -> Option { + env::var(name).ok().filter(|value| !value.is_empty()) +} + +fn device() -> AlgeonGpuDeviceConfig { + AlgeonGpuDeviceConfig::new(AlgeonGpuDeviceProfile::new(0), GpuMemoryOwnership::WholeDeviceExclusive) +} + +fn library_backend(storage: &S3SourceStorageProfile, mode: AlgeonGpuExecutionMode) -> E2eResult { + let optimizer_config = AlgeonNativeOptimizerConfig::default() + .with_execution_mode(mode) + .with_native_execution_config(AlgeonNativeExecutionConfig::default().with_s3_storage(storage.clone())); + Ok(AlgeonGpuBackend::builder().with_optimizer_config(optimizer_config).devices([device()]).build()?) +} + +fn library_session(backend: &AlgeonGpuBackend) -> E2eResult { + let state = backend.install_on(SessionStateBuilder::new_with_default_features())?.build(); + Ok(SessionContext::new_with_state(state)) +} + +fn server_config(dir: &Path, settings: &Settings) -> E2eResult { + fs::create_dir_all(dir)?; + let path = dir.join("server.toml"); + let mut toml = String::from("[native]\nexecution_mode = \"native_required\"\n\n[s3]\n"); + writeln!(toml, "region = \"{}\"", settings.region)?; + if let Some(endpoint) = &settings.endpoint { + writeln!(toml, "endpoint = \"{endpoint}\"\npath_style = true")?; + } + toml.push_str("\n[[admission.device_profiles]]\ndevice_ordinal = 0\n"); + fs::write(&path, toml)?; + let config = ServerConfig::from_vars([(ALGEON_SERVER_CONFIG_FILE_ENV, path.to_string_lossy().into_owned())])?; + Ok(config.with_gpu_memory_ownership([(0, GpuMemoryOwnership::WholeDeviceExclusive)])) +} + +fn close(ctx: SessionContext, backend: AlgeonGpuBackend) -> E2eResult<()> { + // The next backend in this process needs the owner lease this one holds. + drop(ctx); + backend.close_and_wait(CLOSE_TIMEOUT)?; + Ok(()) +} + +fn cases(bucket: &str, run_id: &str) -> Vec { + let prefix = format!("algeon-s3-parquet-e2e/{run_id}"); + let single_key = format!("{prefix}/single/data.parquet"); + let ranged_key = format!("{prefix}/ranged/data.parquet"); + let multi_objects = (0..MULTI_OBJECT_COUNT) + .map(|index| Object { + key: format!("{prefix}/multi/part-{index}.parquet"), + ids: index * SMALL_OBJECT_ROWS..(index + 1) * SMALL_OBJECT_ROWS, + }) + .collect(); + vec![ + Case { + name: "one object", + location: format!("s3://{bucket}/{single_key}"), + objects: vec![Object { key: single_key, ids: 0..SMALL_OBJECT_ROWS }], + ranged: false, + }, + Case { + name: "prefix of three objects", + location: format!("s3://{bucket}/{prefix}/multi/"), + objects: multi_objects, + ranged: false, + }, + Case { + name: "one object in byte ranges", + location: format!("s3://{bucket}/{ranged_key}"), + objects: vec![Object { key: ranged_key, ids: 0..RANGED_OBJECT_ROWS }], + ranged: true, + }, + ] +} + +fn object_batch(ids: &Range) -> E2eResult { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("grp", DataType::Int64, false), + Field::new("val", DataType::Int64, false), + ])); + let ids = ids.clone().collect::>(); + Ok(RecordBatch::try_new( + schema, + vec![ + Arc::new(Int64Array::from(ids.clone())), + Arc::new(Int64Array::from(ids.iter().map(|id| id % GROUPS).collect::>())), + Arc::new(Int64Array::from(ids.iter().map(|id| id * 2).collect::>())), + ], + )?) +} + +/// Writes the generated objects through the object store the installed S3 +/// storage resolves for the bucket. Keys are recorded before each write so a +/// failed run still deletes what reached the bucket. +async fn upload(store: &dyn ObjectStore, cases: &[Case], uploaded: &mut Vec) -> E2eResult<()> { + let properties = WriterProperties::builder().set_max_row_group_row_count(Some(ROW_GROUP_ROWS)).build(); + for object in cases.iter().flat_map(|case| &case.objects) { + let batch = object_batch(&object.ids)?; + let mut writer = ArrowWriter::try_new(Vec::new(), batch.schema(), Some(properties.clone()))?; + writer.write(&batch)?; + let bytes = writer.into_inner()?; + let path = ObjectStorePath::from(object.key.as_str()); + uploaded.push(path.clone()); + store.put(&path, bytes.into()).await?; + } + Ok(()) +} + +async fn delete(store: &dyn ObjectStore, uploaded: &[ObjectStorePath]) -> E2eResult<()> { + for path in uploaded { + store.delete(path).await?; + } + Ok(()) +} + +async fn check_cases( + label: &str, + backend: &AlgeonGpuBackend, + ctx: &SessionContext, + cases: &[Case], + disposition: FinalPlanDisposition, +) -> E2eResult<()> { + for case in cases { + if case.ranged { + split_scans_into_ranges(ctx); + } + let ranged_files = check(backend, ctx, case, disposition) + .await + .map_err(|error| -> Box { format!("{label}, {}: {error}", case.name).into() })?; + println!("{label}, {}: {disposition} plan, {ranged_files} ranged files, result ok", case.name); + } + Ok(()) +} + +/// Lets DataFusion split the one object of the ranged case across several +/// file groups. The setting stays on the session for the cases after it. +fn split_scans_into_ranges(ctx: &SessionContext) { + let state = ctx.state_ref(); + let mut state = state.write(); + let options = state.config_mut().options_mut(); + options.optimizer.repartition_file_min_size = 1; + options.execution.target_partitions = RANGE_TARGET_PARTITIONS; +} + +/// Plans and runs the aggregate over the case's objects as one attempt and +/// returns the number of byte-ranged files in the scanned plan. +async fn check( + backend: &AlgeonGpuBackend, + ctx: &SessionContext, + case: &Case, + disposition: FinalPlanDisposition, +) -> E2eResult { + ctx.register_parquet("t", &case.location, ParquetReadOptions::default()).await?; + let attempt = backend.begin_attempt(AttemptOptions::default())?; + let plan = attempt.plan_sql(ctx, QUERY).await?; + let observed = classify_final_plan(plan.as_ref()); + if observed != disposition { + return Err(format!("final plan is {observed}, expected {disposition}").into()); + } + let ranged_files = ranged_file_count(&attempt, plan.as_ref(), disposition); + if case.ranged != (ranged_files > 1) { + return Err(format!( + "plan has {ranged_files} ranged files, but the case expects {}", + if case.ranged { "several" } else { "none" } + ) + .into()); + } + let batches = attempt.execute_physical_plan(plan, ctx)?.try_collect::>().await?; + let rows = result_rows(&batches)?; + let expected = expected_rows(case); + if rows != expected { + return Err(format!("query returned {rows:?}, expected {expected:?}").into()); + } + ctx.deregister_table("t")?; + Ok(ranged_files) +} + +/// A native plan no longer holds the file scan, so its ranged files come from +/// the attempt's source diagnostics. A DataFusion plan still holds them. +fn ranged_file_count(attempt: &AlgeonAttempt, plan: &dyn ExecutionPlan, disposition: FinalPlanDisposition) -> usize { + match disposition { + FinalPlanDisposition::Native => attempt.planning_diagnostics().map_or(0, |diagnostics| { + diagnostics.source_diagnostics.iter().map(|source| source.ranged_file_count).sum() + }), + _ => datafusion_ranged_files(plan), + } +} + +fn datafusion_ranged_files(plan: &dyn ExecutionPlan) -> usize { + let own = plan + .downcast_ref::() + .and_then(|scan| scan.downcast_to_file_source::()) + .map_or(0, |(file_scan, _)| { + file_scan.file_groups.iter().flat_map(|group| group.files()).filter(|file| file.range.is_some()).count() + }); + own + plan.children().into_iter().map(|child| datafusion_ranged_files(child.as_ref())).sum::() +} + +fn result_rows(batches: &[RecordBatch]) -> E2eResult> { + let mut rows = Vec::new(); + for batch in batches { + let column = |index: usize| -> E2eResult<&Int64Array> { + Ok(batch.column(index).as_any().downcast_ref::().ok_or("result column is not Int64")?) + }; + let (grp, count, total) = (column(0)?, column(1)?, column(2)?); + for row in 0..batch.num_rows() { + rows.push((grp.value(row), count.value(row), total.value(row))); + } + } + Ok(rows) +} + +fn expected_rows(case: &Case) -> Vec<(i64, i64, i64)> { + (0..GROUPS) + .map(|grp| { + let ids = case.objects.iter().flat_map(|object| object.ids.clone()).filter(|id| id % GROUPS == grp); + let count = i64::try_from(ids.clone().count()).unwrap_or(i64::MAX); + (grp, count, ids.map(|id| id * 2).sum::()) + }) + .collect() +} diff --git a/crates/tools/src/main.rs b/crates/tools/src/main.rs index 9714aa158..c6b3c27b4 100644 --- a/crates/tools/src/main.rs +++ b/crates/tools/src/main.rs @@ -21,6 +21,10 @@ use e2e::cugraph_bfs_predicate_fixture; use e2e::embedded_backend_e2e; #[cfg(not(test))] use e2e::flight_sql_cugraph_smoke; +#[cfg(all(feature = "iceberg-local-e2e", not(test)))] +use e2e::iceberg_local_e2e; +#[cfg(all(feature = "s3-parquet-e2e", not(test)))] +use e2e::s3_parquet_e2e; #[cfg(all(feature = "server-runtime-observability-e2e", not(test)))] use e2e::server_runtime_observability; @@ -84,6 +88,8 @@ async fn main() -> ToolResult<()> { run_server_runtime_observability_e2e(env::args().skip(2).collect()).await }, Some("embedded-backend-e2e") => run_embedded_backend_e2e(env::args().skip(2).collect()).await, + Some("s3-parquet-e2e") => run_s3_parquet_e2e(env::args().skip(2).collect()).await, + Some("iceberg-local-e2e") => run_iceberg_local_e2e(env::args().skip(2).collect()).await, Some("cugraph-fixture-e2e") => run_cugraph_fixture_e2e(env::args().nth(2)).await, Some("validate-query") => run_validate_query(env::args().skip(2).collect()).await, Some("clickbench-stage-probe") => run_clickbench_stage_probe(env::args().skip(2).collect()).await, @@ -118,6 +124,8 @@ fn print_usage() { cargo run -p tools [--features flight-sql-stress-check] -- flight-sql-stress --config flight-stress-profile.json --out-dir DIR\n\ flock /tmp/cudf-gpu.lock cargo run -p tools --features server-runtime-observability-e2e -- server-runtime-observability-e2e [OPTIONS]\n\ flock /tmp/cudf-gpu.lock cargo run -p tools --features embedded-backend-e2e -- embedded-backend-e2e [--citation-data DIR]\n\ + flock /tmp/cudf-gpu.lock cargo run -p tools --features s3-parquet-e2e -- s3-parquet-e2e\n\ + flock /tmp/cudf-gpu.lock cargo run -p tools --features iceberg-local-e2e -- iceberg-local-e2e\n\ flock /tmp/cudf-gpu.lock cargo run -p tools --features cugraph-oracle -- cugraph-fixture-e2e \n\ flock /tmp/cudf-gpu.lock cargo run -p tools --features datafusion-baseline-contract-validate -- datafusion-baseline-contract-validate [--out-dir DIR]\n\ cargo run -p tools --features single-node-multi-gpu-validate -- single-node-multi-gpu-validate --config run-profile.json --out-dir run-artifacts --dry-run\n cargo run -p tools --features single-node-multi-gpu-validate -- single-node-multi-gpu-validate --config run-profile.json --out-dir run-artifacts --fixture-preflight\n flock /tmp/cudf-gpu.lock cargo run --release -p tools --features single-node-multi-gpu-validate-run -- single-node-multi-gpu-validate --config run-profile.json --out-dir run-artifacts --measured-run [--nsight-trace PATH]" @@ -227,6 +235,26 @@ async fn run_embedded_backend_e2e(_args: Vec) -> ToolResult<()> { Err("embedded-backend-e2e requires --features embedded-backend-e2e and normal cargo run execution".into()) } +#[cfg(all(feature = "s3-parquet-e2e", not(test)))] +async fn run_s3_parquet_e2e(args: Vec) -> ToolResult<()> { + s3_parquet_e2e::run(args).await +} + +#[cfg(not(all(feature = "s3-parquet-e2e", not(test))))] +async fn run_s3_parquet_e2e(_args: Vec) -> ToolResult<()> { + Err("s3-parquet-e2e requires --features s3-parquet-e2e and normal cargo run execution".into()) +} + +#[cfg(all(feature = "iceberg-local-e2e", not(test)))] +async fn run_iceberg_local_e2e(args: Vec) -> ToolResult<()> { + iceberg_local_e2e::run(args).await +} + +#[cfg(not(all(feature = "iceberg-local-e2e", not(test))))] +async fn run_iceberg_local_e2e(_args: Vec) -> ToolResult<()> { + Err("iceberg-local-e2e requires --features iceberg-local-e2e and normal cargo run execution".into()) +} + #[cfg(all(feature = "cugraph-oracle", not(test)))] async fn run_cugraph_fixture_e2e(case: Option) -> ToolResult<()> { match case.as_deref() { diff --git a/scripts/check_all.sh b/scripts/check_all.sh index 1ce31dedb..ae9735caf 100755 --- a/scripts/check_all.sh +++ b/scripts/check_all.sh @@ -209,7 +209,7 @@ FEATURE_ISOLATION_SHARDS=( "algeon-query-runtime algeon-query-planning" "algeon-query-engine algeon-cudf algeon-cudf-sys algeon-build-support" "algeon-cuml algeon-cuml-sys" - "algeon-server algeon-cugraph-sys" + "algeon-server algeon-iceberg algeon-cugraph-sys" "examples bench algeon-cugraph algeon-cuvs algeon-cuvs-sys" ) diff --git a/website/blog/decision-model-in-the-loop.md b/website/blog/decision-model-in-the-loop.md index 68cac8326..36eee428b 100644 --- a/website/blog/decision-model-in-the-loop.md +++ b/website/blog/decision-model-in-the-loop.md @@ -28,38 +28,48 @@ export const dior = twins.clusters.find(row => row.canonical_name === 'Christian export const estimate = twins.full_pairwise_cost_estimate; export const unjudged = all.ranking.filter(row => row.purified_rank !== null && row.purified_rank <= 120 && row.band === null).length; -The [MerRec case study](/guides/datasets/merrec#bridges) ranks 5,000 brands -by betweenness centrality in a graph of shared fans, to find the brands that -connect shopping worlds. Twelve of the first twenty entries in that -ranking are retailers (Walmart, JCPenney, Macy's), house brands (Mainstays) -and catch-all labels that sellers pick from the brand dropdown (As Seen on -TV, Customized & Personalized, Eyeshadow). Shoppers from every world like -each of them, which makes each one a short cut between worlds that no -manufacturer provides. This post follows what it takes to get those entries -out of the ranking, and what changes when they are gone. +In the [MerRec case study](/guides/datasets/merrec#bridges), we rank 5,000 brands +by betweenness centrality in a graph of shared fans to find the brands that +connect different shopping communities. But when you run this on real marketplace +data, you run into an immediate catalog problem: twelve of the top twenty entries +aren't independent brands at all. They are retailers (Walmart, JCPenney, Macy's), +store brands (Mainstays), or catch-all bucket labels that sellers select from +the brand dropdown (As Seen on TV, Customized & Personalized, Eyeshadow). + +Because shoppers across every category buy from Walmart, Walmart creates artificial +shortcuts between worlds that no actual manufacturer provides. If you want an +honest picture of brand affinity, you have to get those catalog artifacts out of the +graph. This post walks through how we pull them out, why naive filtering fails, and +how the ranking shifts once the graph is rebuilt. {/* truncate */} -A decision model does part of the work. Telling a catch-all label from a -brand is a judgment about catalog text, and a hosted decision model -([Jev 1.13](https://docs.typesafe.ai/model-jaggedness/jev-1.13)) answers it -for a fraction of a cent. The rest falls on the query engine: the graph has -to be rebuilt without the rejected entries, and the rebuilt graph has to be -checked against something other than the model. On Algeon a recompute over -all 182.9 million like events takes nine seconds, which leaves room for a -control run, a second window and a second pass. +A decision model does part of the work. Distinguishing a catch-all label from a +real brand requires subjective judgment about messy catalog text, and a hosted +decision model ([Jev 1.13](https://docs.typesafe.ai/model-jaggedness/jev-1.13)) +can answer that for a fraction of a cent. But the heavy lifting still belongs to the +query engine: we have to rebuild the entire graph without the rejected nodes, and +we need rigorous controls to verify what the model actually changed. On Algeon, +recomputing the graph across all 182.9 million like events takes just nine seconds, +making it practical to run iterative passes, evaluate alternative time windows, and +benchmark against baseline controls. ## Deleting from the output is not enough -Betweenness counts how often a node lies on the shortest paths between -other nodes. Walmart's score comes from paths that run through Walmart, and -so does part of every other node's score, because those same paths decide -which brands are on the way between two worlds. Removing Walmart from the -result table leaves every other number as it was. Removing Walmart from the -graph moves the paths, and with them the ranking. +The most tempting shortcut is post-filtering: just run the query and add a +`WHERE brand NOT IN (...)` on the final results. But graph metrics do not work +that way. -Because of that, the model's answers go into the query. The published query has one CTE that -builds the fan graph. The rebuilt query joins that CTE to a table of -decisions and drops the entries the policy rejected or held for review. +Betweenness centrality measures how often a node sits on the shortest paths +between pairs of other nodes. When Walmart is in the graph, millions of shortest +paths route straight through it. That inflates Walmart's score, but it also +depresses or distorts the centrality of every other brand around it. Simply +deleting Walmart from the final output table leaves everyone else's distorted +scores untouched. To get true shortest paths, you must remove the node from the +graph itself and let the network reroute. + +Because of that, the model's decisions have to feed back into the query. The original +query uses a CTE to construct the fan graph; the rebuilt query joins that CTE against +a decisions table, pruning any node that the policy rejected or flagged for review. A probe query bounds the set of entries worth asking about and @@ -77,36 +87,35 @@ Two of the five steps in the figure are ordinary SQL queries. the original query joined to that table.} /> -The probe is the published bridges query itself, and its output is the top -120 entries. A context query joins each of them to its catalog footprint: -listings, distinct categories, the three largest category paths, mean price -and four sample titles. The model sees that text and no graph numbers. - -For each entry the model answers three questions, each with a fixed shape. -The first picks one of four kinds (standalone brand, multi-brand retailer, -superstore house brand, generic catch-all) and returns a probability for -the pick. The second returns a probability for a yes-or-no question, whether -the entry is a standalone brand. The third returns a score from 0 to 3 for -how much the entry conflates unrelated communities. The application's policy -reads the second answer and checks it against the first: -{bridges.policy.accept_min.toFixed(2)} or more with the kind `standalone_brand` -is accepted, below {bridges.policy.review_min.toFixed(2)} is rejected, and -everything else goes to a review queue, including a high probability paired -with another kind. Review entries stay out of the graph until a person looks -at them. - -Every answer is cached in a registry keyed by a hash of the entry's context -text, and a later run over the same entry makes no call. The model's answers -to the same entry vary slightly between calls; without the registry a rerun -would be a new sample. +Calling an LLM on every one of the 5,000 brands would be wasteful and slow. +Instead, we use a SQL probe, which is just the original bridges query, to identify +the top 120 nodes dominating centrality. A second SQL query then gathers catalog +context for each candidate: active listings, distinct categories, top category paths, +mean price, and four representative titles. The model only inspects this catalog text, +never the graph topology or centrality scores. + +For each candidate, the model answers three structured questions: +1. Classification into one of four kinds (standalone brand, multi-brand retailer, superstore house brand, generic catch-all) with a confidence probability. +2. Direct probability for a binary check: is this a standalone brand? +3. A score from 0 to 3 indicating whether the entry conflates unrelated communities. + +Our application policy cross-checks these answers deterministically: a probability of +{bridges.policy.accept_min.toFixed(2)} or higher paired with `standalone_brand` is accepted, +anything below {bridges.policy.review_min.toFixed(2)} is rejected, and ambiguous cases +(such as high probability alongside an unexpected kind) route to a human review queue. +Items in the review queue remain excluded from the graph until manually confirmed. + +To keep queries deterministic and inexpensive, every decision is cached in a registry +keyed by the hash of the catalog context. Because LLM responses can fluctuate across runs, +caching ensures that subsequent recomputes are stable, reproducible, and incur zero model latency. ## Two passes rebuild the ranking -The first pass judged the published top 120 and the recompute dropped the -rejected and review entries. The rebuilt top 120 contained {all.passes[1].new_entries} -entries the first pass had never seen, so a second pass judged those and -recomputed once more. That recompute brought {unjudged} further entries into -the top 120 that no pass has judged; a third pass would take them. +When we ran the first pass and dropped the rejected entries, the graph recomputed, and +{all.passes[1].new_entries} new brands surfaced into the top 120 that were previously +masked by the superstores. A second pass judged those newcomers and recomputed once more, +leaving only {unjudged} unjudged entries at the tail of the top 120 (which a third pass +could clean up if needed).
@@ -121,119 +130,113 @@ the top 120 that no pass has judged; a third pass would take them. and it cannot place Bella, which was not in the original top 120.} /> -Samsung moves from sixth to first with -{(byName('Samsung').purified_betweenness / byName('Samsung').original_betweenness).toFixed(1)} -times its betweenness. Conair rises from {byName('Conair').original_rank} to -{byName('Conair').purified_rank} and Black & Decker from -{byName('Black & Decker').original_rank} to {byName('Black & Decker').purified_rank}. -Two hardware brands moved near the top of a ranking of the brands that -connect shopping worlds, once the superstore labels left the graph. - -An entry lands in the review queue when the model's two answers disagree or -the probability sits in the band. Claire's has a standalone probability of -{claires.is_standalone_brand.toFixed(2)} and a kind of `multi_brand_retailer` at -{claires.entity_kind_p.toFixed(2)}; the policy excludes it from the graph and lists -it for a person. The {all.review_queue.length} entries -in the queue from both passes, with their context, are in the data file. - -## A list and a second window - -One control is a hand-written list of forty-nine retailer and catch-all -names for this window, run through the same recompute. The list -and the model exclude {all.denylist_vs_model.overlap} of the same entries. The -model also excludes {all.denylist_vs_model.only_model.length} the list missed -(Bioworld, Custom Accessories, Pink Rose and nine more), and the list -excludes {all.denylist_vs_model.only_denylist.length} the model accepted (Charming -Charlie, Cherokee, Loot Crate, Rue21). The two purified rankings share 13 of -their top 15. The structural effect, the hardware brands rising, is the same -either way. The model produces the list more cheaply than a person and -marks in the review queue where it is unsure. - -The other check is a second window. On September to October the probe -returns a different top 120, {sepOct.totals.registry_hits} of the -{sepOct.totals.entities} lookups across both passes hit the registry, and the -remaining {sepOct.totals.api_calls} calls cost {usd(sepOct.totals.cost_usd)}. The -hand list, written for the first window, has {sepOct.denylist_vs_model.denylist_unmatched.length} -names that do not appear in this window's top 120 and misses -{sepOct.denylist_vs_model.only_model.length} entries the model rejects, among them -Amazon, Sephora and QVC. The list fits one ranking, while the model answers -for whatever the probe returns next. - -## The same loop merges split brands - -The [brand twins query](/guides/datasets/merrec#brand-twins) pairs entries -whose fans overlap. Many of the highest pairs are one brand entered twice, -TXT and Tomorrow X Together, Bape and A Bathing Ape, Dior and Christian -Dior, and every fan count for such a brand is computed on a fragment. A -string comparison misses the acronyms and the rebrands. Asking a model about -all {n(estimate.pairs)} pairs of 20,000 names is not an option either; at this -run's cost per decision it would come to about -${Math.round(estimate.estimated_cost_usd).toLocaleString('en-US')}. - -The probe keeps the question affordable. The pairing query, with -the rejected entries of the bridges run removed from the fan graph, returns -the 120 pairs with the highest fan Jaccard among the 2,000 most liked -brands. The model judged those 120 with each pair's score and both entries' -catalog context, for {usd(twins.decisions[0].cost_usd)} in -{twins.decisions[0].wall_s.toFixed(1)} s. The policy merges {twins.band_counts.accept}, -holds {twins.band_counts.review} for review and leaves {twins.band_counts.reject} -apart. The pairs left apart include the ones a fan-overlap rule alone would -merge: Hello Kitty and Sanrio, Konami and Yu-Gi-Oh!, Panini and Topps. - -The accepted pairs are edges. `cugraph_weakly_connected_components` assigns -each entry the smallest id of its component, and a scan of every like event -counts, per merged brand, the exact number of distinct users who liked any -member. +The structural impact on the ranking is striking. Samsung jumps from sixth to first place, +multiplying its betweenness centrality by {(byName('Samsung').purified_betweenness / byName('Samsung').original_betweenness).toFixed(1)}x. +Even more telling is what happens to personal care and tool makers: Conair leaps from +rank {byName('Conair').original_rank} to {byName('Conair').purified_rank}, and Black & Decker +climbs from {byName('Black & Decker').original_rank} to {byName('Black & Decker').purified_rank}. +Once artificial retail hubs are cleared out, practical household and hardware brands emerge +as the true bridges connecting disparate shopping circles. + +The review queue catches borderline cases where automatic decisions would be risky. +For instance, Claire's received a standalone brand probability of {claires.is_standalone_brand.toFixed(2)}, +yet its classification picked `multi_brand_retailer` at {claires.entity_kind_p.toFixed(2)}. +Because the signals conflicted, our policy safely held it out of the graph and flagged it +for human review. Across both passes, {all.review_queue.length} entries entered this queue. + +## Validating with a hand list and a second window + +Whenever you insert an LLM into an analytical pipeline, healthy engineering skepticism +demands two sanity checks: +1. Could we have achieved the same result with a simple hand-curated denylist? +2. Does the model generalize across time, or is it brittle to changing data? + +To test the first question, we compared the model against a curated list of forty-nine +retailers and catch-all labels. Recomputing with either approach yields almost the same +macro outcome: 13 of the top 15 purified brands are identical, and hardware brands rise in +both. But the differences highlight where automation shines: the model caught +{all.denylist_vs_model.only_model.length} subtle catch-alls that the human list missed +(such as Bioworld, Custom Accessories, and Pink Rose), while explicitly flagging uncertain +brands in the review queue rather than forcing a silent guess. + +The second check tested temporal generalization using a September–October window. +Because shopping behavior evolves, the probe surfaced a different top 120. Here, our +cached registry paid off: {sepOct.totals.registry_hits} of {sepOct.totals.entities} lookups +were instant cache hits, and the remaining {sepOct.totals.api_calls} calls cost just +{usd(sepOct.totals.cost_usd)}. Meanwhile, the static human list began to decay immediately—it +contained {sepOct.denylist_vs_model.denylist_unmatched.length} names absent from the new +window and failed to flag {sepOct.denylist_vs_model.only_model.length} seasonal retailers +like Amazon, Sephora, and QVC. A static list quickly goes stale; the query loop adapts +dynamically to whatever the probe returns. + +## Solving entity fragmentation with the same loop + +The exact same loop pattern tackles another classic data engineering headache: duplicate +and fragmented entities. In marketplace catalogs, brands frequently exist under multiple +variations, such as TXT vs. Tomorrow X Together, Bape vs. A Bathing Ape, Dior vs. Christian Dior. +Every analytical metric computed on those brands is splintered across fragments. + +Pure string matching cannot resolve acronyms or rebrands, while prompting a model on all +{n(estimate.pairs)} possible pairs across 20,000 brands is economically out of the question, +costing roughly ${Math.round(estimate.estimated_cost_usd).toLocaleString('en-US')}. + +Once again, a SQL probe makes the problem tractable. We run a pairing query that computes +fan Jaccard similarity across the top 2,000 brands, narrowing the candidate pool down to the +120 highest-overlap pairs. The model then evaluates only those 120 candidates using their +catalog metadata, completing in {twins.decisions[0].wall_s.toFixed(1)} seconds for just +{usd(twins.decisions[0].cost_usd)}. The policy accepts {twins.band_counts.accept} pairs, +flags {twins.band_counts.review} for review, and rejects {twins.band_counts.reject}. +Crucially, the model avoids merging related entities that share heavy fan overlap but are +distinct commercial entities, such as Hello Kitty and Sanrio, or Konami and Yu-Gi-Oh!. + +We treat the accepted pairs as edges in a graph. Running `cugraph_weakly_connected_components` +in SQL assigns each cluster a canonical identifier, and an aggregation over like events calculates +the true deduplicated audience size.
PassEntriesNewAcceptReviewRejectModel callsCostWall
{twins.clusters.slice(0, 6).map(row => )}
BrandMost fans on one idBoth ids summedDistinct fansGain
{row.canonical_name}{n(row.primary_fans)}{n(row.naive_sum_fans)}{n(row.exact_union_fans)}+{row.gain_pct.toFixed(1)}%
-Whichever id a report used for Dior, it was missing about a third of the -brand's fans, and a report that added the two ids would have counted a -quarter of its total twice. These are exact counts over every like event, -and the whole table of {twins.clusters.length} merged brands is on the +The correction is substantial. If an analytics report only tracked the primary ID for Dior, +it missed nearly a third of the brand's total followers. On the other hand, if a naive script +summed both IDs without deduplication, it double-counted a quarter of the audience. +The full table of {twins.clusters.length} unified clusters is available on the [case study page](/guides/datasets/merrec#brand-twins). -The control here is trigram similarity over all 20,000 names. It finds -{twins.trigram_control.accepted_found} of the {twins.trigram_control.accepted_aliases} -pairs the policy merged and misses TXT, Bape, Dior, LOFT, HERMES and -Thierry Mugler. The pairs it proposes that the policy did not merge are -sub-brands and near names, Air Jordan and Jordan, Crystal and Crystal Rock. - -## What the model is for - -The model answers a bounded set of questions about catalog text, with a -probability the application can act on, and the GPU does the rest: the probe -that bounds the set, the recompute that turns the answers into a new graph, -and the controls that show how much of the change depends on the model's -particular answers. - -- The probe limits the cost. Judging the 120 pairs it returned cost under a - cent; judging every pair of 20,000 names would cost about ten thousand - dollars. -- The recompute is cheap enough to run several times. A control list, a - second window and a second pass were each one more query, nine seconds on - the full window and four on the shorter one, and every recompute ran - natively on the GPU with zero host boundaries. -- The answers are typed. A choice with a probability, a probability for a - question, a score. The policy is a few lines the application owns, and a - person handles the review band. - -The question set, the policy and the recompute belong to the application, -and the list control shows the result with no model at all. The loop needs -the recompute and the controls, at a cost below the question itself, and it -works with any model that returns typed answers. +As a baseline comparison, standard trigram similarity across all 20,000 brands only recovers +{twins.trigram_control.accepted_found} of the {twins.trigram_control.accepted_aliases} merged +pairs, completely missing non-textual aliases like TXT, Bape, Dior, and LOFT. Furthermore, the +trigram heuristic produces false positives by eagerly merging sub-brands like Air Jordan with +Jordan, or Crystal with Crystal Rock. + +## Practical lessons for decision loops + +The design pattern here reflects a clear separation of concerns between models and engines: +let the model handle fuzzy semantic judgment on small, bounded batches, and let the GPU query +engine handle large-scale filtering, graph algorithms, and recomputation. + +- **Use SQL probes to bound LLM costs.** Evaluating 120 targeted pairs cost less than a cent; + brute-forcing 20,000 names would have cost ten thousand dollars. The query engine acts as + an intelligent funnel. +- **Fast recomputes enable honest validation.** Because rebuilding the 183M-event graph takes + only nine seconds on the GPU without host data transfers, running multi-pass refinements, + historical comparisons, and sanity-check controls is cheap enough to be standard practice. +- **Keep decisions typed and deterministic.** Models should return structured classifications + and confidence scores, not freeform text. Caching decisions by context hash eliminates + nondeterministic query drift, while a review band keeps human oversight where uncertainty is high. + +When the database engine is fast enough to make graph recomputation trivial, you do not need +to compromise with inaccurate post-query hacks. You can feed verified model judgments directly +back into the query loop and compute exact answers. ## Reproduce -Both follow-ups are on the [MerRec case study](/guides/datasets/merrec#bridges) -page with their SQL, tables and provenance. The data files -([bridges](/data/merrec-decision-bridges.json), [twins](/data/merrec-decision-twins.json)) -hold the question sets, the policy, the executed SQL, the stage metrics, -every judgment and every ranking, under the dataset's CC BY-NC terms. The -runner is in the repository under +Both workflows are documented in depth on the [MerRec case study](/guides/datasets/merrec#bridges) +page with their full SQL, data tables, and provenance. The dataset files +([bridges](/data/merrec-decision-bridges.json) and [twins](/data/merrec-decision-twins.json)) +contain all question sets, decision policies, executed queries, and pipeline metrics under CC BY-NC terms. +You can find the runnable runner in the repository under [`fixture/demo/decision_models`](https://github.com/algeon-dev/algeon/tree/main/fixture/demo/decision_models). -A live run needs a model endpoint of your own; its replay mode reruns the -queries from that run's capture with no model calls. +Running it live requires your own model endpoint; you can also use replay mode to rerun the exact +SQL queries against pre-recorded decisions without making external API calls. diff --git a/website/docs/build/docker.md b/website/docs/build/docker.md index 05910ab93..fd8f54163 100644 --- a/website/docs/build/docker.md +++ b/website/docs/build/docker.md @@ -67,7 +67,7 @@ It cannot create their cuDF, cuVS, cuML, or cuGraph artifacts itself. ::: -The image build selects `cugraph`, `cuvs`, and `cuml` by default. NVML device +The image build selects `cugraph`, `cuvs`, `cuml` and `iceberg` by default. NVML device diagnostics are opt-in through `FEATURES`. A `FEATURES=cuml` payload also stages the canonical cuVS runtime dependency. @@ -175,14 +175,14 @@ bash docker/build-image.sh ``` The default image carries the Flight SQL server with cuVS, cuGraph and cuML -SQL. An unset `FEATURES` selects that list; `FEATURES=''` selects no optional -feature and builds the cuDF-only server. Add `nvml` to any list for device +SQL and Iceberg catalogs. An unset `FEATURES` selects that list; +`FEATURES=''` selects no optional feature and builds the cuDF-only server. Add `nvml` to any list for device diagnostics: ```bash FEATURES='' bash docker/build-image.sh FEATURES=nvml bash docker/build-image.sh -FEATURES=cugraph,cuml,cuvs,nvml bash docker/build-image.sh +FEATURES=cugraph,cuml,cuvs,iceberg,nvml bash docker/build-image.sh ``` The tag is `-` for the default feature set, such as @@ -190,8 +190,8 @@ The tag is `-` for the default feature set, such as any other feature set appends its sorted features joined by `-`. The builds above produce `algeon-server:0.1.2-x86-64-v3-cudf`, `algeon-server:0.1.2-x86-64-v3-nvml` and -`algeon-server:0.1.2-x86-64-v3-cugraph-cuml-cuvs-nvml`, and leave the default -image in place. The build record, payload manifest and `dev.algeon.features` +`algeon-server:0.1.2-x86-64-v3-cugraph-cuml-cuvs-iceberg-nvml`, and leave the +default image in place. The build record, payload manifest and `dev.algeon.features` label record the selected list, which is empty for `FEATURES=''`. `IMAGE_REPOSITORY` replaces `algeon-server`. The image labels record the version, source revision, whether the worktree had uncommitted changes, @@ -211,7 +211,8 @@ cuGraph execution is requested. Set `ALGEON_SERVER_CUGRAPH_ENABLED=false` to hide it. The script always runs its own clean build of `algeon_server`, from an -isolated working directory and a clean `CARGO_HOME`, with the fixed target +isolated working directory and a clean `CARGO_HOME` (an `iceberg` build +fetches iceberg-rust from GitHub into it), with the fixed target triple and `target-cpu=x86-64-v3` from `.config/policies/cpu-packaging.toml`. That file also fixes the tool paths `/usr/bin/gcc-14`, `/usr/bin/g++-14`, `/usr/bin/ar`, `/usr/bin/cmake` and `/usr/bin/ninja`. The build shell and @@ -220,7 +221,7 @@ Cargo `[env]` do not apply, so those paths must exist as described in It writes a build record (binary hash, policy fingerprint, features, target triple, codegen flags, version, source revision and CUDA toolkit release) beside the binary. An ordinary `target/release` build has no such record and -is not reused. A packaging build with `FEATURES=cugraph,cuml,cuvs,nvml` +is not reused. A packaging build with `FEATURES=cugraph,cuml,cuvs,iceberg,nvml` typically takes 7 to 8 minutes with a warm native prefix and compiler cache, or 3 minutes when built with `FEATURES=nvml`. diff --git a/website/docs/build/overview.md b/website/docs/build/overview.md index 953d077d7..08f3d435b 100644 --- a/website/docs/build/overview.md +++ b/website/docs/build/overview.md @@ -100,6 +100,12 @@ build-support crate follow the [workspace and publication contracts](https://github.com/algeon-dev/algeon/blob/main/scripts/test-harness.md#workspace-and-publication-contracts). Platform and toolchain requirements are in [Prerequisites](./prerequisites.md). +`algeon-iceberg` has `publish = false` and is the only crate that depends on +iceberg-rust. The root manifest pins iceberg-rust to a Git revision of +`apache/iceberg-rust` (the one `apache/datafusion-iceberg` pins). A build of +`algeon-iceberg`, directly or through the `iceberg` feature of `algeon-server`, +therefore fetches that revision from GitHub and needs network access to it. + ## Build order Each step of the full-feature build is one command in diff --git a/website/docs/build/tests.md b/website/docs/build/tests.md index 85862dceb..565ec3799 100644 --- a/website/docs/build/tests.md +++ b/website/docs/build/tests.md @@ -299,6 +299,8 @@ itself. | TPC-H execution report | `ALGEON_PARQUET_DIR=/path/to/complete/tpch-sf1 flock /tmp/cudf-gpu.lock cargo run -p bench --all-features --bin native_tpch_execution_report` | A complete TPC-H dataset | | Sustained-load goodput (`query_service_goodput`) | `crates/bench/scripts/run_goodput_matrix.sh calibrate g1` | The GPU and dataset recorded by `calibrate` | | Python-cuGraph comparison | `flock /tmp/cudf-gpu.lock cargo run -p tools --all-features -- cugraph-fixture-e2e bfs-include-edges` | A host with Python cuGraph installed | +| S3 Parquet scans | `ALGEON_S3_PARQUET_E2E_BUCKET= flock /tmp/cudf-gpu.lock cargo run -p tools --features s3-parquet-e2e -- s3-parquet-e2e` | An existing bucket and credentials from the default AWS chain. `ALGEON_S3_PARQUET_E2E_REGION` defaults to `AWS_REGION`, then `us-east-1` | +| Iceberg REST and Glue catalogs | `AWS_ACCESS_KEY_ID=algeon-e2e AWS_SECRET_ACCESS_KEY=algeon-e2e-secret flock /tmp/cudf-gpu.lock cargo run -p tools --features iceberg-local-e2e -- iceberg-local-e2e` | The services of `crates/tools/fixtures/iceberg-local/compose.yaml` | - Scale-factor workloads validate their dataset before starting and fail when it is incomplete. The TPC-DS report emits one row per attempted query. @@ -312,6 +314,33 @@ itself. `bfs-utf8-path`, or `bfs-utf8-predicates` for the other provisioned cases. - External-service E2E tests are invoked explicitly through `cargo run -p tools --all-features`. +- `s3-parquet-e2e` uploads generated Parquet objects under a unique key prefix, + scans one object, a prefix of three objects and one object split into byte + ranges in library and server-configured sessions, compares each result with + the generated data, and deletes the objects. Setting + `ALGEON_S3_PARQUET_E2E_ENDPOINT` selects an S3-compatible service with + path-style addressing; without it the run uses real S3. +- `iceberg-local-e2e` writes Iceberg tables through a REST catalog and a Glue + catalog, serves both from an in-process Flight SQL server in + `native_required` mode, and checks the Flight SQL catalog, schema and table + listings, a native aggregate over each catalog and the + `unsupported_table_feature` rejection of a snapshot with a position-delete + file. Its compose file adds the Iceberg REST fixture on + `http://127.0.0.1:8181` and moto as Glue on `http://127.0.0.1:5001` to the + RustFS service below. Both compose files bind port 9000; stop one stack + before starting the other. + +A local RustFS service for that run starts from the compose file in +`crates/tools/fixtures/s3-local`. It listens on `http://127.0.0.1:9000` with +bucket `algeon-e2e`, access key `algeon-e2e` and secret key `algeon-e2e-secret`. + +```bash +docker compose -f crates/tools/fixtures/s3-local/compose.yaml up -d --wait +ALGEON_S3_PARQUET_E2E_ENDPOINT=http://127.0.0.1:9000 ALGEON_S3_PARQUET_E2E_BUCKET=algeon-e2e \ +AWS_ACCESS_KEY_ID=algeon-e2e AWS_SECRET_ACCESS_KEY=algeon-e2e-secret \ + flock /tmp/cudf-gpu.lock cargo run -p tools --features s3-parquet-e2e -- s3-parquet-e2e +docker compose -f crates/tools/fixtures/s3-local/compose.yaml down -v +``` For any performance claim, use `--release` builds, compare rows produced by the same harness, and label warmup count, iteration count, source cache diff --git a/website/docs/design/data-path.mdx b/website/docs/design/data-path.mdx index da5d04fe4..1b132f3d1 100644 --- a/website/docs/design/data-path.mdx +++ b/website/docs/design/data-path.mdx @@ -19,9 +19,9 @@ Scans prune unneeded data across multiple granularities: | Unit | How the scan is reduced | | --- | --- | -| Files | Native lowering preserves the file set supplied by DataFusion's scan, including upstream partition pruning. | +| Files | Native lowering preserves the file set supplied by DataFusion's scan, including upstream partition pruning. An [Iceberg table](../integration/overview.mdx#iceberg-support) reaches lowering as a standard `s3://` Parquet scan, and Iceberg manifest pruning has already reduced its file set. | | Column chunks | Projection keeps only columns referenced by downstream operators and predicates. | -| Row groups | Lowering evaluates footer statistics (min/max and null counts) and Bloom filters to prune row groups for supported predicates. | +| Row groups | Lowering evaluates footer statistics (min/max and null counts) and Bloom filters to prune row groups for supported predicates. An `s3://` scan prunes at planning time only when the session [holds its footers](#object-storage-and-libcurl). | | Rows | Predicates evaluate within the cuDF reader or a downstream GPU filter, reducing output rows. | When a predicate is pushed into the cuDF reader, statistics filters reduce row @@ -47,7 +47,9 @@ source chunks can then be batched into a reader call. The reader selects the column byte ranges, and KvikIO divides reads into I/O tasks on its shared pool. One source chunk can therefore require many I/O tasks. The I/O task size controls the read subdivision; the source-chunk target still -comes from the query's memory policy. +comes from the query's memory policy. Throughput grouping also widens chunks +of `s3://` files that carry planning-time row-group byte metadata from cached +footers. A chunk never spans files. Every planned read, and every batch of chunks read together, decodes into one cuDF table. Its row count and the child value count of each list column @@ -131,6 +133,21 @@ transfers via cuObject RDMA are not supported. Footer reads and cache fills for `s3://` sources run on a process-wide `algeon-object-store-control` Tokio runtime ([Runtimes and threads](./index.mdx#runtimes-and-threads)). +A physical optimizer rule cannot wait on the network, so planning never reads +a remote footer from the store. It reads the footer of an `s3://` object from +the session's DataFusion file metadata cache, which DataFusion's `ListingTable` +fills during its async scan when `datafusion.execution.collect_statistics` is +true (the default). Planning reaches that cache through its invocation, so it +exists for `AlgeonAttempt` planning, Flight SQL server statements, +`plan_with_native_diagnostics` and `validate_query`. A plain `ctx.sql(...)` on +a session from `install_on` plans without an invocation and has no remote +footers. DataFusion keys the cache by the object path inside the store, without +the bucket, and validates an entry by size and last-modified time. Algeon also +refuses an entry whose e_tag or version differs from the scanned object's when +both have one. A scan of several remote objects with a missing or refused +footer is rejected with `native_source_remote_footer_unavailable` +([remedies](../reference/gpu-coverage-report.mdx#remedies-for-sources)). + KvikIO's [remote transfer implementation](https://github.com/rapidsai/kvikio/blob/0f6900f08c5a7ca150689e5454ae28562f875eec/cpp/src/remote_handle.cpp#L37-L133) reflects this host staging and device copy sequence. @@ -143,7 +160,10 @@ different part of the next read. - The OS page cache holds file pages from buffered `pread`. A hit skips the NVMe read; host staging, the device copy and decoding still run. - Algeon's source chunk planner caches Parquet footer metadata within one - planning pass. Each query plan reads footers again. + planning pass, and each query plan reads local footers again. Remote footers + come from the session's DataFusion file metadata cache, which outlives the + query and is keyed by object path without the bucket + ([Object storage](#object-storage-and-libcurl)). - Decoded local-Parquet sources and constructed graphs can stay on the device under `cache_cap_bytes` ([Admission settings](../integration/configuration.mdx#admission)). The default is `0`, so queries share nothing on the GPU unless a server sets diff --git a/website/docs/design/index.mdx b/website/docs/design/index.mdx index 425bb1700..702bea672 100644 --- a/website/docs/design/index.mdx +++ b/website/docs/design/index.mdx @@ -51,10 +51,13 @@ their dependencies. Ownership across the query crates is: | `query-planning` | IR, optimizer, capability, proof, admission contracts and compiled pipeline | `crates/algeon-query-planning` | | `query-runtime` | Attempt lifecycle, query service, device ledger, immutable grants, cache, observation and metrics | `crates/algeon-query-runtime` | | `query-engine` | Native execution composition | `crates/algeon-query-engine` | +| `algeon-iceberg` | Iceberg REST and Glue catalogs as DataFusion catalogs, and each table as a Parquet scan of its current snapshot with the `unsupported_table_feature` rejections | `crates/algeon-iceberg` | The adapter imports each query crate directly. The engine imports planning and runtime. Runtime imports planning. These three query crates have no DataFusion dependency. Imports come from the crate that owns the item. +`algeon-iceberg` imports the adapter and iceberg-rust; only `algeon-server` +imports it, behind its `iceberg` feature. Below them, the GPU component crates provide Rust APIs over NVIDIA's RAPIDS libraries: diff --git a/website/docs/design/native-lowering.mdx b/website/docs/design/native-lowering.mdx index 415b51be3..d28e610ec 100644 --- a/website/docs/design/native-lowering.mdx +++ b/website/docs/design/native-lowering.mdx @@ -269,6 +269,33 @@ operator, at the site that rejected it. A missing memory estimate cannot reject a candidate. Allocation limits are enforced during [execution](./admission.mdx#allocation-enforcement). +### Parquet source schema {/* #parquet-source-schema */} + +Lowering reads the footer of every distinct object in a Parquet scan, local or +remote, and counts byte-range entries of one object once. All footers must +hold the same columns in the same order with the same Arrow types; metadata on +nested fields is ignored. A column may be required in one file and optional in +another when the table schema declares it nullable. Any other difference, +including a file that declares a column optional while the table declares it +non-null, rejects the scan with `native_source_file_schema_mismatch`. The +source diagnostics then report `schema_mapping_disposition` as `rejected` with +rejected feature `schema_mapping`. Column indices, byte metadata and +row-group statistics follow the footer's column order, which can differ from +DataFusion's file schema. A remote scan of one object whose footer is not +cached takes its schema from DataFusion's file schema and gets no footer +pruning or byte metadata. + +Byte ranges that cover a whole object coalesce on any store. A partial range +maps to row groups through the footer, so a remote partial range needs the +cached footer described under +[Object storage](./data-path.mdx#object-storage-and-libcurl). A remote scan +without its footers is rejected with `native_source_remote_footer_unavailable` +and rejected feature `remote_footer_unavailable`. A rejected remote scan still +reports the read strategy and credential mode of the read it would make, such +as `remote_direct` and `shared_process_env`. Both reasons carry a +[remedy](../reference/gpu-coverage-report.mdx#remedies-for-sources) and a +[`source` blocking subject](../reference/gpu-coverage-report.mdx#what-blocked-a-candidate). + ### Uncorrelated scalar subqueries {/* #uncorrelated-scalar-subqueries */} DataFusion plans an uncorrelated scalar subquery as a `ScalarSubqueryExec` diff --git a/website/docs/integration/configuration.mdx b/website/docs/integration/configuration.mdx index 463a5e9ba..d5bbbaf8c 100644 --- a/website/docs/integration/configuration.mdx +++ b/website/docs/integration/configuration.mdx @@ -1,6 +1,6 @@ --- sidebar_position: 5 -description: Configure the Flight SQL server with one TOML file and derived environment overrides, covering admission, native execution, authentication, TLS, observability, startup SQL and the command line. +description: Configure the Flight SQL server with one TOML file and derived environment overrides, covering admission, native execution, S3 storage, Iceberg catalogs, authentication, TLS, observability, startup SQL and the command line. --- # Configuration @@ -22,8 +22,8 @@ setting has one TOML location, and its environment variable is `ALGEON_SERVER_` followed by the TOML path in upper case with `.` written as `_`: `native.execution_mode` becomes `ALGEON_SERVER_NATIVE_EXECUTION_MODE`. Only strings, integers and booleans have an environment form. The -`[[admission.device_profiles]]` and `[[auth.mode.users]]` arrays live in the -file only. An unknown TOML field stops startup with `invalid_configuration` and the +`[[admission.device_profiles]]`, `[[iceberg.catalogs]]` and +`[[auth.mode.users]]` arrays live in the file only. An unknown TOML field stops startup with `invalid_configuration` and the error lists the table's valid keys; an unknown `ALGEON_SERVER_*` variable stops startup the same way and names the closest known variable when one is similar. @@ -80,7 +80,7 @@ execution_mode = "native_preferred" # ALGEON_SERVER_NATIVE_EXECUTION max_row_groups_per_chunk = 1 # ALGEON_SERVER_NATIVE_MAX_ROW_GROUPS_PER_CHUNK max_retry_attempts = 3 # ALGEON_SERVER_NATIVE_MAX_RETRY_ATTEMPTS max_source_reads_in_flight = 2 # ALGEON_SERVER_NATIVE_MAX_SOURCE_READS_IN_FLIGHT -source_chunk_planning_mode = "local_parquet_throughput" # ALGEON_SERVER_NATIVE_SOURCE_CHUNK_PLANNING_MODE +source_chunk_planning_mode = "parquet_throughput" # ALGEON_SERVER_NATIVE_SOURCE_CHUNK_PLANNING_MODE statistical_aggregate_execution_mode = "gpu_native_tolerant" # ALGEON_SERVER_NATIVE_STATISTICAL_AGGREGATE_EXECUTION_MODE [observability] @@ -105,6 +105,18 @@ max_output_staging_bytes = 1073741824 # ALGEON_SERVER_CUGRAPH_MAX_OUTP [data] # root = "/data" # ALGEON_SERVER_DATA_ROOT +# [s3] # absent: s3:// locations fail +# endpoint = "http://127.0.0.1:9000" # ALGEON_SERVER_S3_ENDPOINT +# region = "us-east-1" # ALGEON_SERVER_S3_REGION +# path_style = true # ALGEON_SERVER_S3_PATH_STYLE +# unsigned = false # ALGEON_SERVER_S3_UNSIGNED + +# [[iceberg.catalogs]] # file only; requires [s3] +# name = "lake" +# kind = "rest" +# uri = "http://127.0.0.1:8181" +# auth = { kind = "none" } + [auth.mode] kind = "disabled" # ALGEON_SERVER_AUTH_MODE_KIND # token_ttl_secs = 3600 # ALGEON_SERVER_AUTH_MODE_TOKEN_TTL_SECS @@ -238,7 +250,7 @@ commit and enabled features. | `max_row_groups_per_chunk` | `1` | | `max_retry_attempts` | `3` | | `max_source_reads_in_flight` | `2` | -| `source_chunk_planning_mode` | `local_parquet_throughput` (or `conservative`) | +| `source_chunk_planning_mode` | `parquet_throughput` (or `conservative`) | | `statistical_aggregate_execution_mode` | `gpu_native_tolerant` (or `datafusion_exact_host_welford`) | - Device capacity comes only from [`[[admission.device_profiles]]`](#admission). @@ -250,9 +262,13 @@ commit and enabled features. - Both targets bound a chunk's decoded size, estimated from the Parquet footer. A dictionary-encoded string column counts at its decoded width, not its encoded size. -- `local_parquet_throughput` lets one chunk of a local Parquet source hold up - to 1,024 row groups within the target when `max_row_groups_per_chunk` is `1`. - Other sources keep one row group per chunk. `conservative` plans +- `parquet_throughput` lets one chunk of a Parquet file hold up to 1,024 row + groups within the target when `max_row_groups_per_chunk` is `1`. It applies + to local files and to `s3://` objects whose footers planning read, and needs + no remote-specific setting: on a g7e.8xlarge reading same-region S3, a `SUM` over the 17.2 GB + TPC-DS SF100 `store_sales` file takes 1.1 s against 270 s with one row group + per chunk. A chunk never spans files, so a table of many small remote files + reads one chunk per file. Other sources keep one row group per chunk. `conservative` plans `max_row_groups_per_chunk` row groups per chunk; compatible chunks then coalesce into batched reads under the same target. - When the grant rejects an allocation while a chunk is read, staged through @@ -481,6 +497,115 @@ path for each table, then the total count. The tables live in the [shared catalog](./flight-sql-server.mdx#shared-catalog). Startup SQL can build views over them or replace one with `CREATE OR REPLACE EXTERNAL TABLE`. +## S3 storage {/* #s3-storage */} + +```toml +[s3] +endpoint = "http://127.0.0.1:9000" +region = "us-east-1" +path_style = true +``` + +An `[s3]` table lets `CREATE EXTERNAL TABLE`, [startup SQL](#startup-sql) +and queries read `s3://` Parquet locations. Without the table those +locations fail. An empty `[s3]` enables them with the defaults below, and so +does setting any `ALGEON_SERVER_S3_*` variable. Native reads, footer pruning +and DataFusion scans of `s3://` sources all use these settings. + +| `[s3]` field | Default | Environment variable | +|---|---|---| +| `endpoint` | Unset: AWS S3, or the endpoint the AWS environment names | `ALGEON_SERVER_S3_ENDPOINT` | +| `region` | Unset: `AWS_REGION`, then `AWS_DEFAULT_REGION`, then `us-east-1` | `ALGEON_SERVER_S3_REGION` | +| `path_style` | Unset | `ALGEON_SERVER_S3_PATH_STYLE` | +| `unsigned` | `false` | `ALGEON_SERVER_S3_UNSIGNED` | + +- Credentials come from the default AWS credential chain: + `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN`, a web + identity token, then container or EC2 instance profile credentials. The + table has no key fields. +- `unsigned = true` sends anonymous requests and reads no credentials, for + public buckets. +- `path_style = true` addresses objects as `//`; + `false` selects virtual-hosted addressing (`./`). +- A custom `endpoint`, such as MinIO or RustFS, is read with path-style + addressing only. Startup and `--check-config` fail with + `invalid_configuration` on `s3.path_style` when `endpoint` is set together + with `path_style = false`. Set `path_style = true` or leave it unset. + +Embedded sessions pass the same settings as an `S3SourceStorageProfile` to +`AlgeonNativeExecutionConfig::with_s3_storage`. + +## Iceberg catalogs {/* #iceberg-catalogs */} + +```toml +[s3] +region = "eu-west-2" + +[[iceberg.catalogs]] +name = "lake" +kind = "rest" +uri = "http://127.0.0.1:8181" +warehouse = "s3://warehouse/" +auth = { kind = "oauth2", credential = "client-id:client-secret" } + +[[iceberg.catalogs]] +name = "glue" +kind = "glue" +warehouse = "s3://bucket/warehouse" +``` + +Each `[[iceberg.catalogs]]` entry names an Iceberg catalog that becomes a +DataFusion catalog called `name`. A server binary reads these catalogs only +when it is built with the `iceberg` feature; a server without it refuses a +non-empty `[[iceberg.catalogs]]` at startup. Supported catalogs, scan behavior +and rejected table features are in +[Iceberg support](./overview.mdx#iceberg-support). + +| Key | Kind | Default | Meaning | +|---|---|---|---| +| `name` | Both | Required | DataFusion catalog name | +| `kind` | Both | Required | `rest` or `glue` | +| `uri` | `rest` | Required | Catalog base URI | +| `warehouse` | `rest` | Unset | Warehouse identifier sent to the catalog | +| `prefix` | `rest` | Unset | Path prefix after the catalog's `/v1` segment | +| `auth` | `rest` | `{ kind = "none" }` | Catalog authentication, below | +| `warehouse` | `glue` | Required | Warehouse location, such as `s3://bucket/warehouse` | +| `catalog_id` | `glue` | Unset: the caller's AWS account | Glue catalog account ID | +| `region` | `glue` | Unset: the AWS environment | Glue service region | +| `endpoint` | `glue` | Unset: the AWS Glue endpoint | Glue service endpoint override | + +| `auth.kind` | Keys | +|---|---| +| `none` | None. Requests carry no authorization | +| `oauth2` | `credential` (required), as `client_id:client_secret` or a client secret alone; `server_uri`, the token endpoint, defaulting to the catalog's own; `scope` | +| `token` | `token` (required), a bearer token sent with every request | + +A Glue catalog uses the default AWS credential chain described under +[S3 storage](#s3-storage). + +The `iceberg` table is file-only. An array of tables has no environment form, +and no `ALGEON_SERVER_ICEBERG_*` variable exists. `credential` and `token` +can therefore be set only in the file. `--check-config` prints +`# credential is set` or `# token is set` in their place, and the `Debug` +output of `ServerConfig` prints ``. + +Startup and `--check-config` stop with `invalid_configuration` when: + +- an entry is present without an [`[s3]` table](#s3-storage), which the + server reads Iceberg data files through (reason + `iceberg_catalogs_require_s3`); +- an entry lacks a required key, or holds a key that belongs to the other + kind, to another `auth.kind`, or to neither, such as a property map; +- two entries share a `name` (`duplicate_iceberg_catalog`), or a `name` is + empty or `datafusion`, the default DataFusion catalog + (`iceberg_catalog_name_reserved`); +- a Glue entry sets `endpoint` while `[s3]` names no `endpoint` + (`iceberg_glue_endpoint_requires_s3_endpoint`). iceberg-rust's Glue catalog + then reads table metadata through the Glue endpoint, so set `s3.endpoint`, + for example to the regional S3 endpoint. + +Embedded Rust reads the validated entries from `ServerConfig::iceberg_catalogs`. + ## Startup SQL {/* #startup-sql */} `init_sql` names a SQL file that the server executes on its shared session before diff --git a/website/docs/integration/handling-errors.mdx b/website/docs/integration/handling-errors.mdx index c0698ee72..d6c061136 100644 --- a/website/docs/integration/handling-errors.mdx +++ b/website/docs/integration/handling-errors.mdx @@ -196,9 +196,35 @@ native_required mode rejected a plan that needs DataFusion CPU: GPU execution do ~~~ That status carries `x-algeon-blocking-kind`, from a fixed vocabulary such as -`cast`, `aggregate_function` or `operator` (`column_type` for a column), and -`x-algeon-blocking-subject`, the text before the instruction. Each surface -carries these names under its own [disclosure rules](../design/errors.mdx#disclosure-model). +`cast`, `aggregate_function`, `operator` or `source` (`column_type` for a +column), and `x-algeon-blocking-subject`, the text before the instruction. Each +surface carries these names under its own [disclosure rules](../design/errors.mdx#disclosure-model). + +A rejected Parquet scan carries kind `source`. Its `x-algeon-reason` is the +capability reason, for example `native_source_remote_footer_unavailable` when +planning found no cached footer for an `s3://` object, and the subject reads +like ``Parquet scan of 2 objects in `s3://bucket/` ``. The reason's +[remedy](../reference/gpu-coverage-report.mdx#remedies-for-sources) names the +planning path and settings that fill the footer cache. + +### Unsupported Iceberg table features {/* #unsupported-table-feature */} + +A snapshot the Iceberg scan cannot read correctly fails with +`unsupported_table_feature`, which has kind `unsupported`, status `permanent` +and Flight code `UNIMPLEMENTED`. The check runs in the table provider before +native lowering, so every execution mode raises it. + +The status carries the `reason` as `x-algeon-reason`. Its values are +`iceberg_delete_files`, `iceberg_field_id_mismatch`, +`iceberg_missing_field_ids`, `iceberg_partition_column_absent`, +`iceberg_required_column_absent`, `iceberg_encrypted_file`, +`iceberg_non_parquet_file`, `iceberg_multiple_buckets`, +`iceberg_unsupported_location_scheme`, `iceberg_storage_properties_mismatch` +and `iceberg_column_type_mismatch`. The +[Iceberg support](./overview.mdx#iceberg-rejections) table gives each trigger. +The table, snapshot id, data file path and column are raw facts and stay out of +metadata and logs. A retry fails the same way until the table or the `[s3]` +settings change. ### Rejections in EXPLAIN GPU {/* #rejections-in-explain-gpu */} diff --git a/website/docs/integration/overview.mdx b/website/docs/integration/overview.mdx index bb1f6802c..909448beb 100644 --- a/website/docs/integration/overview.mdx +++ b/website/docs/integration/overview.mdx @@ -27,14 +27,73 @@ executable; see [Dependencies](./datafusion-session.mdx#dependencies). ## Iceberg support -Iceberg tables are not a supported source. The earlier Algeon integration was -removed together with its `iceberg` Cargo feature and its server catalog -settings. A future integration would start from the upstream DataFusion -Iceberg table provider in -[apache/datafusion-iceberg](https://github.com/apache/datafusion-iceberg). -Because the Flight SQL server registers no object store, `s3://` locations -fail there; an embedded session can read `s3://` Parquet through -`AlgeonNativeExecutionConfig::with_s3_storage`. +A server built with the `iceberg` feature registers each +[`[[iceberg.catalogs]]`](./configuration.mdx#iceberg-catalogs) entry as a +DataFusion catalog. Two catalog kinds connect: Iceberg REST catalogs that need +no SigV4 request signing, such as `iceberg-rest-fixture`, Polaris and +Lakekeeper (authentication `none`, OAuth2 client credentials or a bearer +token), and AWS Glue with the default AWS credential chain. Glue's Iceberg REST +endpoint and S3 Tables REST require SigV4, which iceberg-rust does not +implement, so they are not supported. GCP catalogs are out of scope. + +Data files must be in S3. Native scans read them with KvikIO through the +server's [`[s3]` table](./configuration.mdx#s3-storage), and the catalog's +metadata files (table metadata, manifest lists and manifests) are read through +iceberg-rust's FileIO with the endpoint, region and path style of the same +table, which `[[iceberg.catalogs]]` therefore requires. A loaded table whose +FileIO endpoint differs from `[s3]`, or whose region or path style differs +when both sides set one, fails with `iceberg_storage_properties_mismatch`. +Credentials a catalog returns for a table are neither requested nor used. An +embedded session uses `AlgeonNativeExecutionConfig::with_s3_storage`. + +Access is read-only. Every scan loads the table again and plans its current +snapshot; there is no time-travel syntax and no metadata table. A table +without a snapshot scans as empty. Namespaces and table names are listed once, +at server startup, and only single-level namespaces appear. A query loads a table +by name, so a table created after startup can be queried but is missing from +the listings. + +Filters convert to Iceberg predicates that prune manifests and data files. +DataFusion keeps every filter in a `FilterExec` above the scan, so a filter +that does not convert only loses pruning. The scan is a standard Parquet +`DataSourceExec` with one whole-file entry per data file, the exact row count +from the manifest, and each footer placed in the session's file metadata +cache. Native lowering therefore sees a multi-object `s3://` Parquet scan (see +[Object storage and libcurl](../design/data-path.mdx#object-storage-and-libcurl)). +String and binary columns use the view types that a `ListingTable` infers. +Columns bind by name, after a per-file check that name and Iceberg field id +(or the table's name mapping) select the same column. + +### Rejected table features {/* #iceberg-rejections */} + +A snapshot that a Parquet scan would read incorrectly fails with +`unsupported_table_feature`; the +[error page](./handling-errors.mdx#unsupported-table-feature) lists its +metadata. The checks run in the provider's scan before native lowering, so +they apply in every execution mode, and no CPU fallback exists. + +| `reason` | Raised when | +| --- | --- | +| `iceberg_delete_files` | A data file of the snapshot has delete files attached. | +| `iceberg_field_id_mismatch` | In a file that carries field ids, a column's name and its field id select different file columns, as after a rename. | +| `iceberg_missing_field_ids` | A file has no field ids and the table has no default name mapping, or the mapping resolves a column to another name than the file uses. | +| `iceberg_partition_column_absent` | A file lacks an identity-partition column, whose value Iceberg keeps only in partition data. | +| `iceberg_required_column_absent` | A file lacks a required column. A missing optional column reads as null. | +| `iceberg_encrypted_file` | A data file has encryption key metadata. | +| `iceberg_non_parquet_file` | A data file is not Parquet. | +| `iceberg_multiple_buckets` | The data files left after pruning span more than one bucket. | +| `iceberg_unsupported_location_scheme` | A data file location is not `s3://`, `s3a://` or `s3n://`. `file://` locations are read from local disk. | +| `iceberg_storage_properties_mismatch` | The table's FileIO storage properties disagree with `[s3]`, as described above. | +| `iceberg_column_type_mismatch` | A file stores a projected column with another type than the table declares, as after an `int` to `long` promotion. String and binary widths and dictionary encoding count as the same type. | + +Catalog and storage failures keep the existing codes. A missing namespace or +table is `object_not_found`. Credential and access failures from the catalog or +metadata storage are `source_unauthenticated` or `source_access_denied`. A +retryable failure is `source_temporary_failure`, and any other is +`source_dependency_failure`. A table schema that changes between name +resolution and the scan fails with `source_temporary_failure`; a data file +whose size differs from its manifest entry fails with +`source_dependency_failure`. ## Common flow diff --git a/website/docs/intro.mdx b/website/docs/intro.mdx index 86835742f..9f2abd7ed 100644 --- a/website/docs/intro.mdx +++ b/website/docs/intro.mdx @@ -170,9 +170,9 @@ query runs on one GPU, and the limits below follow from that. toward the grant. - GPU scans read Parquet from local disk or S3-compatible storage, read-only, and cover a [subset of SQL](./reference/native-sql-support.mdx) and - [data types](./reference/data-type-support.md). Lakehouse catalog - integrations, beginning with Apache Iceberg, represent the - [next milestone on the roadmap](./reference/roadmap.md#mid-term). + [data types](./reference/data-type-support.md). Iceberg tables from REST and + Glue catalogs read as Parquet scans; see + [Iceberg support](./integration/overview.mdx#iceberg-support). - Indexes, fitted models and graphs exist only while the query that builds them runs. A bounded server can [cache graphs](./cugraph/overview.mdx#reuse-in-bounded-servers). diff --git a/website/docs/reference/benchmarks/index.mdx b/website/docs/reference/benchmarks/index.mdx index 02737b8a5..67545300e 100644 --- a/website/docs/reference/benchmarks/index.mdx +++ b/website/docs/reference/benchmarks/index.mdx @@ -354,7 +354,7 @@ measures execution through result collection, including reads and GPU result transfer. Successful-query SQL and physical planning time are excluded. GPU runs use the `sf100_gpu_throughput` profile with -`local_parquet_throughput` source planning, an explicit 32 GiB +`parquet_throughput` source planning, an explicit 32 GiB `max_source_chunk_bytes` target and at most 1,024 row groups per chunk. The byte target groups source data; the query's separate [memory grant](../../design/admission.mdx#device-profiles-and-grants) limits diff --git a/website/docs/reference/gpu-coverage-report.mdx b/website/docs/reference/gpu-coverage-report.mdx index 144538c2a..564564bf8 100644 --- a/website/docs/reference/gpu-coverage-report.mdx +++ b/website/docs/reference/gpu-coverage-report.mdx @@ -64,6 +64,7 @@ the one-line `text`, and structured fields for that kind: | `expression` | A rejected expression, or one the contract could not narrow further. | `expression`, `data_type` | | `column_type` | A column whose Arrow type is not native. | `column`, `data_type`, `role` | | `ordering` | The declared output ordering, or the input ordering a window needs, that the native plan cannot reproduce. | `required_by`, `ordering` | +| `source` | A Parquet scan the native source contract cannot read. The candidate's `native_source_*` reason names the feature. | `object_store_url`, `object_count` | | `compiled_contract` | A rule of the compiled pipeline or plan validation contract, when no operator or expression is the cause. | `reason`, `boundary` | - An `aggregate_function` defect is the function itself, `DISTINCT`, @@ -204,6 +205,24 @@ inside an executable plan, and the statement is not rejected. - A union on the collected side of a join is merged into one partition, and the setting still removes the stages or the merge there. +### Remedies for sources {/* #remedies-for-sources */} + +| `reason_code` | `remedy_code` (`remedy_kind`) | When it applies | +| --- | --- | --- | +| `native_source_file_schema_mismatch` | `unify_file_schemas` (`user_action`) | The scan's files differ in column names, order or Arrow types, or a file declares optional a column the table declares non-null. | +| `native_source_remote_footer_unavailable` | `cache_remote_footers` (`config_action`) | Planning found no usable footer in the session's file metadata cache for an `s3://` object. | + +- The table scan fills the cache while `datafusion.execution.collect_statistics` + is true. Planning reads it only under an invocation: `AlgeonAttempt` + planning, the Flight SQL server, `plan_with_native_diagnostics` or + `validate_query`. +- A table whose footers exceed `datafusion.runtime.metadata_cache_limit` + (50 MB by default) evicts its own entries during the scan. An embedded + session can raise the limit; the Flight SQL server runs with the default. +- The cache key is the object path without the bucket. An entry is refused when + its size, last-modified time, or e_tag or version (when both sides have one) + differs from the scanned object. + ## Validation context {/* #validated-under */} Validation clones the caller's `SessionState`. If that state has an Algeon diff --git a/website/docs/reference/native-sql-support.mdx b/website/docs/reference/native-sql-support.mdx index 865ea190f..d59d7dfa4 100644 --- a/website/docs/reference/native-sql-support.mdx +++ b/website/docs/reference/native-sql-support.mdx @@ -113,8 +113,9 @@ other distinct aggregates and ordered aggregate arguments are not. Parquet is the native file format, read from local paths or `s3://` object stores. Schema mapping and remote read capability are validated before execution, and a scan that fails either check keeps the DataFusion baseline. -Iceberg tables are not a supported source. See -[Iceberg support](../integration/overview.mdx#iceberg-support). +Iceberg tables from REST and Glue catalogs scan as Parquet; see +[Iceberg support](../integration/overview.mdx#iceberg-support) for the table +features that are rejected. Validate ad-hoc SQL with `algeon_explain_coverage`, or `EXPLAIN GPU` over Flight SQL, against the catalog and server configuration that will execute it. The [coverage page](../integration/gpu-coverage-validation.mdx) shows diff --git a/website/docs/reference/roadmap.md b/website/docs/reference/roadmap.md index d14ea569b..f6fa3fdcb 100644 --- a/website/docs/reference/roadmap.md +++ b/website/docs/reference/roadmap.md @@ -31,9 +31,12 @@ choice. ## Mid term -- Lakehouse sources, Iceberg first, with the Glue and REST catalogs. - `iceberg-rust` and `datafusion-iceberg` are maturing upstream, and Algeon - will integrate them once a release can be pinned. +- Iceberg reads beyond the current Parquet path: tables with delete files, + column binding by field id, tables spanning several buckets, per-table + catalog credentials and SigV4-signed REST catalogs. Today these are + [rejected](../integration/overview.mdx#iceberg-rejections) or unsupported. + `algeon-iceberg` moves from a pinned iceberg-rust Git revision to a + published release once one builds against the workspace's Arrow version. - Build against NVIDIA's own repositories instead of the current forks, so that users do not compile the shared libraries themselves. Each fork change either goes upstream as a pull request, stays as an Algeon-specific feature diff --git a/website/scripts/generate_tpcds_sf100_benchmark_ts.py b/website/scripts/generate_tpcds_sf100_benchmark_ts.py index c78a90a44..075c4e8b7 100755 --- a/website/scripts/generate_tpcds_sf100_benchmark_ts.py +++ b/website/scripts/generate_tpcds_sf100_benchmark_ts.py @@ -222,7 +222,7 @@ def main() -> None: gpu_meta.get("tpcds_benchmark_native_min_retry_chunk_row_groups", "0") or 0 ), "nativeSourceChunkPlanningMode": gpu_meta.get( - "tpcds_benchmark_native_source_chunk_planning_mode", "local_parquet_throughput" + "tpcds_benchmark_native_source_chunk_planning_mode", "parquet_throughput" ), "nativeStatisticalAggregateExecutionMode": gpu_meta.get( "tpcds_benchmark_native_statistical_aggregate_execution_mode", "gpu_native_tolerant" diff --git a/website/scripts/generate_tpch_sf100_benchmark_ts.py b/website/scripts/generate_tpch_sf100_benchmark_ts.py index d7129f4b5..cb41d27a3 100755 --- a/website/scripts/generate_tpch_sf100_benchmark_ts.py +++ b/website/scripts/generate_tpch_sf100_benchmark_ts.py @@ -184,7 +184,7 @@ def meta_or(key: str, default: Any) -> Any: meta_or("tpch_benchmark_native_min_retry_chunk_row_groups", "1") ), "nativeSourceChunkPlanningMode": meta_or( - "tpch_benchmark_native_source_chunk_planning_mode", "local_parquet_throughput" + "tpch_benchmark_native_source_chunk_planning_mode", "parquet_throughput" ), "nativeMaxSourceReadsInFlight": inum( meta_or("tpch_benchmark_native_max_source_reads_in_flight", "2") diff --git a/website/src/data/tpcdsSf100Benchmark.ts b/website/src/data/tpcdsSf100Benchmark.ts index e2e08499c..02298a4b6 100644 --- a/website/src/data/tpcdsSf100Benchmark.ts +++ b/website/src/data/tpcdsSf100Benchmark.ts @@ -247,7 +247,7 @@ export const tpcdsSf100Benchmark = { "nativeMaxRowGroupsPerChunk": 1024, "nativeMaxRetryAttempts": 3, "nativeMinRetryChunkRowGroups": 1, - "nativeSourceChunkPlanningMode": "local_parquet_throughput", + "nativeSourceChunkPlanningMode": "parquet_throughput", "nativeStatisticalAggregateExecutionMode": "gpu_native_tolerant", "kvikioCompatMode": "ON", "parity": { diff --git a/website/src/data/tpchSf100Benchmark.ts b/website/src/data/tpchSf100Benchmark.ts index d7f5c4d87..2ad277648 100644 --- a/website/src/data/tpchSf100Benchmark.ts +++ b/website/src/data/tpchSf100Benchmark.ts @@ -94,7 +94,7 @@ export const tpchSf100Benchmark = { "nativeMaxRowGroupsPerChunk": 1024, "nativeMaxRetryAttempts": 4, "nativeMinRetryChunkRowGroups": 1, - "nativeSourceChunkPlanningMode": "local_parquet_throughput", + "nativeSourceChunkPlanningMode": "parquet_throughput", "nativeMaxSourceReadsInFlight": 2, "nativeRuntimePolicy": "shared_per_benchmark_run", "kvikioCompatMode": "ON"