Skip to content

Native s3:// Parquet sources and read-only Iceberg tables #9

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morristai merged 3 commits into from
Oct 25, 2026
Merged

Native s3:// Parquet sources and read-only Iceberg tables #9

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8110849
feat: native s3:// Parquet sources and read-only Iceberg tables
morristai Oct 24, 2026
68ef7e3
fix: address review findings on remote Parquet and Iceberg reads
morristai Oct 24, 2026
5034911
docs: update website blog
morristai Oct 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .config/policies/cpu-packaging.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,4 +53,4 @@ deployment_requirement = "The host CPU must implement the full x86-64-v3 ISA; a
# `scripts/build/cpu_packaging_policy.py shipped-features`. NVML device
# diagnostics are opt-in through FEATURES and are not part of this list.
[server]
features = ["cugraph", "cuml", "cuvs"]
features = ["cugraph", "cuml", "cuvs", "iceberg"]
34 changes: 34 additions & 0 deletions .config/policies/public-api/algeon-datafusion.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,12 +146,14 @@ pub algeon_datafusion::capability::CapabilityReason::NativeSortPreservingMergeUn
pub algeon_datafusion::capability::CapabilityReason::NativeSourceByteRange
pub algeon_datafusion::capability::CapabilityReason::NativeSourceEmpty
pub algeon_datafusion::capability::CapabilityReason::NativeSourceExprAdapter
pub algeon_datafusion::capability::CapabilityReason::NativeSourceFileSchemaMismatch
pub algeon_datafusion::capability::CapabilityReason::NativeSourceNonLocalObjectStore
pub algeon_datafusion::capability::CapabilityReason::NativeSourceOutputOrdering
pub algeon_datafusion::capability::CapabilityReason::NativeSourceParquetFileReaderFactory
pub algeon_datafusion::capability::CapabilityReason::NativeSourcePartitionValues
pub algeon_datafusion::capability::CapabilityReason::NativeSourcePreserveOrder
pub algeon_datafusion::capability::CapabilityReason::NativeSourcePushedDownParquetPredicate
pub algeon_datafusion::capability::CapabilityReason::NativeSourceRemoteFooterUnavailable
pub algeon_datafusion::capability::CapabilityReason::NativeSourceRowSelection
pub algeon_datafusion::capability::CapabilityReason::NativeSourceSchemaMapping
pub algeon_datafusion::capability::CapabilityReason::UnsupportedExpression
Expand Down Expand Up @@ -743,6 +745,7 @@ pub algeon_datafusion::error::ErrorCode::SourceUnauthenticated
pub algeon_datafusion::error::ErrorCode::SqlParse
pub algeon_datafusion::error::ErrorCode::UnsupportedExpression
pub algeon_datafusion::error::ErrorCode::UnsupportedOperator
pub algeon_datafusion::error::ErrorCode::UnsupportedTableFeature
pub algeon_datafusion::error::ErrorCode::UnsupportedType
impl algeon_datafusion::error::ErrorCode
pub const algeon_datafusion::error::ErrorCode::ALL: &'static [Self]
Expand Down Expand Up @@ -1976,6 +1979,16 @@ pub algeon_datafusion::server::PasswordHashError::InvalidHash(alloc::string::Str
impl core::error::Error for algeon_datafusion::server::PasswordHashError
impl core::fmt::Display for algeon_datafusion::server::PasswordHashError
pub fn algeon_datafusion::server::PasswordHashError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result
pub enum algeon_datafusion::server::ServerIcebergCatalogKind
pub algeon_datafusion::server::ServerIcebergCatalogKind::Glue(algeon_datafusion::server::ServerIcebergGlueCatalogConfig)
pub algeon_datafusion::server::ServerIcebergCatalogKind::Rest(algeon_datafusion::server::ServerIcebergRestCatalogConfig)
pub enum algeon_datafusion::server::ServerIcebergRestAuth
pub algeon_datafusion::server::ServerIcebergRestAuth::None
pub algeon_datafusion::server::ServerIcebergRestAuth::OAuth2
pub algeon_datafusion::server::ServerIcebergRestAuth::OAuth2::credential: algeon_datafusion::server::ServerIcebergSecret
pub algeon_datafusion::server::ServerIcebergRestAuth::OAuth2::scope: core::option::Option<alloc::string::String>
pub algeon_datafusion::server::ServerIcebergRestAuth::OAuth2::server_uri: core::option::Option<alloc::string::String>
pub algeon_datafusion::server::ServerIcebergRestAuth::Token(algeon_datafusion::server::ServerIcebergSecret)
pub enum algeon_datafusion::server::ServerShutdown
pub algeon_datafusion::server::ServerShutdown::OsSignals
pub algeon_datafusion::server::ServerShutdown::Token(tokio_util::sync::cancellation_token::CancellationToken)
Expand Down Expand Up @@ -2022,6 +2035,7 @@ pub algeon_datafusion::server::ServerConfig::data_root: core::option::Option<std
pub algeon_datafusion::server::ServerConfig::gpu_device_profiles: alloc::vec::Vec<query_runtime::runtime::backend_core::GpuBackendDeviceProfile>
pub algeon_datafusion::server::ServerConfig::gpu_memory_ownership: alloc::vec::Vec<(usize, query_runtime::runtime::backend_core::GpuMemoryOwnership)>
pub algeon_datafusion::server::ServerConfig::http2_keepalive: algeon_datafusion::server::Http2Keepalive
pub algeon_datafusion::server::ServerConfig::iceberg_catalogs: alloc::vec::Vec<algeon_datafusion::server::ServerIcebergCatalogConfig>
pub algeon_datafusion::server::ServerConfig::init_sql: core::option::Option<std::path::PathBuf>
pub algeon_datafusion::server::ServerConfig::models: core::option::Option<algeon_datafusion::cuml_sql::CumlModelStoreConfig>
pub algeon_datafusion::server::ServerConfig::native_optimizer_config: algeon_datafusion::planner::AlgeonNativeOptimizerConfig
Expand All @@ -2035,6 +2049,7 @@ impl algeon_datafusion::server::ServerConfig
pub fn algeon_datafusion::server::ServerConfig::from_env() -> algeon_datafusion::server::Result<Self>
pub fn algeon_datafusion::server::ServerConfig::from_vars<K, V>(impl core::iter::traits::collect::IntoIterator<Item = (K, V)>) -> algeon_datafusion::server::Result<Self> where K: core::convert::Into<alloc::string::String>, V: core::convert::Into<alloc::string::String>
pub fn algeon_datafusion::server::ServerConfig::gpu_devices(&self) -> alloc::vec::Vec<usize>
pub fn algeon_datafusion::server::ServerConfig::s3_storage(&self) -> core::option::Option<&query_runtime::runtime::object_store::S3SourceStorageProfile>
pub fn algeon_datafusion::server::ServerConfig::with_gpu_memory_ownership(self, impl core::iter::traits::collect::IntoIterator<Item = (usize, query_runtime::runtime::backend_core::GpuMemoryOwnership)>) -> Self
pub struct algeon_datafusion::server::ServerError
impl algeon_datafusion::server::ServerError
Expand All @@ -2057,6 +2072,25 @@ impl core::error::Error for algeon_datafusion::server::ServerError
pub fn algeon_datafusion::server::ServerError::source(&self) -> core::option::Option<&(dyn core::error::Error + 'static)>
impl core::fmt::Display for algeon_datafusion::server::ServerError
pub fn algeon_datafusion::server::ServerError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result
pub struct algeon_datafusion::server::ServerIcebergCatalogConfig
pub algeon_datafusion::server::ServerIcebergCatalogConfig::kind: algeon_datafusion::server::ServerIcebergCatalogKind
pub algeon_datafusion::server::ServerIcebergCatalogConfig::name: alloc::string::String
pub struct algeon_datafusion::server::ServerIcebergGlueCatalogConfig
pub algeon_datafusion::server::ServerIcebergGlueCatalogConfig::catalog_id: core::option::Option<alloc::string::String>
pub algeon_datafusion::server::ServerIcebergGlueCatalogConfig::endpoint: core::option::Option<alloc::string::String>
pub algeon_datafusion::server::ServerIcebergGlueCatalogConfig::region: core::option::Option<alloc::string::String>
pub algeon_datafusion::server::ServerIcebergGlueCatalogConfig::warehouse: alloc::string::String
pub struct algeon_datafusion::server::ServerIcebergRestCatalogConfig
pub algeon_datafusion::server::ServerIcebergRestCatalogConfig::auth: algeon_datafusion::server::ServerIcebergRestAuth
pub algeon_datafusion::server::ServerIcebergRestCatalogConfig::prefix: core::option::Option<alloc::string::String>
pub algeon_datafusion::server::ServerIcebergRestCatalogConfig::uri: alloc::string::String
pub algeon_datafusion::server::ServerIcebergRestCatalogConfig::warehouse: core::option::Option<alloc::string::String>
pub struct algeon_datafusion::server::ServerIcebergSecret(_)
impl algeon_datafusion::server::ServerIcebergSecret
pub fn algeon_datafusion::server::ServerIcebergSecret::expose(&self) -> &str
pub fn algeon_datafusion::server::ServerIcebergSecret::new(impl core::convert::Into<alloc::string::String>) -> Self
impl core::fmt::Debug for algeon_datafusion::server::ServerIcebergSecret
pub fn algeon_datafusion::server::ServerIcebergSecret::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result
pub struct algeon_datafusion::server::ServerObservabilityConfig
pub algeon_datafusion::server::ServerObservabilityConfig::active_logical_capture_max_bytes: core::option::Option<u64>
pub algeon_datafusion::server::ServerObservabilityConfig::global_sealed_max_bytes: u64
Expand Down
9 changes: 6 additions & 3 deletions .config/policies/public-api/algeon-query-planning.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -449,7 +449,7 @@ pub fn query_planning::chunk::SourceChunkPlan::row_group_count(&self) -> usize
#[non_exhaustive] pub struct query_planning::chunk::SourceChunkPlanningOptions
impl query_planning::chunk::SourceChunkPlanningOptions
pub fn query_planning::chunk::SourceChunkPlanningOptions::validate(&self) -> query_planning::error::Result<()>
pub fn query_planning::chunk::SourceChunkPlanningOptions::with_local_parquet_throughput_max_row_groups_per_chunk(self, usize) -> Self
pub fn query_planning::chunk::SourceChunkPlanningOptions::with_parquet_throughput_max_row_groups_per_chunk(self, usize) -> Self
impl core::default::Default for query_planning::chunk::SourceChunkPlanningOptions
pub fn query_planning::chunk::SourceChunkPlanningOptions::default() -> Self
pub struct query_planning::chunk::SourceChunkPlanningPolicy
Expand Down Expand Up @@ -482,7 +482,7 @@ pub const fn query_planning::chunk::SourceTaskMemoryModel::source_adjacent_opera
pub const fn query_planning::chunk::SourceTaskMemoryModel::source_read_preflight_slack_slots(self) -> u64
pub const fn query_planning::chunk::SourceTaskMemoryModel::source_read_slots(self) -> u64
pub const fn query_planning::chunk::SourceTaskMemoryModel::unary_chain_frame() -> Self
pub const query_planning::chunk::LOCAL_PARQUET_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK: usize
pub const query_planning::chunk::PARQUET_THROUGHPUT_MAX_ROW_GROUPS_PER_CHUNK: usize
pub fn query_planning::chunk::annotate_source_chunk_batching_reasons(&mut [query_planning::chunk::SourceChunk], core::option::Option<&'static str>, core::option::Option<u64>)
pub fn query_planning::chunk::plan_source_chunk_batches(&[query_planning::chunk::SourceChunk], core::option::Option<u64>) -> alloc::vec::Vec<query_planning::chunk::SourceChunkBatch>
pub fn query_planning::chunk::plan_source_chunk_batches_with_row_cap(&[query_planning::chunk::SourceChunk], core::option::Option<u64>, core::option::Option<u64>) -> alloc::vec::Vec<query_planning::chunk::SourceChunkBatch>
Expand Down Expand Up @@ -852,7 +852,7 @@ pub type query_planning::error::Result<T> = core::result::Result<T, query_planni
pub mod query_planning::execution_contracts
pub enum query_planning::execution_contracts::SourceChunkPlanningMode
pub query_planning::execution_contracts::SourceChunkPlanningMode::Conservative
pub query_planning::execution_contracts::SourceChunkPlanningMode::LocalParquetThroughput
pub query_planning::execution_contracts::SourceChunkPlanningMode::ParquetThroughput
impl query_planning::execution_contracts::SourceChunkPlanningMode
pub const fn query_planning::execution_contracts::SourceChunkPlanningMode::as_str(self) -> &'static str
impl core::fmt::Display for query_planning::execution_contracts::SourceChunkPlanningMode
Expand Down Expand Up @@ -3360,6 +3360,7 @@ pub query_planning::source::RejectedSourceFeature::PartitionColumns
pub query_planning::source::RejectedSourceFeature::PartitionValues
pub query_planning::source::RejectedSourceFeature::PreserveOrder
pub query_planning::source::RejectedSourceFeature::PushedDownParquetPredicate
pub query_planning::source::RejectedSourceFeature::RemoteFooterUnavailable
pub query_planning::source::RejectedSourceFeature::RowSelection
pub query_planning::source::RejectedSourceFeature::ScanLimit
pub query_planning::source::RejectedSourceFeature::SchemaMapping
Expand Down Expand Up @@ -3593,6 +3594,8 @@ pub query_planning::source::SourceDiagnostics::selected_row_group_stats: core::o
pub query_planning::source::SourceDiagnostics::selected_row_groups: core::option::Option<usize>
pub query_planning::source::SourceDiagnostics::source_read_strategy: alloc::string::String
pub query_planning::source::SourceDiagnostics::zero_column_count_disposition: query_planning::source::ZeroColumnCountDisposition
impl query_planning::source::SourceDiagnostics
pub fn query_planning::source::SourceDiagnostics::set_resolved_read_labels(&mut self, &[query_planning::source::LocalParquetFile])
pub struct query_planning::source::SourceFooterFacts<'a>
impl<'a> query_planning::source::SourceFooterFacts<'a>
pub const fn query_planning::source::SourceFooterFacts<'a>::new(&'a [core::option::Option<&'a parquet::file::metadata::ParquetMetaData>]) -> Self
Expand Down
Loading