From b7a5594e1795ed6ad88d826b173729f6a74c101a Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Wed, 12 Aug 2026 18:23:31 +0000 Subject: [PATCH 01/36] Initial commit Signed-off-by: teodordelibasic-db --- cpp/NEXT_CHANGELOG.md | 3 + cpp/README.md | 9 +- cpp/examples/json/README.md | 2 +- cpp/examples/json/batch.cpp | 10 +- cpp/include/zerobus/ack_callback.hpp | 14 +- dotnet/CONTRIBUTING.md | 2 +- dotnet/NEXT_CHANGELOG.md | 4 + dotnet/README.md | 42 ++++-- dotnet/examples/JsonSingle/Program.cs | 10 +- dotnet/examples/ProtoSingle/Program.cs | 18 ++- .../examples/ProtoSingle/ProtoSingle.csproj | 6 + .../ProtoSingle/Protos/my_message.proto | 9 ++ dotnet/src/Zerobus/ZerobusStream.cs | 8 +- java/NEXT_CHANGELOG.md | 6 + java/README.md | 60 ++++----- java/examples/json/BatchIngestionExample.java | 9 +- java/examples/json/README.md | 4 +- java/examples/json/SingleRecordExample.java | 9 +- java/examples/legacy/LegacyStreamExample.java | 12 +- .../examples/proto/BatchIngestionExample.java | 9 +- java/examples/proto/README.md | 4 +- java/examples/proto/SingleRecordExample.java | 9 +- .../com/databricks/zerobus/AckCallback.java | 12 +- python/CONTRIBUTING.md | 12 +- python/NEXT_CHANGELOG.md | 4 + python/README.md | 123 +++++++++--------- python/examples/README.md | 18 +-- python/examples/async_example_json.py | 30 ++--- python/examples/async_example_proto.py | 28 ++-- python/examples/sync_example_arrow.py | 8 +- python/examples/sync_example_json.py | 23 ++-- python/examples/sync_example_proto.py | 21 +-- python/rust/src/common.rs | 9 +- python/zerobus/__init__.py | 17 ++- python/zerobus/_zerobus_core.pyi | 21 +-- python/zerobus/sdk/aio/zerobus_sdk.py | 14 +- python/zerobus/sdk/shared/config.py | 30 +++-- python/zerobus/sdk/sync/zerobus_sdk.py | 18 +-- rust/NEXT_CHANGELOG.md | 3 + rust/README.md | 10 +- rust/ffi/NEXT_CHANGELOG.md | 3 + rust/ffi/src/common.rs | 3 +- rust/ffi/zerobus.h | 3 +- rust/sdk/src/lib.rs | 2 +- rust/sdk/src/stream_configuration.rs | 7 +- typescript/NEXT_CHANGELOG.md | 3 + typescript/README.md | 71 ++++++---- typescript/examples/json/README.md | 20 +-- typescript/examples/proto/README.md | 28 ++-- typescript/examples/proto/batch.ts | 26 ++-- typescript/examples/proto/single.ts | 10 +- typescript/src/headers_provider.ts | 20 ++- typescript/src/lib.rs | 25 +++- typescript/tsconfig.json | 2 +- 54 files changed, 492 insertions(+), 391 deletions(-) create mode 100644 dotnet/examples/ProtoSingle/Protos/my_message.proto diff --git a/cpp/NEXT_CHANGELOG.md b/cpp/NEXT_CHANGELOG.md index 7927f801..4a6df1e1 100644 --- a/cpp/NEXT_CHANGELOG.md +++ b/cpp/NEXT_CHANGELOG.md @@ -18,6 +18,9 @@ ### Documentation +- Corrected custom-header examples and clarified that acknowledgment callbacks + run once per logical ingest submission rather than once per record in a batch. + ### Internal Changes ### Breaking Changes diff --git a/cpp/README.md b/cpp/README.md index bc8723fd..8488fe46 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -256,7 +256,8 @@ Arrow Flight is **Beta** — the API may change. class MyProvider : public zerobus::HeadersProvider { public: std::map get_headers() override { - return {{"Authorization", "Bearer " + current_token()}}; + return {{"authorization", "Bearer " + current_token()}, + {"x-databricks-zerobus-table-name", "main.analytics.events"}}; } }; @@ -289,7 +290,7 @@ options.ack_callback = zerobus::AckCallback::from( // Durable up to `offset` (acks are monotonic: offset N => all <= N acked). }, [](std::int64_t offset, const std::string& msg) noexcept { - // The record at `offset` failed terminally. + // The logical submission at `offset` failed terminally. }); zerobus::Stream stream = @@ -299,8 +300,8 @@ zerobus::Stream stream = Contract (see [`ack_callback.hpp`](include/zerobus/ack_callback.hpp) for the canonical version): -- `on_ack` fires once per record in monotonic offset order; `on_error` fires per - unacked record on terminal failure (errors also still surface from +- `on_ack` fires once per logical ingest submission in monotonic offset order; + `on_error` fires per unacked submission on terminal failure (errors also still surface from `ingest`/`flush`/`wait_for_offset()`). - Both methods are **`noexcept`** — an escaping exception crosses the C FFI boundary, which is UB, so it calls `std::terminate`. Handle errors inside the diff --git a/cpp/examples/json/README.md b/cpp/examples/json/README.md index 1f30eb1c..2d19f664 100644 --- a/cpp/examples/json/README.md +++ b/cpp/examples/json/README.md @@ -130,7 +130,7 @@ Each `UnackedRecord` exposes `is_json()`, the raw `data()` bytes, and ``` Batch of 3 records queued; batch offset ID: 0 Batch acknowledged at offset ID: 0 -Stream closed successfully. Callback observed 3 acknowledgements. +Stream closed successfully. Callback observed 1 logical submission acknowledgement. ``` ### Code Highlights diff --git a/cpp/examples/json/batch.cpp b/cpp/examples/json/batch.cpp index 2f3b9963..af28e4ed 100644 --- a/cpp/examples/json/batch.cpp +++ b/cpp/examples/json/batch.cpp @@ -74,10 +74,10 @@ std::string make_order_json(int id, const std::string& customer, // core calls get_headers() whenever it needs fresh headers (possibly from // another thread), and you return whatever the endpoint expects — at minimum an // "authorization" bearer token and "x-databricks-zerobus-table-name". Throwing -// surfaces the message to the core as a headers-provider error. The provider -// must outlive the Stream (which holds a shared_ptr to it). See -// include/zerobus/headers_provider.hpp for the full contract. Used in the -// commented-out create_stream() call below. +// surfaces the message to the core as a headers-provider error. Provider +// ownership is handed to the FFI, so the caller does not need to retain its own +// shared_ptr after stream creation. See include/zerobus/headers_provider.hpp +// for the full contract. Used in the commented-out create_stream() call below. class BearerTokenProvider : public zerobus::HeadersProvider { public: BearerTokenProvider(std::string table_name, std::string token) @@ -183,7 +183,7 @@ int main() { stream.flush(); stream.close(); std::cout << "Stream closed successfully. Callback observed " - << acked.load() << " acknowledgements.\n"; + << acked.load() << " logical submission acknowledgement(s).\n"; } catch (const zerobus::ZerobusException& e) { std::cerr << "Zerobus error: " << e.what() << " (retryable=" << (e.is_retryable() ? "true" : "false") diff --git a/cpp/include/zerobus/ack_callback.hpp b/cpp/include/zerobus/ack_callback.hpp index 8ee9295b..567890c6 100644 --- a/cpp/include/zerobus/ack_callback.hpp +++ b/cpp/include/zerobus/ack_callback.hpp @@ -12,9 +12,10 @@ namespace zerobus { /// in `wait_for_offset()` / `flush()`. Register via /// `StreamOptions::ack_callback`. /// -/// `on_ack` fires once per record, in monotonic offset order (offset `N` => -/// all `<= N` acked); `on_error` fires per unacked record on terminal failure, -/// which may also surface from `ingest`/`flush`/`wait_for_offset()`. Callbacks +/// `on_ack` fires once per logical ingest submission, in monotonic offset order +/// (offset `N` => all `<= N` acked); `on_error` fires per unacked submission on +/// terminal failure, which may also surface from `ingest`/`flush`/ +/// `wait_for_offset()`. Callbacks /// run serialized on a background task, possibly on another thread: synchronize /// shared state, keep them light, and don't call back into the owning `Stream` /// (that is concurrent use of a non-thread-safe object). @@ -38,12 +39,13 @@ class AckCallback { public: virtual ~AckCallback() = default; - /// Called when the record at @p offset has been durably acknowledged. + /// Called when the logical submission at @p offset has been durably + /// acknowledged. virtual void on_ack(std::int64_t offset) noexcept = 0; - /// Called when the record at @p offset failed terminally. + /// Called when the logical submission at @p offset failed terminally. /// - /// @param offset The logical offset of the failed record. + /// @param offset The logical offset of the failed submission. /// @param error_message Human-readable error text from the core. virtual void on_error(std::int64_t offset, const std::string& error_message) noexcept = 0; diff --git a/dotnet/CONTRIBUTING.md b/dotnet/CONTRIBUTING.md index 2979c9d4..cee926e7 100644 --- a/dotnet/CONTRIBUTING.md +++ b/dotnet/CONTRIBUTING.md @@ -9,7 +9,7 @@ This document covers .NET-specific development setup and workflow. ### Prerequisites - Git -- .NET SDK 8.0 or higher +- .NET SDK 10.0 or higher (the projects target both .NET 8 and .NET 10) - Rust toolchain (`cargo`) - [Install Rust](https://rustup.rs/) - Bash shell (used by `build_native.sh`) diff --git a/dotnet/NEXT_CHANGELOG.md b/dotnet/NEXT_CHANGELOG.md index d033ef68..b645defd 100644 --- a/dotnet/NEXT_CHANGELOG.md +++ b/dotnet/NEXT_CHANGELOG.md @@ -21,6 +21,10 @@ ### Documentation +- Corrected installation and source-build prerequisites, separated JSON and + Protobuf stream examples, added a runnable generated-message example, and + replaced per-record waits with one final flush in bulk-ingestion examples. + ### Internal Changes - Made the .NET release workflow build-only, consistent with the other SDKs. It now packs the NuGet package and uploads it as an artifact; publishing and the GitHub Release happen downstream. diff --git a/dotnet/README.md b/dotnet/README.md index 52f5c95c..425d0353 100644 --- a/dotnet/README.md +++ b/dotnet/README.md @@ -4,8 +4,8 @@ High-performance .NET SDK for streaming data ingestion into Databricks Delta tab ## Requirements -- **.NET 8** or **.NET 10** -- **Rust toolchain** (for building the native `zerobus_ffi` library from source) +- Consumers: .NET 8 or .NET 10 +- Building from source: .NET 10 SDK and a Rust toolchain ## Quick Start @@ -31,19 +31,20 @@ using var stream = sdk.CreateJsonStream( clientSecret, options); -// 4. Ingest records. -long offset = stream.IngestRecord("""{"id": 1, "message": "Hello"}"""); - -// 5. Wait for acknowledgment. -stream.WaitForOffset(offset); +// 4. Queue records, then confirm the whole run with one flush. +for (int id = 1; id <= 100; id++) +{ + stream.IngestRecord($$"""{"id": {{id}}, "message": "Hello"}"""); +} +stream.Flush(); ``` ## Installation -### NuGet (when published) +### NuGet ```bash -dotnet add package Databricks.Zerobus.Ingest.Sdk +dotnet add package Databricks.Zerobus --version 0.5.1 ``` ### From Source @@ -187,13 +188,26 @@ If you use this untyped API, JSON streams must set `RecordType.Json` and use the Proto streams must provide `DescriptorProto` and use the byte-oriented overloads. ```csharp -// JSON -long offset = stream.IngestRecord("""{"field": "value"}"""); +// JSON stream +var jsonOptions = options with { RecordType = RecordType.Json }; +using var jsonStream = sdk.CreateStream( + new TableProperties("catalog.schema.json_table"), + clientId, + clientSecret, + jsonOptions); +long jsonOffset = jsonStream.IngestRecord("""{"field": "value"}"""); +jsonStream.WaitForOffset(jsonOffset); -// Protobuf +// Protobuf stream +var protoOptions = options with { RecordType = RecordType.Proto }; byte[] protoBytes = myMessage.ToByteArray(); -long offset = stream.IngestRecord(protoBytes); -stream.WaitForOffset(offset); +using var protoStream = sdk.CreateStream( + new TableProperties("catalog.schema.proto_table", descriptorProto), + clientId, + clientSecret, + protoOptions); +long protoOffset = protoStream.IngestRecord(protoBytes); +protoStream.WaitForOffset(protoOffset); ``` #### `IngestRecords` diff --git a/dotnet/examples/JsonSingle/Program.cs b/dotnet/examples/JsonSingle/Program.cs index e349bec3..340bc2c5 100644 --- a/dotnet/examples/JsonSingle/Program.cs +++ b/dotnet/examples/JsonSingle/Program.cs @@ -32,7 +32,6 @@ options); Console.WriteLine("Ingesting records..."); -var offsets = new List(); for (int i = 0; i < 5; i++) { @@ -49,7 +48,6 @@ { long offset = stream.IngestRecord(jsonRecord); Console.WriteLine($"Ingested record {i} at offset {offset}"); - offsets.Add(offset); } catch (ZerobusException ex) when (ex.IsRetryable) { @@ -61,12 +59,8 @@ } } -// Wait for specific offsets to be acknowledged. +// Confirm every successfully queued record with one durability barrier. Console.WriteLine("Waiting for acknowledgments..."); -foreach (var offset in offsets) -{ - stream.WaitForOffset(offset); - Console.WriteLine($"Record at offset {offset} acknowledged"); -} +stream.Flush(); Console.WriteLine("All records successfully ingested and acknowledged!"); diff --git a/dotnet/examples/ProtoSingle/Program.cs b/dotnet/examples/ProtoSingle/Program.cs index aa8d4353..2567a7ff 100644 --- a/dotnet/examples/ProtoSingle/Program.cs +++ b/dotnet/examples/ProtoSingle/Program.cs @@ -1,4 +1,6 @@ using Databricks.Zerobus; +using Databricks.Zerobus.Examples; +using Google.Protobuf; // Get configuration from environment. var zerobusEndpoint = Environment.GetEnvironmentVariable("ZEROBUS_SERVER_ENDPOINT") @@ -12,10 +14,8 @@ var tableName = Environment.GetEnvironmentVariable("ZEROBUS_TABLE_NAME") ?? throw new InvalidOperationException("ZEROBUS_TABLE_NAME not set"); -// In a real application, you would load the DescriptorProto from your compiled .proto file. -// For example, using Google.Protobuf.Reflection: -// var descriptor = MyMessage.Descriptor.File.SerializedData.ToByteArray(); -byte[] descriptorProto = []; // Replace with your actual descriptor bytes. +// Use the descriptor for the exact generated message type sent on this stream. +byte[] descriptorProto = MyMessage.Descriptor.ToProto().ToByteArray(); // Create SDK instance. using var sdk = ZerobusSdk.CreateBuilder() @@ -41,9 +41,13 @@ for (int i = 0; i < 5; i++) { - // In a real application, serialize your protobuf message: - // byte[] protoBytes = myMessage.ToByteArray(); - byte[] protoBytes = [0x08, 0x01]; // Placeholder — replace with real protobuf data. + var message = new MyMessage + { + DeviceName = $"sensor-{i}", + Temp = 20 + i, + Humidity = 60 + i, + }; + byte[] protoBytes = message.ToByteArray(); long offset = stream.IngestRecord(protoBytes); Console.WriteLine($"Ingested protobuf record {i} at offset {offset}"); diff --git a/dotnet/examples/ProtoSingle/ProtoSingle.csproj b/dotnet/examples/ProtoSingle/ProtoSingle.csproj index ce333cf0..4fc0bb36 100644 --- a/dotnet/examples/ProtoSingle/ProtoSingle.csproj +++ b/dotnet/examples/ProtoSingle/ProtoSingle.csproj @@ -9,4 +9,10 @@ + + + + + + diff --git a/dotnet/examples/ProtoSingle/Protos/my_message.proto b/dotnet/examples/ProtoSingle/Protos/my_message.proto new file mode 100644 index 00000000..bde5b77b --- /dev/null +++ b/dotnet/examples/ProtoSingle/Protos/my_message.proto @@ -0,0 +1,9 @@ +syntax = "proto2"; + +option csharp_namespace = "Databricks.Zerobus.Examples"; + +message MyMessage { + optional string device_name = 1; + optional int32 temp = 2; + optional int64 humidity = 3; +} diff --git a/dotnet/src/Zerobus/ZerobusStream.cs b/dotnet/src/Zerobus/ZerobusStream.cs index 305d78e0..8777b4ef 100644 --- a/dotnet/src/Zerobus/ZerobusStream.cs +++ b/dotnet/src/Zerobus/ZerobusStream.cs @@ -67,12 +67,12 @@ public bool IsClosed() /// /// /// - /// // JSON - /// long offset = stream.IngestRecord("{\"id\": 1, \"message\": \"Hello\"}"); + /// // JSON stream + /// long jsonOffset = jsonStream.IngestRecord("{\"id\": 1, \"message\": \"Hello\"}"); /// - /// // Protobuf + /// // Protobuf stream /// byte[] protoBytes = SerializeMyProto(myMessage); - /// long offset = stream.IngestRecord(protoBytes); + /// long protoOffset = protoStream.IngestRecord(protoBytes); /// /// public long IngestRecord(string payload) diff --git a/java/NEXT_CHANGELOG.md b/java/NEXT_CHANGELOG.md index f2acbc72..15dc1f58 100644 --- a/java/NEXT_CHANGELOG.md +++ b/java/NEXT_CHANGELOG.md @@ -10,6 +10,12 @@ ### Documentation +- Updated dependency snippets to version 1.3.0 and corrected README and example + code for stream cleanup, recreation, unique local variables, and a single + durability barrier after queued ingestion. Clarified that acknowledgment + callbacks fire once per logical ingest submission, including one callback per + batch ingest call. + ### Internal Changes ### Breaking Changes diff --git a/java/README.md b/java/README.md index dd77c119..735d054a 100644 --- a/java/README.md +++ b/java/README.md @@ -142,7 +142,7 @@ Add the SDK as a dependency in your `pom.xml`: com.databricks zerobus-ingest-sdk - 0.2.0 + 1.3.0 ``` @@ -151,7 +151,7 @@ Or with Gradle (`build.gradle`): ```groovy dependencies { - implementation 'com.databricks:zerobus-ingest-sdk:0.2.0' + implementation 'com.databricks:zerobus-ingest-sdk:1.3.0' } ``` @@ -164,7 +164,7 @@ dependencies { com.databricks zerobus-ingest-sdk - 0.2.0 + 1.3.0 @@ -195,7 +195,7 @@ If you prefer the self-contained fat JAR with all dependencies included: com.databricks zerobus-ingest-sdk - 0.2.0 + 1.3.0 jar-with-dependencies @@ -205,7 +205,7 @@ Or with Gradle: ```groovy dependencies { - implementation 'com.databricks:zerobus-ingest-sdk:0.2.0:jar-with-dependencies' + implementation 'com.databricks:zerobus-ingest-sdk:1.3.0:jar-with-dependencies' } ``` @@ -223,11 +223,11 @@ mvn clean package -Dzerobus.skipNativeLibCheck=true This generates two JAR files in the `target/` directory: -- **Regular JAR**: `zerobus-ingest-sdk-0.2.0.jar` (~12MB, includes native libraries) +- **Regular JAR**: `zerobus-ingest-sdk-1.3.0.jar` (~12MB, includes native libraries) - Contains only the SDK classes - Requires all dependencies on the classpath -- **Fat JAR**: `zerobus-ingest-sdk-0.2.0-jar-with-dependencies.jar` (~19MB, includes native libraries + all dependencies) +- **Fat JAR**: `zerobus-ingest-sdk-1.3.0-jar-with-dependencies.jar` (~19MB, includes native libraries + all dependencies) - Contains SDK classes plus all dependencies bundled - Self-contained, easier to deploy @@ -275,7 +275,7 @@ Create `pom.xml`: com.databricks zerobus-ingest-sdk - 0.2.0 + 1.3.0 @@ -339,16 +339,16 @@ The proto generation tool requires the fat JAR (all dependencies included): ```bash # Download from Maven Central -wget https://repo1.maven.org/maven2/com/databricks/zerobus-ingest-sdk/0.2.0/zerobus-ingest-sdk-0.2.0-jar-with-dependencies.jar +wget https://repo1.maven.org/maven2/com/databricks/zerobus-ingest-sdk/1.3.0/zerobus-ingest-sdk-1.3.0-jar-with-dependencies.jar # Or if you built from source, it's in target/ -# cp target/zerobus-ingest-sdk-0.2.0-jar-with-dependencies.jar . +# cp target/zerobus-ingest-sdk-1.3.0-jar-with-dependencies.jar . ``` **Run the tool:** ```bash -java -jar zerobus-ingest-sdk-0.2.0-jar-with-dependencies.jar \ +java -jar zerobus-ingest-sdk-1.3.0-jar-with-dependencies.jar \ --uc-endpoint "https://dbc-a1b2c3d4-e5f6.cloud.databricks.com" \ --client-id "your-service-principal-application-id" \ --client-secret "your-service-principal-secret" \ @@ -449,20 +449,13 @@ public class ZerobusClient { String clientId = "your-service-principal-application-id"; String clientSecret = "your-service-principal-secret"; - // Initialize SDK - ZerobusSdk sdk = new ZerobusSdk(serverEndpoint, workspaceUrl); - - // Create stream (recommended offset-based proto stream) - ZerobusProtoStream stream = sdk.streamBuilder() - .table(tableName) - .oauth(clientId, clientSecret) - .compiledProto(AirQuality.getDescriptor().toProto()) - .build() - .join(); - - try { - long lastOffset = -1; - + try (ZerobusSdk sdk = new ZerobusSdk(serverEndpoint, workspaceUrl); + ZerobusProtoStream stream = sdk.streamBuilder() + .table(tableName) + .oauth(clientId, clientSecret) + .compiledProto(AirQuality.getDescriptor().toProto()) + .build() + .join()) { // Ingest in a loop. ingestRecordOffset() returns as soon as the record is // queued; the SDK sends it and tracks its acknowledgment in the background. for (int i = 0; i < 100; i++) { @@ -472,17 +465,13 @@ public class ZerobusClient { .setHumidity(50 + (i % 40)) .build(); - lastOffset = stream.ingestRecordOffset(record); // returns immediately + stream.ingestRecordOffset(record); // returns immediately } - // Confirm everything is durably committed. flush() does the same; - // waiting on the last offset works because acks are ordered. - stream.waitForOffset(lastOffset); + // Confirm everything is durably committed with one barrier. + stream.flush(); System.out.println("Successfully ingested 100 records!"); - } finally { - stream.close(); - sdk.close(); } } } @@ -1566,17 +1555,18 @@ List unacked = stream.getUnackedRecords(json -> gson.fromJson(json, MyDa ### AckCallback (Interface) -Callback interface for acknowledgment notifications. +Callback interface for acknowledgment notifications. The callback is invoked once per logical +ingest submission, so a batch ingest call produces one callback rather than one per record. ```java void onAck(long offsetId) ``` -Called when records up to `offsetId` are acknowledged. +Called when a logical ingest submission is acknowledged. Records up to `offsetId` are durable. ```java void onError(long offsetId, String errorMessage) ``` -Called when an error occurs for records at or after `offsetId`. +Called when an error occurs for the logical ingest submission at `offsetId`. **Track durability progress without blocking.** Register an `AckCallback` to observe acknowledgments as they arrive on a background thread while you keep ingesting — a natural diff --git a/java/examples/json/BatchIngestionExample.java b/java/examples/json/BatchIngestionExample.java index ad474b5a..05dd038b 100644 --- a/java/examples/json/BatchIngestionExample.java +++ b/java/examples/json/BatchIngestionExample.java @@ -39,7 +39,7 @@ public static void main(String[] args) throws Exception { System.out.println("=== JSON Batch Ingestion Example ===\n"); - ZerobusSdk sdk = new ZerobusSdk(serverEndpoint, workspaceUrl); + try (ZerobusSdk sdk = new ZerobusSdk(serverEndpoint, workspaceUrl)) { ZerobusJsonStream stream = sdk.streamBuilder() .table(tableName) .oauth(clientId, clientSecret) @@ -147,12 +147,11 @@ public static void main(String[] args) throws Exception { System.out.println("\n--- Demonstrating recreateStream ---"); // Recreate the stream (would re-ingest any unacked records if there were any) - ZerobusJsonStream newStream = sdk.recreateStream(stream).join(); + try (ZerobusJsonStream newStream = sdk.recreateStream(stream).join()) { System.out.println(" New stream created successfully"); // Ingest a batch on the new stream int newRecords = 0; - try { List newBatch = new ArrayList<>(); for (int i = 0; i < 5; i++) { newBatch.add(String.format( @@ -164,13 +163,11 @@ public static void main(String[] args) throws Exception { newRecords = 5; newStream.flush(); System.out.println(" " + newRecords + " new records ingested on recreated stream"); - } finally { - newStream.close(); } System.out.println("\n=== RecreateStream demo complete ==="); - sdk.close(); + } } // Simple JSON parser for Map (in production, use Gson or Jackson) diff --git a/java/examples/json/README.md b/java/examples/json/README.md index ed8b9f6c..8c72157c 100644 --- a/java/examples/json/README.md +++ b/java/examples/json/README.md @@ -105,10 +105,10 @@ if (offset.isPresent()) { ```java // As JSON strings -List unacked = stream.getUnackedRecords(); +List unackedJson = stream.getUnackedRecords(); // As deserialized objects -List unacked = stream.getUnackedRecords(json -> gson.fromJson(json, MyData.class)); +List unackedObjects = stream.getUnackedRecords(json -> gson.fromJson(json, MyData.class)); ``` ## Examples diff --git a/java/examples/json/SingleRecordExample.java b/java/examples/json/SingleRecordExample.java index d15c6d2c..987b9196 100644 --- a/java/examples/json/SingleRecordExample.java +++ b/java/examples/json/SingleRecordExample.java @@ -34,7 +34,7 @@ public static void main(String[] args) throws Exception { System.out.println("=== JSON Single Record Example ===\n"); - ZerobusSdk sdk = new ZerobusSdk(serverEndpoint, workspaceUrl); + try (ZerobusSdk sdk = new ZerobusSdk(serverEndpoint, workspaceUrl)) { ZerobusJsonStream stream = sdk.streamBuilder() .table(tableName) .oauth(clientId, clientSecret) @@ -120,12 +120,11 @@ public static void main(String[] args) throws Exception { System.out.println("\n--- Demonstrating recreateStream ---"); // Recreate the stream (would re-ingest any unacked records if there were any) - ZerobusJsonStream newStream = sdk.recreateStream(stream).join(); + try (ZerobusJsonStream newStream = sdk.recreateStream(stream).join()) { System.out.println(" New stream created successfully"); // Ingest a few more records on the new stream int newRecords = 0; - try { for (int i = 0; i < 3; i++) { String json = String.format( "{\"device_name\": \"json-recreate-%d\", \"temp\": %d, \"humidity\": %d}", @@ -136,13 +135,11 @@ public static void main(String[] args) throws Exception { } newStream.flush(); System.out.println(" " + newRecords + " new records ingested on recreated stream"); - } finally { - newStream.close(); } System.out.println("\n=== RecreateStream demo complete ==="); - sdk.close(); + } } // Simple JSON parser for Map (in production, use Gson or Jackson) diff --git a/java/examples/legacy/LegacyStreamExample.java b/java/examples/legacy/LegacyStreamExample.java index 959626a0..00fffc75 100644 --- a/java/examples/legacy/LegacyStreamExample.java +++ b/java/examples/legacy/LegacyStreamExample.java @@ -19,6 +19,7 @@ */ public class LegacyStreamExample { + @SuppressWarnings("deprecation") public static void main(String[] args) throws Exception { String serverEndpoint = System.getenv("ZEROBUS_SERVER_ENDPOINT"); String workspaceUrl = System.getenv("DATABRICKS_WORKSPACE_URL"); @@ -36,14 +37,13 @@ public static void main(String[] args) throws Exception { System.out.println("=== Legacy ZerobusStream Example (Future-based) ===\n"); - ZerobusSdk sdk = new ZerobusSdk(serverEndpoint, workspaceUrl); + try (ZerobusSdk sdk = new ZerobusSdk(serverEndpoint, workspaceUrl)) { TableProperties tableProperties = new TableProperties<>( tableName, AirQuality.getDefaultInstance() ); - @SuppressWarnings("deprecation") ZerobusStream stream = sdk.createStream( tableProperties, clientId, clientSecret ).join(); @@ -105,8 +105,7 @@ public static void main(String[] args) throws Exception { System.out.println("\n--- Demonstrating recreateStream ---"); // Recreate the stream (would re-ingest any unacked records if there were any) - @SuppressWarnings("deprecation") - ZerobusStream newStream = sdk.recreateStream(stream).join(); + try (ZerobusStream newStream = sdk.recreateStream(stream).join()) { System.out.println(" New stream created successfully"); // Ingest a few more records on the new stream. @@ -114,7 +113,6 @@ public static void main(String[] args) throws Exception { // durability. New code should prefer the offset-based ZerobusProtoStream / // ZerobusJsonStream APIs. int newRecords = 0; - try { java.util.concurrent.CompletableFuture lastFuture = null; for (int i = 0; i < 3; i++) { AirQuality record = AirQuality.newBuilder() @@ -129,12 +127,10 @@ public static void main(String[] args) throws Exception { lastFuture.join(); // confirm durability once } System.out.println(" " + newRecords + " new records ingested on recreated stream"); - } finally { - newStream.close(); } System.out.println("\n=== RecreateStream demo complete ==="); - sdk.close(); + } } } diff --git a/java/examples/proto/BatchIngestionExample.java b/java/examples/proto/BatchIngestionExample.java index 5629f83c..de4c90a2 100644 --- a/java/examples/proto/BatchIngestionExample.java +++ b/java/examples/proto/BatchIngestionExample.java @@ -38,7 +38,7 @@ public static void main(String[] args) throws Exception { System.out.println("=== Proto Batch Ingestion Example ===\n"); - ZerobusSdk sdk = new ZerobusSdk(serverEndpoint, workspaceUrl); + try (ZerobusSdk sdk = new ZerobusSdk(serverEndpoint, workspaceUrl)) { ZerobusProtoStream stream = sdk.streamBuilder() .table(tableName) .oauth(clientId, clientSecret) @@ -149,12 +149,11 @@ public static void main(String[] args) throws Exception { System.out.println("\n--- Demonstrating recreateStream ---"); // Recreate the stream (would re-ingest any unacked records if there were any) - ZerobusProtoStream newStream = sdk.recreateStream(stream).join(); + try (ZerobusProtoStream newStream = sdk.recreateStream(stream).join()) { System.out.println(" New stream created successfully"); // Ingest a batch on the new stream int newRecords = 0; - try { List newBatch = new ArrayList<>(); for (int i = 0; i < 5; i++) { newBatch.add(AirQuality.newBuilder() @@ -167,12 +166,10 @@ public static void main(String[] args) throws Exception { newRecords = 5; newStream.flush(); System.out.println(" " + newRecords + " new records ingested on recreated stream"); - } finally { - newStream.close(); } System.out.println("\n=== RecreateStream demo complete ==="); - sdk.close(); + } } } diff --git a/java/examples/proto/README.md b/java/examples/proto/README.md index 0ba156d3..693632f5 100644 --- a/java/examples/proto/README.md +++ b/java/examples/proto/README.md @@ -88,10 +88,10 @@ if (offset.isPresent()) { ```java // As raw bytes -List unacked = stream.getUnackedRecords(); +List unackedBytes = stream.getUnackedRecords(); // As parsed messages -List unacked = stream.getUnackedRecords(AirQuality.parser()); +List unackedMessages = stream.getUnackedRecords(AirQuality.parser()); ``` ## Examples diff --git a/java/examples/proto/SingleRecordExample.java b/java/examples/proto/SingleRecordExample.java index 9996ba3a..b2640c89 100644 --- a/java/examples/proto/SingleRecordExample.java +++ b/java/examples/proto/SingleRecordExample.java @@ -33,7 +33,7 @@ public static void main(String[] args) throws Exception { System.out.println("=== Proto Single Record Example ===\n"); - ZerobusSdk sdk = new ZerobusSdk(serverEndpoint, workspaceUrl); + try (ZerobusSdk sdk = new ZerobusSdk(serverEndpoint, workspaceUrl)) { ZerobusProtoStream stream = sdk.streamBuilder() .table(tableName) .oauth(clientId, clientSecret) @@ -125,12 +125,11 @@ public static void main(String[] args) throws Exception { System.out.println("\n--- Demonstrating recreateStream ---"); // Recreate the stream (would re-ingest any unacked records if there were any) - ZerobusProtoStream newStream = sdk.recreateStream(stream).join(); + try (ZerobusProtoStream newStream = sdk.recreateStream(stream).join()) { System.out.println(" New stream created successfully"); // Ingest a few more records on the new stream int newRecords = 0; - try { for (int i = 0; i < 3; i++) { AirQuality record = AirQuality.newBuilder() .setDeviceName("proto-recreate-" + i) @@ -142,12 +141,10 @@ public static void main(String[] args) throws Exception { } newStream.flush(); System.out.println(" " + newRecords + " new records ingested on recreated stream"); - } finally { - newStream.close(); } System.out.println("\n=== RecreateStream demo complete ==="); - sdk.close(); + } } } diff --git a/java/src/main/java/com/databricks/zerobus/AckCallback.java b/java/src/main/java/com/databricks/zerobus/AckCallback.java index bc2005b4..75890661 100644 --- a/java/src/main/java/com/databricks/zerobus/AckCallback.java +++ b/java/src/main/java/com/databricks/zerobus/AckCallback.java @@ -7,6 +7,9 @@ * replaces the deprecated {@code Consumer} callback with a more type-safe and * flexible API. * + *

A callback is invoked once per logical ingest submission. In particular, a batch ingest call + * produces one callback rather than one callback per record in the batch. + * *

Implementations should be thread-safe as callbacks may be invoked from multiple threads. * Callbacks should be lightweight to avoid blocking the internal processing threads. * @@ -35,8 +38,7 @@ public interface AckCallback { /** - * Called when a record (or records up to this offset) has been durably acknowledged by the - * server. + * Called when a logical ingest submission has been durably acknowledged by the server. * *

The offset ID represents the durability acknowledgment up to and including this offset. All * records with offset IDs less than or equal to this value have been durably stored. @@ -49,10 +51,10 @@ public interface AckCallback { void onAck(long offsetId); /** - * Called when an error occurs for a specific record or offset. + * Called when an error occurs for a logical ingest submission. * - *

This method is called when the SDK encounters an error that affects a specific offset. The - * error may be retryable or non-retryable depending on the nature of the failure. + *

This method is called when the SDK encounters an error that affects a specific submission + * offset. The error may be retryable or non-retryable depending on the nature of the failure. * *

This method should not throw exceptions. If an exception is thrown, it will be logged but * will not affect stream operation. diff --git a/python/CONTRIBUTING.md b/python/CONTRIBUTING.md index f796bb70..54acd801 100644 --- a/python/CONTRIBUTING.md +++ b/python/CONTRIBUTING.md @@ -116,26 +116,26 @@ xdg-open htmlcov/index.html # Linux Example docstring: ```python -def ingest_record(self, record) -> RecordAcknowledgment: +def ingest_record_offset(self, record) -> int: """ Submits a single record for ingestion into the stream. - This method may block if the maximum number of in-flight records - has been reached. + This method returns after the record is queued. Call ``flush()`` after + queueing a group of records to confirm durability. Args: record: The Protobuf message object to be ingested. Returns: - RecordAcknowledgment: An object to wait on for the server's acknowledgment. + int: The logical offset assigned to the record. Raises: ZerobusException: If the stream is not in a valid state for ingestion. Example: >>> record = AirQuality(device_name="sensor-1", temp=25) - >>> ack = stream.ingest_record(record) - >>> ack.wait_for_ack() + >>> offset = stream.ingest_record_offset(record) + >>> stream.flush() """ ``` diff --git a/python/NEXT_CHANGELOG.md b/python/NEXT_CHANGELOG.md index 7eeaf5d4..3c2c1bb3 100644 --- a/python/NEXT_CHANGELOG.md +++ b/python/NEXT_CHANGELOG.md @@ -10,6 +10,10 @@ ### Documentation +- Corrected README, example, and docstring snippets for record-format selection, + exception handling, recovery, iterator return values, custom headers, async + contexts, and durability-aware throughput measurement. + ### Internal Changes ### Breaking Changes diff --git a/python/README.md b/python/README.md index 123ff83f..322871b4 100644 --- a/python/README.md +++ b/python/README.md @@ -120,15 +120,14 @@ and JSON) does not need `pyarrow` at all. ```python from zerobus.sdk.sync import ZerobusSdk -from zerobus.sdk.shared import RecordType, StreamConfigurationOptions, TableProperties +from zerobus.sdk.shared import TableProperties server_endpoint = "https://1234567890123456.zerobus.us-west-2.cloud.databricks.com" workspace_url = "https://dbc-a1b2c3d4-e5f6.cloud.databricks.com" sdk = ZerobusSdk(server_endpoint, workspace_url) table_properties = TableProperties("main.default.air_quality") -options = StreamConfigurationOptions(record_type=RecordType.JSON) -stream = sdk.create_stream(client_id, client_secret, table_properties, options) +stream = sdk.create_stream(client_id, client_secret, table_properties) try: for i in range(100): @@ -147,7 +146,7 @@ finally: ```python import asyncio from zerobus.sdk.aio import ZerobusSdk -from zerobus.sdk.shared import RecordType, StreamConfigurationOptions, TableProperties +from zerobus.sdk.shared import TableProperties async def main(): server_endpoint = "https://1234567890123456.zerobus.us-west-2.cloud.databricks.com" @@ -155,8 +154,7 @@ async def main(): sdk = ZerobusSdk(server_endpoint, workspace_url) table_properties = TableProperties("main.default.air_quality") - options = StreamConfigurationOptions(record_type=RecordType.JSON) - stream = await sdk.create_stream(client_id, client_secret, table_properties, options) + stream = await sdk.create_stream(client_id, client_secret, table_properties) try: for i in range(100): @@ -294,7 +292,7 @@ See the [`examples/`](examples/) directory for complete runnable examples. Configure stream behavior by passing a `StreamConfigurationOptions` object to `create_stream()`: ```python -from zerobus.sdk.shared import StreamConfigurationOptions, RecordType, AckCallback +from zerobus.sdk.shared import AckCallback, StreamConfigurationOptions class MyCallback(AckCallback): def on_ack(self, offset: int): @@ -304,7 +302,6 @@ class MyCallback(AckCallback): print(f"Error at offset {offset}: {error_message}") options = StreamConfigurationOptions( - record_type=RecordType.JSON, max_inflight_records=10000, recovery=True, ack_callback=MyCallback() @@ -315,9 +312,13 @@ stream = sdk.create_stream(client_id, client_secret, table_properties, options) ### Available Options +The record format is inferred from `TableProperties`: omitting `descriptor_proto` selects JSON, +while providing a Protobuf descriptor selects Protobuf. `record_type` is retained for backward +compatibility but does not select the format. + | Option | Type | Default | Description | | -------------------------------- | --------------- | ------------------ | -------------------------------------------------------------------------------------------------------------------- | -| `record_type` | `RecordType` | `RecordType.PROTO` | Serialization format: `PROTO` or `JSON` | +| `record_type` | `RecordType` | `RecordType.PROTO` | Retained for backward compatibility; format comes from `TableProperties.descriptor_proto` | | `max_inflight_records` | `int` | `1000000` | Maximum number of unacknowledged records | | `recovery` | `bool` | `True` | Enable automatic stream recovery | | `recovery_timeout_ms` | `int` | `15000` | Timeout for recovery operations (ms) | @@ -327,56 +328,51 @@ stream = sdk.create_stream(client_id, client_secret, table_properties, options) | `server_lack_of_ack_timeout_ms` | `int` | `60000` | Server acknowledgment timeout (ms) | | `stream_paused_max_wait_time_ms` | `Optional[int]` | `None` | Max wait during graceful stream close. `None` = full server duration, `0` = immediate, `x` = min(x, server_duration) | | `callback_max_wait_time_ms` | `Optional[int]` | `5000` | Max wait for callbacks after `close()`. `None` = wait forever | -| `ack_callback` | `AckCallback` | `None` | Callback invoked on record acknowledgment or error | +| `ack_callback` | `AckCallback` | `None` | Callback invoked once per logical ingest submission (one record call or one batch call) | ## Error Handling -The SDK raises two types of exceptions: - -- `ZerobusException` - Retriable errors (network issues, temporary server errors) -- `NonRetriableException` - Non-retriable errors (invalid credentials, missing table) +SDK operation failures currently surface as `ZerobusException`. The +`NonRetriableException` class is exported for compatibility, but the current native binding does +not construct it. Do not use the Python exception class to decide whether an operation is safe to +retry. ```python -from zerobus.sdk.shared import ZerobusException, NonRetriableException +from zerobus.sdk.shared import ZerobusException try: stream.ingest_record_offset(record) -except NonRetriableException as e: - print(f"Fatal error: {e}") - raise except ZerobusException as e: - print(f"Retriable error: {e}") + print(f"Ingestion failed: {e}") ``` ## Handling Stream Failures -The SDK automatically handles retries for transient errors. Use `get_unacked_records()` only when a stream has **permanently failed** (non-retriable error or max retries exceeded): +The SDK automatically handles retries for transient errors. Use `get_unacked_records()` only after +the stream has permanently closed following a failure: ```python -from zerobus.sdk.shared import NonRetriableException +from zerobus.sdk.shared import ZerobusException try: for i in range(10000): stream.ingest_record_offset(record) - stream.flush() -except NonRetriableException as e: - unacked = stream.get_unacked_records() # Returns List[bytes] + stream.close() +except ZerobusException as e: + unacked = list(stream.get_unacked_records()) print(f"Stream failed: {e}. {len(unacked)} records unacknowledged.") - # Retry with a new stream - new_stream = sdk.create_stream(client_id, client_secret, table_properties, options) - for record_bytes in unacked: - new_stream.ingest_record_offset(record_bytes) # Pass bytes directly + # Preserve the record format and original batch grouping while retrying. + new_stream = sdk.recreate_stream(stream) new_stream.flush() new_stream.close() ``` -Use `get_unacked_batches()` for batch-level retry: +Use `get_unacked_batches()` to inspect the original batch grouping after the stream closes: ```python -unacked_batches = stream.get_unacked_batches() # Returns List[List[bytes]] -for batch in unacked_batches: - new_stream.ingest_records_offset(batch) +unacked_batches = list(stream.get_unacked_batches()) +print(f"{len(unacked_batches)} batches remain unacknowledged") ``` **Decoding unacked records:** @@ -403,17 +399,21 @@ throughput to one record per round-trip, so save it for confirming a specific re **Idiomatic flow:** ```python -for record in records: - await stream.ingest_record_offset(record) # queues immediately, no round-trip -await stream.flush() # one wait for everything +async def ingest_all(stream, records): + for record in records: + await stream.ingest_record_offset(record) # queues immediately, no round-trip + await stream.flush() # one wait for everything ``` **Confirming a specific record** (waiting on the last offset confirms all prior records): ```python -for record in records: - offset = await stream.ingest_record_offset(record) -await stream.wait_for_offset(offset) # confirm the run before continuing +async def ingest_and_confirm(stream, records): + offset = None + for record in records: + offset = await stream.ingest_record_offset(record) + if offset is not None: + await stream.wait_for_offset(offset) # confirm the run before continuing ``` ## API Reference @@ -423,7 +423,11 @@ await stream.wait_for_offset(offset) # confirm the run before continu Main entry point. Sync: `from zerobus.sdk.sync import ZerobusSdk` / Async: `from zerobus.sdk.aio import ZerobusSdk` ```python -sdk = ZerobusSdk(server_endpoint: str, unity_catalog_endpoint: str, application_name: Optional[str] = None) +sdk = ZerobusSdk( + host="https://.zerobus..cloud.databricks.com", + unity_catalog_url="https://", + application_name="my-app/1.0", +) ``` `application_name` is optional; when set it is appended to the `user-agent` header on gRPC requests to the Zerobus server (not on the OAuth token requests to the login service). It follows the `"/"` convention (e.g. `my-app/1.0`). @@ -432,7 +436,8 @@ sdk = ZerobusSdk(server_endpoint: str, unity_catalog_endpoint: str, application_ # Sync stream = sdk.create_stream(client_id, client_secret, table_properties, options=None, headers_provider=None) # Async -stream = await sdk.create_stream(client_id, client_secret, table_properties, options=None, headers_provider=None) +async def create_async_stream(sdk): + return await sdk.create_stream(client_id, client_secret, table_properties, options=None, headers_provider=None) ``` ### `ZerobusStream` @@ -467,9 +472,10 @@ offset = stream.ingest_record_offset(record) stream.wait_for_offset(offset) # Block until durably written # Async -offset = await stream.ingest_record_offset(record) -# ... do other work ... -await stream.wait_for_offset(offset) # Block until durably written +async def confirm_async(stream, record): + offset = await stream.ingest_record_offset(record) + # ... do other work ... + await stream.wait_for_offset(offset) # Block until durably written ``` Acks are ordered, so waiting on the last offset returned confirms all prior records too. @@ -482,32 +488,33 @@ stream.flush() # Wait for all pending records to be acknowledged stream.close() # Flush and close gracefully (always call in finally) # Async -await stream.flush() -await stream.close() +async def close_async(stream): + await stream.flush() + await stream.close() ``` **Unacknowledged records:** ```python # Sync -records = stream.get_unacked_records() # List[bytes] -batches = stream.get_unacked_batches() # List[List[bytes]] +records = stream.get_unacked_records() # Iterator[bytes] +batches = stream.get_unacked_batches() # Iterator[List[bytes]] # Async -records = await stream.get_unacked_records() -batches = await stream.get_unacked_batches() +async def get_unacked_async(stream): + records = await stream.get_unacked_records() + batches = await stream.get_unacked_batches() + return records, batches ``` ### `TableProperties` ```python -TableProperties(table_name: str, descriptor: Descriptor = None) - # JSON mode TableProperties("catalog.schema.table") # Protobuf mode -TableProperties("catalog.schema.table", MyMessage.DESCRIPTOR) +TableProperties("catalog.schema.table", descriptor_proto=MyMessage.DESCRIPTOR) ``` ### `StreamConfigurationOptions` @@ -521,11 +528,11 @@ from zerobus.sdk.shared import AckCallback class MyCallback(AckCallback): def on_ack(self, offset: int) -> None: - # Called when a record is acknowledged by the server + # Called once for each acknowledged single-record or batch submission pass def on_error(self, offset: int, error_message: str) -> None: - # Called when a record encounters an error + # Called once when a single-record or batch submission encounters an error pass ``` @@ -535,16 +542,16 @@ For custom authentication (e.g. custom token providers), implement `HeadersProvi ### `RecordAcknowledgment` (Sync only, deprecated) -```python +```text ack.wait_for_ack(timeout_sec=None) # Block until acknowledged ack.is_done() -> bool -ack.add_done_callback(callback) ``` ### Exceptions -- `ZerobusException(message, cause=None)` - Retriable errors -- `NonRetriableException(message, cause=None)` - Non-retriable errors (extends `ZerobusException`) +- `ZerobusException(message, cause=None)` - Base exception raised by current SDK operations +- `NonRetriableException(message, cause=None)` - Exported subclass reserved for non-retriable errors; + the current native binding does not construct it ## Debugging diff --git a/python/examples/README.md b/python/examples/README.md index 1768d80e..864fcb55 100644 --- a/python/examples/README.md +++ b/python/examples/README.md @@ -1,6 +1,7 @@ # Zerobus SDK Examples -This directory contains runnable example applications demonstrating both synchronous and asynchronous usage of the Zerobus Ingest SDK for Python, with examples for both both record type modes: **protobuf** and **JSON**. +This directory contains runnable synchronous and asynchronous examples for Protobuf, +JSON, and Arrow Flight ingestion with the Zerobus Ingest SDK for Python. For complete SDK documentation including installation, API reference, and configuration details, see the [main README](../README.md). @@ -16,7 +17,7 @@ cd zerobus-sdk/python ### 2. Install Dependencies ```bash -pip install -e . +pip install -e ".[arrow]" ``` The examples use a pre-generated protobuf file (`record_pb2.py`) based on the included `record.proto` schema. @@ -43,10 +44,12 @@ export ZEROBUS_TABLE_NAME="catalog.schema.table" # Synchronous examples (blocking I/O) python examples/sync_example_proto.py # Protobuf python examples/sync_example_json.py # JSON +python examples/sync_example_arrow.py # Arrow Flight # Asynchronous examples (non-blocking I/O) python examples/async_example_proto.py # Protobuf python examples/async_example_json.py # JSON +python examples/async_example_arrow.py # Arrow Flight ``` ## Examples Overview @@ -62,7 +65,8 @@ Each example includes detailed comments explaining when to use each method and t ### Serialization Formats -The SDK supports two serialization formats: +The row-oriented examples cover two serialization formats. The Arrow Flight +examples use `pyarrow.RecordBatch` data instead. #### Protocol Buffers **Files:** `sync_example_proto.py`, `async_example_proto.py` @@ -75,7 +79,6 @@ More efficient over the wire. You can pass either: # Create protobuf record record = record_pb2.AirQuality(device_name="sensor-1", temp=25, humidity=60) table_properties = TableProperties(TABLE_NAME, record_pb2.AirQuality.DESCRIPTOR) -options = StreamConfigurationOptions(record_type=RecordType.PROTO) # Recommended: Use ingest_record_offset() for better performance offset = stream.ingest_record_offset(record) @@ -98,7 +101,6 @@ Good for getting started. No protobuf schema required. You can pass either: # Create JSON record record_dict = {"device_name": "sensor-1", "temp": 25, "humidity": 60} table_properties = TableProperties(TABLE_NAME) -options = StreamConfigurationOptions(record_type=RecordType.JSON) # Recommended: Use ingest_record_offset() for better performance offset = stream.ingest_record_offset(record_dict) @@ -173,8 +175,8 @@ Both APIs provide the same functionality and performance. The key differences ar | Format | Record Input | Configuration | |--------|-------------|---------------| -| **Protobuf** (Default) | `Message` object or `bytes` | `TableProperties(table_name, descriptor)` | -| **JSON** | `dict` or `str` (JSON string) | `TableProperties(table_name)` + `StreamConfigurationOptions(record_type=RecordType.JSON)` | +| **Protobuf** | `Message` object or `bytes` | `TableProperties(table_name, descriptor_proto=descriptor)` | +| **JSON** | `dict` or `str` (JSON string) | `TableProperties(table_name)` | ## Authentication @@ -203,7 +205,7 @@ To use your own JSON structure: ```python json_record = json.dumps({"field1": "value1", "field2": 123}) ``` -2. Configure `StreamConfigurationOptions` with `record_type=RecordType.JSON` +2. Construct `TableProperties` without a Protobuf descriptor to select JSON 3. Ensure your JSON structure matches the schema of your Databricks table Note: The SDK sends JSON strings directly without client-side schema validation. diff --git a/python/examples/async_example_json.py b/python/examples/async_example_json.py index 2a907886..22807d6b 100644 --- a/python/examples/async_example_json.py +++ b/python/examples/async_example_json.py @@ -5,7 +5,7 @@ Record Type Mode: JSON - Records are sent as JSON-encoded strings - - Uses RecordType.JSON to specify JSON serialization + - Omitting a descriptor from TableProperties selects JSON serialization - Best for dynamic schemas or when working with JSON data Use Case: Best for applications already using asyncio, async web frameworks (FastAPI, aiohttp), @@ -28,7 +28,6 @@ from zerobus.sdk.aio import ZerobusSdk from zerobus.sdk.shared import ( AckCallback, - RecordType, StreamConfigurationOptions, TableProperties, ) @@ -85,8 +84,9 @@ class CustomHeadersProvider(HeadersProvider): for custom headers (e.g., custom metadata, existing token management, etc.). """ - def __init__(self, custom_token: str): + def __init__(self, custom_token: str, table_name: str): self.custom_token = custom_token + self.table_name = table_name def get_headers(self): """ @@ -97,6 +97,7 @@ def get_headers(self): """ return [ ("authorization", f"Bearer {self.custom_token}"), + ("x-databricks-zerobus-table-name", self.table_name), ("x-custom-header", "custom-value"), ] @@ -105,19 +106,20 @@ class MyAckCallback(AckCallback): """ Example acknowledgment callback that logs progress. - The callback is invoked by the SDK whenever records are acknowledged by the server. + The callback is invoked once per logical ingest submission. A batch call produces + one callback, not one callback per record in the batch. """ def __init__(self): super().__init__() - self.ack_count = 0 + self.submission_count = 0 def on_ack(self, offset): - """Called when records are acknowledged by the server.""" - self.ack_count += 1 - # Log every 100 acknowledgments - if self.ack_count % 100 == 0: - logger.info(f" Acknowledged up to offset: {offset} (batch #{self.ack_count})") + """Called when a logical ingest submission is acknowledged by the server.""" + self.submission_count += 1 + # Log every 100 acknowledged submissions + if self.submission_count % 100 == 0: + logger.info(f" Acknowledged up to offset: {offset} (submission #{self.submission_count})") async def main(): @@ -145,9 +147,8 @@ async def main(): sdk = ZerobusSdk(SERVER_ENDPOINT, UNITY_CATALOG_ENDPOINT, application_name="my-app/1.0") logger.info("✓ SDK initialized") - # Step 2: Configure stream options with JSON record type and ack callback + # Step 2: Configure stream options with an ack callback options = StreamConfigurationOptions( - record_type=RecordType.JSON, max_inflight_records=10_000, # Allow 10k records in flight recovery=True, # Enable automatic recovery ack_callback=MyAckCallback(), # Track acknowledgments @@ -168,7 +169,7 @@ async def main(): # Advanced: Custom headers provider (for special use cases only) # Uncomment to use custom headers instead of OAuth: - # custom_provider = CustomHeadersProvider(custom_token="your-custom-token") + # custom_provider = CustomHeadersProvider("your-custom-token", TABLE_NAME) # stream = await sdk.create_stream( # CLIENT_ID, CLIENT_SECRET, table_properties, options, # headers_provider=custom_provider @@ -274,8 +275,7 @@ async def main(): print(f" Total time: {total_duration:.2f} seconds") print(f" Throughput: {records_per_second:.2f} records/sec") print(f" Average latency: {avg_latency_ms:.2f} ms/record") - print(f" Stream state: {stream.get_state()}") - print(f" Record type: JSON (explicit)") + print(" Record type: JSON") print("=" * 60) except Exception as e: diff --git a/python/examples/async_example_proto.py b/python/examples/async_example_proto.py index 914c87e1..7711eafc 100644 --- a/python/examples/async_example_proto.py +++ b/python/examples/async_example_proto.py @@ -25,7 +25,6 @@ from zerobus.sdk.aio import ZerobusSdk from zerobus.sdk.shared import ( AckCallback, - RecordType, StreamConfigurationOptions, TableProperties, ) @@ -82,8 +81,9 @@ class CustomHeadersProvider(HeadersProvider): for custom headers (e.g., custom metadata, existing token management, etc.). """ - def __init__(self, custom_token: str): + def __init__(self, custom_token: str, table_name: str): self.custom_token = custom_token + self.table_name = table_name def get_headers(self): """ @@ -94,6 +94,7 @@ def get_headers(self): """ return [ ("authorization", f"Bearer {self.custom_token}"), + ("x-databricks-zerobus-table-name", self.table_name), ("x-custom-header", "custom-value"), ] @@ -102,19 +103,20 @@ class MyAckCallback(AckCallback): """ Example acknowledgment callback that logs progress. - The callback is invoked by the SDK whenever records are acknowledged by the server. + The callback is invoked once per logical ingest submission. A batch call produces + one callback, not one callback per record in the batch. """ def __init__(self): super().__init__() - self.ack_count = 0 + self.submission_count = 0 def on_ack(self, offset): - """Called when records are acknowledged by the server.""" - self.ack_count += 1 - # Log every 100 acknowledgments - if self.ack_count % 100 == 0: - logger.info(f" Acknowledged up to offset: {offset} (batch #{self.ack_count})") + """Called when a logical ingest submission is acknowledged by the server.""" + self.submission_count += 1 + # Log every 100 acknowledged submissions + if self.submission_count % 100 == 0: + logger.info(f" Acknowledged up to offset: {offset} (submission #{self.submission_count})") async def main(): @@ -142,14 +144,13 @@ async def main(): sdk = ZerobusSdk(SERVER_ENDPOINT, UNITY_CATALOG_ENDPOINT, application_name="my-app/1.0") logger.info("✓ SDK initialized") - # Step 2: Configure stream options with protobuf record type and ack callback + # Step 2: Configure stream options with an ack callback options = StreamConfigurationOptions( - record_type=RecordType.PROTO, max_inflight_records=10_000, # Allow 10k records in flight recovery=True, # Enable automatic recovery ack_callback=MyAckCallback(), # Track acknowledgments ) - logger.info("✓ Stream configuration created (Protobuf mode)") + logger.info("✓ Stream configuration created") # Step 3: Define table properties # Pass the serialized FileDescriptorProto as bytes @@ -166,7 +167,7 @@ async def main(): # Advanced: Custom headers provider (for special use cases only) # Uncomment to use custom headers instead of OAuth: - # custom_provider = CustomHeadersProvider(custom_token="your-custom-token") + # custom_provider = CustomHeadersProvider("your-custom-token", TABLE_NAME) # stream = await sdk.create_stream( # CLIENT_ID, CLIENT_SECRET, table_properties, options, # headers_provider=custom_provider @@ -268,7 +269,6 @@ async def main(): print(f" Total time: {total_duration:.2f} seconds") print(f" Throughput: {records_per_second:.2f} records/sec") print(f" Average latency: {avg_latency_ms:.2f} ms/record") - print(f" Stream state: {stream.get_state()}") print(" Record type: Protobuf") print("=" * 60) diff --git a/python/examples/sync_example_arrow.py b/python/examples/sync_example_arrow.py index a6321fdf..cd58c9cf 100644 --- a/python/examples/sync_example_arrow.py +++ b/python/examples/sync_example_arrow.py @@ -152,15 +152,15 @@ def main(): stream.wait_for_offset(offset) logger.info(f" Offset {offset} acknowledged") - end_time = time.time() - duration_seconds = end_time - start_time - rows_per_second = total_rows / duration_seconds - # Step 5: Flush and close the stream logger.info("\nFlushing stream...") stream.flush() logger.info("Stream flushed") + end_time = time.time() + duration_seconds = end_time - start_time + rows_per_second = total_rows / duration_seconds + stream.close() logger.info("Stream closed") diff --git a/python/examples/sync_example_json.py b/python/examples/sync_example_json.py index db24c6d7..ac0b14b1 100644 --- a/python/examples/sync_example_json.py +++ b/python/examples/sync_example_json.py @@ -5,7 +5,7 @@ Record Type Mode: JSON - Records are sent as JSON-encoded strings - - Uses RecordType.JSON to specify JSON serialization + - Omitting a descriptor from TableProperties selects JSON serialization - Best for dynamic schemas or when working with JSON data Authentication: @@ -21,7 +21,7 @@ import os import time -from zerobus.sdk.shared import RecordType, StreamConfigurationOptions, TableProperties +from zerobus.sdk.shared import StreamConfigurationOptions, TableProperties from zerobus.sdk.shared.headers_provider import HeadersProvider from zerobus.sdk.sync import ZerobusSdk @@ -76,8 +76,9 @@ class CustomHeadersProvider(HeadersProvider): for custom headers (e.g., custom metadata, existing token management, etc.). """ - def __init__(self, custom_token: str): + def __init__(self, custom_token: str, table_name: str): self.custom_token = custom_token + self.table_name = table_name def get_headers(self): """ @@ -88,6 +89,7 @@ def get_headers(self): """ return [ ("authorization", f"Bearer {self.custom_token}"), + ("x-databricks-zerobus-table-name", self.table_name), ("x-custom-header", "custom-value"), ] @@ -122,9 +124,8 @@ def main(): table_properties = TableProperties(TABLE_NAME) logger.info(f"✓ Table properties configured for: {TABLE_NAME} (JSON mode)") - # Step 3: Create stream configuration with JSON record type + # Step 3: Create stream configuration options = StreamConfigurationOptions( - record_type=RecordType.JSON, max_inflight_records=1000, recovery=True, recovery_timeout_ms=15000, @@ -142,7 +143,7 @@ def main(): # Advanced: Custom headers provider (for special use cases only) # Uncomment to use custom headers instead of OAuth: - # custom_provider = CustomHeadersProvider(custom_token="your-custom-token") + # custom_provider = CustomHeadersProvider("your-custom-token", TABLE_NAME) # stream = sdk.create_stream( # CLIENT_ID, CLIENT_SECRET, table_properties, options, # headers_provider=custom_provider @@ -228,15 +229,15 @@ def main(): # ack = stream.ingest_record(record_dict) # Deprecated # offset = ack.wait_for_ack() # Extra step needed - end_time = time.time() - duration_seconds = end_time - start_time - records_per_second = NUM_RECORDS / duration_seconds - # Step 6: Flush and close the stream logger.info("\nFlushing stream...") stream.flush() logger.info("✓ Stream flushed") + end_time = time.time() + duration_seconds = end_time - start_time + records_per_second = NUM_RECORDS / duration_seconds + stream.close() logger.info("✓ Stream closed") @@ -248,7 +249,7 @@ def main(): print(f" Failed: {NUM_RECORDS - success_count}") print(f" Duration: {duration_seconds:.2f} seconds") print(f" Throughput: {records_per_second:.2f} records/sec") - print(" Record type: JSON (explicit)") + print(" Record type: JSON") print("=" * 60) except Exception as e: diff --git a/python/examples/sync_example_proto.py b/python/examples/sync_example_proto.py index fc7a236a..dc521e95 100644 --- a/python/examples/sync_example_proto.py +++ b/python/examples/sync_example_proto.py @@ -20,7 +20,7 @@ # Import the generated protobuf module import record_pb2 -from zerobus.sdk.shared import RecordType, StreamConfigurationOptions, TableProperties +from zerobus.sdk.shared import StreamConfigurationOptions, TableProperties from zerobus.sdk.shared.headers_provider import HeadersProvider from zerobus.sdk.sync import ZerobusSdk @@ -75,8 +75,9 @@ class CustomHeadersProvider(HeadersProvider): for custom headers (e.g., custom metadata, existing token management, etc.). """ - def __init__(self, custom_token: str): + def __init__(self, custom_token: str, table_name: str): self.custom_token = custom_token + self.table_name = table_name def get_headers(self): """ @@ -87,6 +88,7 @@ def get_headers(self): """ return [ ("authorization", f"Bearer {self.custom_token}"), + ("x-databricks-zerobus-table-name", self.table_name), ("x-custom-header", "custom-value"), ] @@ -122,16 +124,15 @@ def main(): table_properties = TableProperties(TABLE_NAME, descriptor_bytes) logger.info(f"✓ Table properties configured for: {TABLE_NAME}") - # Step 3: Create stream configuration with protobuf record type + # Step 3: Create stream configuration options = StreamConfigurationOptions( - record_type=RecordType.PROTO, max_inflight_records=1000, recovery=True, recovery_timeout_ms=15000, recovery_backoff_ms=2000, recovery_retries=4, ) - logger.info("✓ Stream configuration created (Protobuf mode)") + logger.info("✓ Stream configuration created") # Step 4: Create a stream with OAuth 2.0 authentication # @@ -142,7 +143,7 @@ def main(): # Advanced: Custom headers provider (for special use cases only) # Uncomment to use custom headers instead of OAuth: - # custom_provider = CustomHeadersProvider(custom_token="your-custom-token") + # custom_provider = CustomHeadersProvider("your-custom-token", TABLE_NAME) # stream = sdk.create_stream( # CLIENT_ID, CLIENT_SECRET, table_properties, options, # headers_provider=custom_provider @@ -221,15 +222,15 @@ def main(): # ack = stream.ingest_record(record) # Deprecated # offset = ack.wait_for_ack() # Extra step needed - end_time = time.time() - duration_seconds = end_time - start_time - records_per_second = NUM_RECORDS / duration_seconds - # Step 6: Flush and close the stream logger.info("\nFlushing stream...") stream.flush() logger.info("✓ Stream flushed") + end_time = time.time() + duration_seconds = end_time - start_time + records_per_second = NUM_RECORDS / duration_seconds + stream.close() logger.info("✓ Stream closed") diff --git a/python/rust/src/common.rs b/python/rust/src/common.rs index 54300ff7..abd0058b 100644 --- a/python/rust/src/common.rs +++ b/python/rust/src/common.rs @@ -168,7 +168,10 @@ impl TableProperties { } } -/// Base class for record acknowledgment callbacks +/// Base class for logical ingest submission acknowledgment callbacks. +/// +/// A batch ingest is one logical submission and produces one callback, not one +/// callback per record in the batch. #[pyclass(subclass, skip_from_py_object)] #[derive(Clone)] pub struct AckCallback { @@ -186,13 +189,13 @@ impl AckCallback { } } - /// Called when a record is acknowledged by the server. + /// Called when a logical ingest submission is acknowledged by the server. fn on_ack(&self, py: Python, offset: i64) -> PyResult<()> { let _ = (py, offset); Ok(()) } - /// Called when a record encounters an error during ingestion. + /// Called when a logical ingest submission encounters an error during ingestion. fn on_error(&self, py: Python, offset: i64, error_message: &str) -> PyResult<()> { let _ = (py, offset, error_message); Ok(()) diff --git a/python/zerobus/__init__.py b/python/zerobus/__init__.py index f30d78dc..c86ffa07 100644 --- a/python/zerobus/__init__.py +++ b/python/zerobus/__init__.py @@ -26,7 +26,7 @@ ... ) >>> >>> # New optimized API - >>> offset = stream.ingest_record_offset(b"data") + >>> offset = stream.ingest_record_offset('{"value": "data"}') >>> stream.flush() >>> stream.close() @@ -35,9 +35,18 @@ >>> from zerobus.sdk.aio import ZerobusSdk, TableProperties >>> >>> async def main(): - ... sdk = ZerobusSdk(host, unity_catalog_url, application_name="my-app/1.0") - ... stream = await sdk.create_stream(props, client_id, client_secret) - ... offset = await stream.ingest_record_offset(b"data") + ... sdk = ZerobusSdk( + ... host="https://your-shard-id.zerobus.region.cloud.databricks.com", + ... unity_catalog_url="https://your-workspace.cloud.databricks.com", + ... application_name="my-app/1.0", + ... ) + ... props = TableProperties("catalog.schema.table") + ... stream = await sdk.create_stream( + ... client_id="your-client-id", + ... client_secret="your-client-secret", + ... table_properties=props, + ... ) + ... offset = await stream.ingest_record_offset('{"value": "data"}') ... await stream.flush() ... await stream.close() >>> diff --git a/python/zerobus/_zerobus_core.pyi b/python/zerobus/_zerobus_core.pyi index 48d3f504..346a2291 100644 --- a/python/zerobus/_zerobus_core.pyi +++ b/python/zerobus/_zerobus_core.pyi @@ -42,36 +42,37 @@ class TableProperties: class AckCallback: """ - Base class for record acknowledgment callbacks. + Base class for logical ingest submission acknowledgment callbacks. Subclass this in Python to create custom callbacks that are invoked - when records are acknowledged or encounter errors. + once per logical ingest submission. A batch submission produces one + callback, not one callback per record in the batch. Example: class MyCallback(AckCallback): def on_ack(self, offset: int): - print(f"Record acknowledged at offset {offset}") + print(f"Submission acknowledged at offset {offset}") def on_error(self, offset: int, error_message: str): - print(f"Record at offset {offset} failed: {error_message}") + print(f"Submission at offset {offset} failed: {error_message}") """ def __init__(self) -> None: ... def on_ack(self, offset: int) -> None: """ - Called when a record is acknowledged by the server. + Called when a logical ingest submission is acknowledged by the server. Args: - offset: The offset of the acknowledged record + offset: The offset of the acknowledged submission """ ... def on_error(self, offset: int, error_message: str) -> None: """ - Called when a record encounters an error. + Called when a logical ingest submission encounters an error. Args: - offset: The offset of the failed record + offset: The offset of the failed submission error_message: Description of the error """ ... @@ -114,7 +115,7 @@ class StreamConfigurationOptions: """Maximum time in milliseconds to wait for callbacks to finish after calling close() (default: 5000)""" ack_callback: Optional[AckCallback] - """Callback to be invoked when records are acknowledged (default: None)""" + """Callback invoked once per logical ingest submission (default: None)""" def __init__( self, @@ -144,7 +145,7 @@ class StreamConfigurationOptions: record_type: Serialization format (default: RecordType.PROTO) stream_paused_max_wait_time_ms: Max wait time during graceful close in ms (default: None) callback_max_wait_time_ms: Max wait time for callbacks after close in ms (default: 5000) - ack_callback: Callback invoked on record acknowledgment (default: None) + ack_callback: Callback invoked once per logical ingest submission (default: None) """ ... diff --git a/python/zerobus/sdk/aio/zerobus_sdk.py b/python/zerobus/sdk/aio/zerobus_sdk.py index 8eb738e3..f67b9452 100644 --- a/python/zerobus/sdk/aio/zerobus_sdk.py +++ b/python/zerobus/sdk/aio/zerobus_sdk.py @@ -24,15 +24,21 @@ ... ) ... ... # Optimized async API - returns offset directly - ... offset = await stream.ingest_record_offset(b"record_data") + ... offset = await stream.ingest_record_offset('{"value": "record_data"}') ... print(f"Queued at offset {offset}") ... ... # Batch API - returns one offset for the batch - ... batch_offset = await stream.ingest_records_offset([b"record1", b"record2"]) + ... batch_offset = await stream.ingest_records_offset([ + ... '{"value": "record1"}', + ... '{"value": "record2"}', + ... ]) ... ... # Fire-and-forget for maximum throughput - ... stream.ingest_record_nowait(b"record_data") # Not awaited! - ... stream.ingest_records_nowait([b"record1", b"record2"]) # Not awaited! + ... stream.ingest_record_nowait('{"value": "record3"}') # Not awaited! + ... stream.ingest_records_nowait([ + ... '{"value": "record4"}', + ... '{"value": "record5"}', + ... ]) # Not awaited! ... ... await stream.flush() # Ensure all records are sent ... await stream.close() diff --git a/python/zerobus/sdk/shared/config.py b/python/zerobus/sdk/shared/config.py index ec632dd7..a04d1bd1 100644 --- a/python/zerobus/sdk/shared/config.py +++ b/python/zerobus/sdk/shared/config.py @@ -11,15 +11,16 @@ # Add module-level documentation AckCallback.__doc__ = """ -Base class for record acknowledgment callbacks. +Base class for logical ingest submission acknowledgment callbacks. -Subclass this in Python to create custom callbacks that are invoked -when records are acknowledged by the server or encounter errors. +Subclass this in Python to create custom callbacks that are invoked once per +logical ingest submission. A batch submission produces one callback, not one +callback per record in the batch. Example: >>> class MyCallback(AckCallback): ... def on_ack(self, offset: int): - ... print(f"Record acknowledged at offset {offset}") + ... print(f"Submission acknowledged at offset {offset}") ... ... def on_error(self, offset: int, error_message: str): ... print(f"Error at offset {offset}: {error_message}") @@ -28,16 +29,16 @@ Methods: on_ack(offset: int) -> None: - Called when a record is successfully acknowledged by the server. + Called when a logical ingest submission is acknowledged by the server. Args: - offset: The offset of the acknowledged record + offset: The offset of the acknowledged submission on_error(offset: int, error_message: str) -> None: - Called when a record encounters an error. + Called when a logical ingest submission encounters an error. Args: - offset: The offset of the failed record + offset: The offset of the failed submission error_message: Description of the error """ @@ -47,8 +48,9 @@ All parameters are optional and will use defaults if not specified. Args: - record_type: Serialization format (RecordType.PROTO or RecordType.JSON). - Default: RecordType.PROTO + record_type: Retained for backward compatibility. The record format is inferred + from TableProperties: a descriptor selects Protobuf and no descriptor selects + JSON. Default: RecordType.PROTO max_inflight_records: Maximum number of records that can be sent to the server before waiting for acknowledgment. Default: 1000000 recovery: Whether to enable automatic recovery of the stream in case of @@ -75,18 +77,18 @@ - None: Wait forever - x: Wait up to x milliseconds Default: 5000 - ack_callback: Callback to be invoked when records are acknowledged or encounter - errors. Must be a class extending AckCallback. Default: None + ack_callback: Callback invoked once per logical ingest submission when it is + acknowledged or encounters an error. A batch submission produces one callback. + Must be a class extending AckCallback. Default: None Example: - >>> from zerobus.sdk.shared import StreamConfigurationOptions, RecordType, AckCallback + >>> from zerobus.sdk.shared import StreamConfigurationOptions, AckCallback >>> >>> class MyCallback(AckCallback): ... def on_ack(self, offset: int): ... print(f"Ack: {offset}") ... >>> options = StreamConfigurationOptions( - ... record_type=RecordType.JSON, ... max_inflight_records=10000, ... recovery=True, ... ack_callback=MyCallback() diff --git a/python/zerobus/sdk/sync/zerobus_sdk.py b/python/zerobus/sdk/sync/zerobus_sdk.py index a82a7ad6..d4a8900a 100644 --- a/python/zerobus/sdk/sync/zerobus_sdk.py +++ b/python/zerobus/sdk/sync/zerobus_sdk.py @@ -22,21 +22,23 @@ ... ) >>> >>> # Optimized API - returns offset directly - >>> offset = stream.ingest_record_offset(b"record_data") + >>> offset = stream.ingest_record_offset('{"value": "record_data"}') >>> >>> # Batch API - returns one offset for the batch - >>> offsets = stream.ingest_records_offset([b"record1", b"record2"]) + >>> batch_offset = stream.ingest_records_offset([ + ... '{"value": "record1"}', + ... '{"value": "record2"}', + ... ]) >>> >>> # Fire-and-forget for maximum throughput - >>> stream.ingest_record_nowait(b"record_data") - >>> stream.ingest_records_nowait([b"record1", b"record2"]) + >>> stream.ingest_record_nowait('{"value": "record3"}') + >>> stream.ingest_records_nowait([ + ... '{"value": "record4"}', + ... '{"value": "record5"}', + ... ]) >>> >>> stream.flush() # Ensure all records are sent >>> stream.close() - >>> - >>> # Legacy API (deprecated) - returns acknowledgment object - >>> ack = stream.ingest_record(b"record_data") - >>> offset = ack.wait_for_ack(timeout_sec=30) """ from typing import Iterator, Optional diff --git a/rust/NEXT_CHANGELOG.md b/rust/NEXT_CHANGELOG.md index 6338da1f..5f2f3b66 100644 --- a/rust/NEXT_CHANGELOG.md +++ b/rust/NEXT_CHANGELOG.md @@ -36,6 +36,9 @@ ### Documentation +- Corrected README and rustdoc examples so their dependencies, feature flags, + imports, and mutable stream bindings compile as shown. + ### Internal Changes - Added Arrow C Data `RecordBatch` conversion behind a disabled-by-default diff --git a/rust/README.md b/rust/README.md index a7d41e34..3b4c4172 100644 --- a/rust/README.md +++ b/rust/README.md @@ -58,11 +58,14 @@ Add the SDK to your `Cargo.toml`: ```bash cargo add databricks-zerobus-ingest-sdk +cargo add async-trait tonic cargo add prost prost-types cargo add tokio --features macros,rt-multi-thread ``` **Why these dependencies?** - **`databricks-zerobus-ingest-sdk`** - The SDK itself +- **`async-trait`** - Required when implementing the async `HeadersProvider` trait +- **`tonic`** - Provides `transport::Endpoint` for custom `TlsConfig` implementations - **`prost`** and **`prost-types`** - Required for encoding your data to Protocol Buffers and loading schema descriptors - **`tokio`** - Async runtime required for running async functions (the SDK is fully async) @@ -437,6 +440,10 @@ let sdk = ZerobusSdk::builder() By default, the SDK uses `SecureTlsConfig` which enables TLS with the operating system's trusted CA certificates. For testing against a local `http://` server, use `NoTlsConfig` (requires the `testing` feature): +```bash +cargo add databricks-zerobus-ingest-sdk --features testing +``` + ```rust use databricks_zerobus_ingest_sdk::{ZerobusSdk, NoTlsConfig}; use std::sync::Arc; @@ -543,7 +550,7 @@ let mut stream = sdk When the table's schema is known only at runtime — for example a descriptor fetched from Unity Catalog or built in code with `schema::descriptor_from_uc_columns` — there is no compiled `prost::Message` type. Resolve the descriptor with `message_descriptor`, pass it to `.dynamic_proto(descriptor)`, and fill records field-by-field with `DynamicRecord`: ```rust -use databricks_zerobus_ingest_sdk::{message_descriptor, DynamicRecord, ProtoBytes}; +use databricks_zerobus_ingest_sdk::{message_descriptor, ProtoBytes}; use databricks_zerobus_ingest_sdk::schema::{descriptor_from_uc_columns, UcColumn}; // Build the descriptor at runtime (a column's proto field number is `position + 1`). @@ -560,7 +567,6 @@ let mut stream = sdk // Fill records field-by-field; `set()` validates the field name and type (the // value must match the field's proto type, e.g. a BIGINT column takes an i64). // `encode()` then checks proto2 required fields before producing the bytes. -use databricks_zerobus_ingest_sdk::ProtoBytes; for i in 0..100_000i64 { let mut record = stream.new_record()?; // bound to the stream's schema record.set("id", i)?.set("customer_name", "Alice Smith")?; diff --git a/rust/ffi/NEXT_CHANGELOG.md b/rust/ffi/NEXT_CHANGELOG.md index 392e468d..b13fdf73 100644 --- a/rust/ffi/NEXT_CHANGELOG.md +++ b/rust/ffi/NEXT_CHANGELOG.md @@ -10,6 +10,9 @@ ### Documentation +- Clarified that acknowledgment callbacks fire once per logical ingest + submission, so one batch ingest call produces one callback. + ### Internal Changes ### Behavior Changes diff --git a/rust/ffi/src/common.rs b/rust/ffi/src/common.rs index 9eb710ca..d8153e93 100644 --- a/rust/ffi/src/common.rs +++ b/rust/ffi/src/common.rs @@ -144,7 +144,8 @@ pub struct CStreamConfigurationOptions { /// delivered asynchronously instead of only via wait_for_offset / flush. /// Fired serialized on a background task, so keep them lightweight; /// ack_user_data and shared state need their own sync. - /// ack_on_ack: once per record, in order; monotonic (offset N => all <= N). + /// ack_on_ack: once per logical ingest submission (one batch call produces + /// one callback), in order; monotonic (offset N => all <= N). /// ack_on_error: relays core error text as-is; may also surface from ingest / flush. /// A synchronously running callback can outlive close() (abort only cancels /// at an await), so keep both pointers and ack_user_data alive until the diff --git a/rust/ffi/zerobus.h b/rust/ffi/zerobus.h index 2d7e341e..a194e9e0 100644 --- a/rust/ffi/zerobus.h +++ b/rust/ffi/zerobus.h @@ -149,7 +149,8 @@ typedef struct CStreamConfigurationOptions { * delivered asynchronously instead of only via wait_for_offset / flush. * Fired serialized on a background task, so keep them lightweight; * ack_user_data and shared state need their own sync. - * ack_on_ack: once per record, in order; monotonic (offset N => all <= N). + * ack_on_ack: once per logical ingest submission (one batch call produces + * one callback), in order; monotonic (offset N => all <= N). * ack_on_error: relays core error text as-is; may also surface from ingest / flush. * A synchronously running callback can outlive close() (abort only cancels * at an await), so keep both pointers and ack_user_data alive until the diff --git a/rust/sdk/src/lib.rs b/rust/sdk/src/lib.rs index 3ed4d019..177c6e3c 100644 --- a/rust/sdk/src/lib.rs +++ b/rust/sdk/src/lib.rs @@ -12,7 +12,7 @@ //! .unity_catalog_url(uc_endpoint) //! .build()?; //! -//! let stream = sdk +//! let mut stream = sdk //! .stream_builder() //! .table("catalog.schema.table") //! .oauth(client_id, client_secret) diff --git a/rust/sdk/src/stream_configuration.rs b/rust/sdk/src/stream_configuration.rs index 108d0674..dfd88312 100644 --- a/rust/sdk/src/stream_configuration.rs +++ b/rust/sdk/src/stream_configuration.rs @@ -30,7 +30,8 @@ pub struct StreamConfigurationOptions { /// Maximum number of requests that can be sending or pending acknowledgement at any given time. /// /// This limit controls memory usage and backpressure. When this limit is reached, - /// `ingest_record()` and `ingest_records()` calls will block until acknowledgments free up space. + /// `ingest_record_offset()` and `ingest_records_offset()` calls will wait until + /// acknowledgments free up space. /// /// Default: 1,000,000 pub max_inflight_requests: usize, @@ -93,7 +94,7 @@ pub struct StreamConfigurationOptions { /// /// When the server sends a CloseStreamSignal indicating it will close the stream, /// the SDK can enter a "paused" state where it: - /// - Continues accepting and buffering new ingest_record() calls + /// - Continues accepting and buffering new ingest calls /// - Stops sending buffered records to the server /// - Continues processing acknowledgments for in-flight records /// - Waits for either all in-flight records to be acknowledged or the timeout to expire @@ -161,7 +162,7 @@ pub struct StreamConfigurationOptions { /// Maximum total encoded record byte size allowed per ingest call. /// /// This is the sum of all record bytes passed to a single - /// `ingest_record()` / `ingest_records()` (and their `_offset` variants) call. + /// `ingest_record_offset()` or `ingest_records_offset()` call. /// Calls exceeding this limit fail fast with /// [`ZerobusError::InvalidArgument`](crate::ZerobusError::InvalidArgument) /// before any network I/O, matching the server-side limit. diff --git a/typescript/NEXT_CHANGELOG.md b/typescript/NEXT_CHANGELOG.md index 0035a5d5..43190dbf 100644 --- a/typescript/NEXT_CHANGELOG.md +++ b/typescript/NEXT_CHANGELOG.md @@ -28,6 +28,9 @@ - Simplified the README quick start to install the published npm package first and moved clone/build instructions into a source-development path. +- Corrected README, example, and JSDoc snippets for CommonJS async entry points, + generated Protobuf field names, variable declarations, stream recovery, and + custom-header callbacks. - Clarified the high-throughput ingestion pattern across the README, API reference, JSDoc doc comments (`ingestRecordOffset`, `ingestRecordsOffset`, `waitForOffset`, `flush`), and diff --git a/typescript/README.md b/typescript/README.md index eb9067bc..98ef8a97 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -108,6 +108,7 @@ JSON mode is the simplest way to get started. You don't need to define or compil ```typescript import { ZerobusSdk, RecordType } from '@databricks/zerobus-ingest-sdk'; +async function main(): Promise { // Configuration // For AWS: const zerobusEndpoint = 'https://.zerobus..cloud.databricks.com'; @@ -160,6 +161,12 @@ try { } finally { await stream.close(); } + +} + +main().catch((error) => { + console.error('Fatal error:', error); +}); ``` ### Option 2: Using Protocol Buffers (Default, Recommended) @@ -275,6 +282,7 @@ import { ZerobusSdk, RecordType } from '@databricks/zerobus-ingest-sdk'; import * as airQuality from './examples/generated/air_quality'; import { loadDescriptorProto } from '@databricks/zerobus-ingest-sdk/utils/descriptor.js'; +async function main(): Promise { // Configuration const zerobusEndpoint = 'https://.zerobus..cloud.databricks.com'; const workspaceUrl = 'https://.cloud.databricks.com'; @@ -314,7 +322,7 @@ try { // Send all records for (let i = 0; i < 100; i++) { const record = AirQuality.create({ - device_name: `sensor-${i}`, + deviceName: `sensor-${i}`, temp: 20 + i, humidity: 50 + i }); @@ -329,6 +337,12 @@ try { } finally { await stream.close(); } + +} + +main().catch((error) => { + console.error('Fatal error:', error); +}); ``` #### Type Mapping: Delta ↔ Protocol Buffers @@ -493,7 +507,7 @@ For higher throughput, use batch ingestion to send multiple records with a singl ```typescript const records = Array.from({ length: 1000 }, (_, i) => - AirQuality.create({ device_name: `sensor-${i}`, temp: 20 + i, humidity: 50 + i }) + AirQuality.create({ deviceName: `sensor-${i}`, temp: 20 + i, humidity: 50 + i }) ); // Protobuf Type 1: Message objects (high-level) - SDK auto-serializes @@ -576,7 +590,7 @@ const stream = await sdk.createStream( '', // client_secret (ignored when headers_provider is provided) options, { - getHeadersCallback: async () => [ + getHeadersCallback: () => [ ["authorization", `Bearer ${myToken}`], ["x-databricks-zerobus-table-name", tableName] ] @@ -682,11 +696,7 @@ try { } catch (error) { console.error('Ingestion failed:', error); - // When stream fails, close it first - await stream.close(); - console.log('Stream closed after error'); - - // Optional: Inspect what needs recovery (must be called on closed stream) + // Optional: Inspect what needs recovery after a terminal stream failure. const unackedBatches = await stream.getUnackedBatches(); console.log(`Batches to recover: ${unackedBatches.length}`); @@ -696,14 +706,24 @@ try { // 2. Creates a new stream with the same configuration // 3. Re-ingests all unacknowledged batches automatically // 4. Returns the new stream ready for continued use - const newStream = await sdk.recreateStream(stream); - console.log(`Stream recreated with ${unackedBatches.length} batches re-ingested`); - - // Continue using newStream for further ingestion try { - // Continue ingesting... + const newStream = await sdk.recreateStream(stream); + console.log(`Stream recreated with ${unackedBatches.length} batches re-ingested`); + + // Continue using newStream for further ingestion + try { + // Continue ingesting... + } finally { + await newStream.close(); + } } finally { - await newStream.close(); + // close() releases the failed wrapper's native handle even if it reports + // the terminal stream error again. + try { + await stream.close(); + } catch (closeError) { + console.error('Failed stream released:', closeError); + } } } ``` @@ -766,7 +786,8 @@ This method is the **recommended approach** for recovering from stream failures. 4. Returns the new stream ready for continued ingestion **Parameters:** -- `stream` - The failed or closed stream to recreate +- `stream` - The terminally failed stream to recreate. Do not call `stream.close()` + first because the TypeScript wrapper releases its native handle on close. **Returns:** Promise resolving to a new `ZerobusStream` with all unacknowledged batches re-ingested @@ -775,14 +796,14 @@ This method is the **recommended approach** for recovering from stream failures. try { await stream.ingestRecords(batch); } catch (error) { - await stream.close(); // Automatically recreate stream and recover all unacked batches const newStream = await sdk.recreateStream(stream); // Continue ingesting with newStream } ``` -**Note:** This method preserves batch structure and re-ingests batches atomically. For debugging, you can inspect what was recovered using `getUnackedBatches()` after closing the stream. +**Note:** This method preserves batch structure and re-ingests batches atomically. For +debugging, inspect `getUnackedBatches()` after a terminal failure and before closing the wrapper. --- @@ -936,7 +957,8 @@ async getUnackedRecords(): Promise Returns unacknowledged record payloads as a flat array for inspection purposes. -**Important:** Can only be called on **closed streams**. Call `stream.close()` first, or this will throw an error. +**Important:** This can only be called after a terminal stream failure. Do not call +`stream.close()` first: the TypeScript wrapper releases the underlying stream handle on close. **Returns:** Array of Buffer containing the raw record payloads @@ -950,7 +972,8 @@ async getUnackedBatches(): Promise Returns unacknowledged records grouped by their original batches for inspection purposes. -**Important:** Can only be called on **closed streams**. Call `stream.close()` first, or this will throw an error. +**Important:** This can only be called after a terminal stream failure. Do not call +`stream.close()` first: the TypeScript wrapper releases the underlying stream handle on close. **Returns:** Array of arrays, where each inner array represents a batch of records as Buffers @@ -963,15 +986,11 @@ try { await stream.ingestRecords(batch2); // ... error occurs } catch (error) { - await stream.close(); const unackedBatches = await stream.getUnackedBatches(); // unackedBatches[0] contains records from batch1 (if not acked) // unackedBatches[1] contains records from batch2 (if not acked) - // Re-ingest with new stream - for (const batch of unackedBatches) { - await newStream.ingestRecords(batch); - } + console.log(`Batches available for recovery: ${unackedBatches.length}`); } ``` @@ -994,10 +1013,10 @@ interface TableProperties { ```typescript // JSON mode -const tableProperties = { tableName: 'main.default.air_quality' }; +const jsonTableProperties = { tableName: 'main.default.air_quality' }; // Protocol Buffers mode -const tableProperties = { +const protoTableProperties = { tableName: 'main.default.air_quality', descriptorProto: descriptorBase64 // Required for protobuf }; diff --git a/typescript/examples/json/README.md b/typescript/examples/json/README.md index f26c8d86..748e0530 100644 --- a/typescript/examples/json/README.md +++ b/typescript/examples/json/README.md @@ -67,13 +67,13 @@ Stream closed successfully ```typescript // 1. Auto-serializing: pass object directly const record = { device_name: 'sensor-001', temp: 22, humidity: 65 }; -const offset = await stream.ingestRecordOffset(record); -await stream.waitForOffset(offset); +const objectOffset = await stream.ingestRecordOffset(record); +await stream.waitForOffset(objectOffset); // 2. Pre-serialized: pass JSON string const jsonString = JSON.stringify({ device_name: 'sensor-002', temp: 24, humidity: 70 }); -const offset = await stream.ingestRecordOffset(jsonString); -await stream.waitForOffset(offset); +const stringOffset = await stream.ingestRecordOffset(jsonString); +await stream.waitForOffset(stringOffset); // 3. High-throughput: send many, wait once let lastOffset: bigint; @@ -137,22 +137,22 @@ Stream closed successfully ```typescript // 1. Auto-serializing: array of objects -const batch = [ +const objectBatch = [ { device_name: 'sensor-001', temp: 22, humidity: 65 }, { device_name: 'sensor-002', temp: 23, humidity: 67 }, { device_name: 'sensor-003', temp: 24, humidity: 69 } ]; -const offset = await stream.ingestRecordsOffset(batch); -if (offset !== null) { - await stream.waitForOffset(offset); +const objectBatchOffset = await stream.ingestRecordsOffset(objectBatch); +if (objectBatchOffset !== null) { + await stream.waitForOffset(objectBatchOffset); } // 2. Pre-serialized: array of JSON strings -const batch = [ +const stringBatch = [ JSON.stringify({ device_name: 'sensor-004', temp: 25, humidity: 71 }), JSON.stringify({ device_name: 'sensor-005', temp: 26, humidity: 73 }) ]; -const offset = await stream.ingestRecordsOffset(batch); +const stringBatchOffset = await stream.ingestRecordsOffset(stringBatch); ``` **Batch semantics:** diff --git a/typescript/examples/proto/README.md b/typescript/examples/proto/README.md index e12c243a..17318c4a 100644 --- a/typescript/examples/proto/README.md +++ b/typescript/examples/proto/README.md @@ -108,17 +108,17 @@ const AirQuality = airQuality.examples.AirQuality; // 1. Auto-encoding: pass message directly const record = AirQuality.create({ - device_name: 'sensor-001', + deviceName: 'sensor-001', temp: 22, humidity: 65 }); -const offset = await stream.ingestRecordOffset(record); -await stream.waitForOffset(offset); +const messageOffset = await stream.ingestRecordOffset(record); +await stream.waitForOffset(messageOffset); // 2. Pre-encoded: pass Buffer const buffer = Buffer.from(AirQuality.encode(record).finish()); -const offset = await stream.ingestRecordOffset(buffer); -await stream.waitForOffset(offset); +const bufferOffset = await stream.ingestRecordOffset(buffer); +await stream.waitForOffset(bufferOffset); ``` ## Batch Example @@ -143,19 +143,19 @@ await stream.waitForOffset(offset); ```typescript // 1. Auto-encoding: array of messages -const batch = [ - AirQuality.create({ device_name: 'sensor-001', temp: 22, humidity: 65 }), - AirQuality.create({ device_name: 'sensor-002', temp: 23, humidity: 67 }), - AirQuality.create({ device_name: 'sensor-003', temp: 24, humidity: 69 }) +const messageBatch = [ + AirQuality.create({ deviceName: 'sensor-001', temp: 22, humidity: 65 }), + AirQuality.create({ deviceName: 'sensor-002', temp: 23, humidity: 67 }), + AirQuality.create({ deviceName: 'sensor-003', temp: 24, humidity: 69 }) ]; -const offset = await stream.ingestRecordsOffset(batch); -if (offset !== null) { - await stream.waitForOffset(offset); +const messageBatchOffset = await stream.ingestRecordsOffset(messageBatch); +if (messageBatchOffset !== null) { + await stream.waitForOffset(messageBatchOffset); } // 2. Pre-encoded: array of Buffers -const batch = records.map(r => Buffer.from(AirQuality.encode(r).finish())); -const offset = await stream.ingestRecordsOffset(batch); +const bufferBatch = records.map(r => Buffer.from(AirQuality.encode(r).finish())); +const bufferBatchOffset = await stream.ingestRecordsOffset(bufferBatch); ``` ## Adapting for Your Custom Table diff --git a/typescript/examples/proto/batch.ts b/typescript/examples/proto/batch.ts index 1e112aae..97dc7ee9 100644 --- a/typescript/examples/proto/batch.ts +++ b/typescript/examples/proto/batch.ts @@ -105,9 +105,9 @@ async function main() { // 1. Auto-encoding: array of Message objects - SDK handles encoding const batch1 = [ - AirQuality.create({ device_name: 'sensor-001', temp: 22, humidity: 65 }), - AirQuality.create({ device_name: 'sensor-002', temp: 23, humidity: 67 }), - AirQuality.create({ device_name: 'sensor-003', temp: 24, humidity: 69 }) + AirQuality.create({ deviceName: 'sensor-001', temp: 22, humidity: 65 }), + AirQuality.create({ deviceName: 'sensor-002', temp: 23, humidity: 67 }), + AirQuality.create({ deviceName: 'sensor-003', temp: 24, humidity: 69 }) ]; const offset1 = await stream.ingestRecordsOffset(batch1); @@ -119,9 +119,9 @@ async function main() { // 2. Pre-encoded: array of Buffers const batch2 = [ - AirQuality.create({ device_name: 'sensor-004', temp: 25, humidity: 71 }), - AirQuality.create({ device_name: 'sensor-005', temp: 26, humidity: 73 }), - AirQuality.create({ device_name: 'sensor-006', temp: 27, humidity: 75 }) + AirQuality.create({ deviceName: 'sensor-004', temp: 25, humidity: 71 }), + AirQuality.create({ deviceName: 'sensor-005', temp: 26, humidity: 73 }), + AirQuality.create({ deviceName: 'sensor-006', temp: 27, humidity: 75 }) ].map(record => Buffer.from(AirQuality.encode(record).finish())); const offset2 = await stream.ingestRecordsOffset(batch2); @@ -135,7 +135,7 @@ async function main() { console.log('\n[Large batch] Sending batch of 100 records...'); const largeBatch = Array.from({ length: 100 }, (_, i) => AirQuality.create({ - device_name: `sensor-${i.toString().padStart(3, '0')}`, + deviceName: `sensor-${i.toString().padStart(3, '0')}`, temp: 20 + (i % 15), humidity: 50 + (i % 40) }) @@ -160,9 +160,9 @@ async function main() { // 1. Auto-encoding: array of Message objects const batch1 = [ - AirQuality.create({ device_name: 'sensor-legacy-001', temp: 28, humidity: 77 }), - AirQuality.create({ device_name: 'sensor-legacy-002', temp: 29, humidity: 79 }), - AirQuality.create({ device_name: 'sensor-legacy-003', temp: 30, humidity: 81 }) + AirQuality.create({ deviceName: 'sensor-legacy-001', temp: 28, humidity: 77 }), + AirQuality.create({ deviceName: 'sensor-legacy-002', temp: 29, humidity: 79 }), + AirQuality.create({ deviceName: 'sensor-legacy-003', temp: 30, humidity: 81 }) ]; const offset1 = await stream.ingestRecords(batch1); @@ -172,9 +172,9 @@ async function main() { // 2. Pre-encoded: array of Buffers const batch2 = [ - AirQuality.create({ device_name: 'sensor-legacy-004', temp: 31, humidity: 83 }), - AirQuality.create({ device_name: 'sensor-legacy-005', temp: 32, humidity: 85 }), - AirQuality.create({ device_name: 'sensor-legacy-006', temp: 33, humidity: 87 }) + AirQuality.create({ deviceName: 'sensor-legacy-004', temp: 31, humidity: 83 }), + AirQuality.create({ deviceName: 'sensor-legacy-005', temp: 32, humidity: 85 }), + AirQuality.create({ deviceName: 'sensor-legacy-006', temp: 33, humidity: 87 }) ].map(record => Buffer.from(AirQuality.encode(record).finish())); const offset2 = await stream.ingestRecords(batch2); diff --git a/typescript/examples/proto/single.ts b/typescript/examples/proto/single.ts index f056e627..047aa1fe 100644 --- a/typescript/examples/proto/single.ts +++ b/typescript/examples/proto/single.ts @@ -110,7 +110,7 @@ async function main() { // 1. Auto-encoding: Message object - SDK handles encoding const record1 = AirQuality.create({ - device_name: 'sensor-001', + deviceName: 'sensor-001', temp: 22, humidity: 65 }); @@ -122,7 +122,7 @@ async function main() { // 2. Pre-encoded: Buffer - pass pre-serialized bytes const record2 = AirQuality.create({ - device_name: 'sensor-002', + deviceName: 'sensor-002', temp: 24, humidity: 70 }); @@ -138,7 +138,7 @@ async function main() { let lastOffset: bigint = BigInt(0); for (let i = 0; i < 10; i++) { const record = AirQuality.create({ - device_name: `sensor-${i.toString().padStart(3, '0')}`, + deviceName: `sensor-${i.toString().padStart(3, '0')}`, temp: 20 + i, humidity: 50 + i * 2 }); @@ -156,7 +156,7 @@ async function main() { // 1. Auto-encoding: Message object const record1 = AirQuality.create({ - device_name: 'sensor-legacy-001', + deviceName: 'sensor-legacy-001', temp: 23, humidity: 68 }); @@ -166,7 +166,7 @@ async function main() { // 2. Pre-encoded: Buffer const record2 = AirQuality.create({ - device_name: 'sensor-legacy-002', + deviceName: 'sensor-legacy-002', temp: 25, humidity: 72 }); diff --git a/typescript/src/headers_provider.ts b/typescript/src/headers_provider.ts index d4ae76d2..678b930a 100644 --- a/typescript/src/headers_provider.ts +++ b/typescript/src/headers_provider.ts @@ -42,24 +42,20 @@ export interface HeadersProvider { * ); * ``` * - * **How to use custom authentication (PAT, etc.):** + * The native `createStream` adapter currently requires its callback to return + * header tuples synchronously. Pass a callback object directly: * ```typescript - * class CustomHeadersProvider implements HeadersProvider { - * async getHeaders() { - * return [ - * ["authorization", `Bearer ${myToken}`], - * ["x-databricks-zerobus-table-name", tableName] - * ]; - * } - * } - * - * const provider = new CustomHeadersProvider(); * const stream = await sdk.createStream( * tableProperties, * '', // ignored * '', // ignored * options, - * { getHeadersCallback: provider.getHeaders.bind(provider) } + * { + * getHeadersCallback: () => [ + * ["authorization", `Bearer ${myToken}`], + * ["x-databricks-zerobus-table-name", tableName] + * ] + * } * ); * ``` */ diff --git a/typescript/src/lib.rs b/typescript/src/lib.rs index c0390e58..0abb81ed 100644 --- a/typescript/src/lib.rs +++ b/typescript/src/lib.rs @@ -220,8 +220,7 @@ fn convert_js_to_record_payload(env: &Env, payload: Unknown) -> Result [ + /// getHeadersCallback: () => [ /// ["authorization", `Bearer ${myToken}`], /// ["x-databricks-zerobus-table-name", tableName] /// ] @@ -1119,7 +1118,8 @@ impl ZerobusSdk { /// /// # Arguments /// - /// * `stream` - The failed or closed stream to recreate + /// * `stream` - The terminally failed stream to recreate. The TypeScript wrapper + /// must not have been closed because `close()` releases its native handle. /// /// # Returns /// @@ -1137,10 +1137,21 @@ impl ZerobusSdk { /// try { /// await stream.ingestRecords(batch); /// } catch (error) { - /// await stream.close(); /// // Recreate stream with all unacked batches re-ingested - /// const newStream = await sdk.recreateStream(stream); - /// // Continue ingesting with newStream + /// try { + /// const newStream = await sdk.recreateStream(stream); + /// try { + /// // Continue ingesting with newStream + /// } finally { + /// await newStream.close(); + /// } + /// } finally { + /// try { + /// await stream.close(); + /// } catch (closeError) { + /// console.error("Failed stream released:", closeError); + /// } + /// } /// } /// ``` #[napi] diff --git a/typescript/tsconfig.json b/typescript/tsconfig.json index daf4afa8..475ad4d0 100644 --- a/typescript/tsconfig.json +++ b/typescript/tsconfig.json @@ -5,7 +5,7 @@ "lib": ["ES2020"], "declaration": true, "outDir": "./dist", - "rootDir": "./examples", + "rootDir": ".", "strict": true, "esModuleInterop": true, "skipLibCheck": true, From ded6311f639c5cfa1b132a75d2b60ee9b4c8d530 Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Thu, 13 Aug 2026 14:23:14 +0000 Subject: [PATCH 02/36] Address Python review comments --- python/CLAUDE.md | 10 +-- python/NEXT_CHANGELOG.md | 5 ++ python/README.md | 85 ++++++++++++++++---------- python/examples/README.md | 42 ++++++------- python/examples/async_example_json.py | 24 +------- python/examples/async_example_proto.py | 21 +------ python/examples/sync_example_json.py | 23 +------ python/examples/sync_example_proto.py | 20 +----- python/zerobus/__init__.py | 2 +- python/zerobus/_zerobus_core.pyi | 11 ++-- python/zerobus/sdk/aio/zerobus_sdk.py | 19 +++--- python/zerobus/sdk/shared/config.py | 13 ++-- python/zerobus/sdk/sync/zerobus_sdk.py | 19 +++--- 13 files changed, 123 insertions(+), 171 deletions(-) diff --git a/python/CLAUDE.md b/python/CLAUDE.md index 73a0dacb..7d745b6f 100644 --- a/python/CLAUDE.md +++ b/python/CLAUDE.md @@ -7,10 +7,12 @@ Python wrapper around the Rust core via PyO3 and maturin. When writing or reviewing client code that uses this SDK, follow the cross-SDK performance flow from the root `CLAUDE.md`: -- **Idiomatic flow:** ingest in a loop (`ingest_record_offset()` or - `ingest_record_nowait()`), then call `flush()` once to confirm durability. - `ingest_record_offset()` returns as soon as the record is queued; the SDK sends it and - tracks its acknowledgment in the background. +- **Idiomatic flow:** ingest in a loop with `ingest_record_offset()`, or prefer + `ingest_records_offset()` for bulk data, then call `flush()` once to confirm + durability. `ingest_record_offset()` returns as soon as the record is queued; the SDK + sends it and tracks its acknowledgment in the background. Do not feature + `ingest_record_nowait()` in examples: it spawns a detached task and is not safely + synchronized with `flush()`. - In async code, an `AckCallback` is a good way to track durability without blocking. - `wait_for_offset()` blocks until a specific offset is acknowledged — use it to confirm a specific record before continuing. Acks are ordered, so waiting on the LAST offset diff --git a/python/NEXT_CHANGELOG.md b/python/NEXT_CHANGELOG.md index 3c2c1bb3..3c9a095b 100644 --- a/python/NEXT_CHANGELOG.md +++ b/python/NEXT_CHANGELOG.md @@ -13,6 +13,11 @@ - Corrected README, example, and docstring snippets for record-format selection, exception handling, recovery, iterator return values, custom headers, async contexts, and durability-aware throughput measurement. +- Documented that `get_unacked_records()` and `recreate_stream()` require a closed + stream, and that enqueue failures must be closed before recovery. +- Removed nowait APIs from featured examples. Those calls spawn detached tasks and + are not safely synchronized with `flush()`. Recommend `ingest_records_offset()` + plus one `flush()` for bulk ingestion. ### Internal Changes diff --git a/python/README.md b/python/README.md index 322871b4..f189a814 100644 --- a/python/README.md +++ b/python/README.md @@ -251,7 +251,7 @@ try: temp=20 + (i % 15), humidity=50 + (i % 40) ) - stream.ingest_record_nowait(record) + stream.ingest_record_offset(record) stream.flush() finally: stream.close() @@ -277,7 +277,7 @@ async def main(): temp=20 + (i % 15), humidity=50 + (i % 40) ) - stream.ingest_record_nowait(record) + await stream.ingest_record_offset(record) await stream.flush() finally: await stream.close() @@ -328,7 +328,7 @@ compatibility but does not select the format. | `server_lack_of_ack_timeout_ms` | `int` | `60000` | Server acknowledgment timeout (ms) | | `stream_paused_max_wait_time_ms` | `Optional[int]` | `None` | Max wait during graceful stream close. `None` = full server duration, `0` = immediate, `x` = min(x, server_duration) | | `callback_max_wait_time_ms` | `Optional[int]` | `5000` | Max wait for callbacks after `close()`. `None` = wait forever | -| `ack_callback` | `AckCallback` | `None` | Callback invoked once per logical ingest submission (one record call or one batch call) | +| `ack_callback` | `AckCallback` | `None` | Callback invoked once per successfully queued ingest submission that later acknowledges or fails | ## Error Handling @@ -348,8 +348,11 @@ except ZerobusException as e: ## Handling Stream Failures -The SDK automatically handles retries for transient errors. Use `get_unacked_records()` only after -the stream has permanently closed following a failure: +The SDK automatically handles retries for transient errors. Enqueue, flush, and close +failures all surface as `ZerobusException`. An enqueue failure can leave the stream +active, and both `get_unacked_records()` and `recreate_stream()` require a closed +stream. Close first, then inspect or recreate. `recreate_stream()` re-queues records +that were already accepted; it does not retry a payload that failed to enqueue. ```python from zerobus.sdk.shared import ZerobusException @@ -357,15 +360,24 @@ from zerobus.sdk.shared import ZerobusException try: for i in range(10000): stream.ingest_record_offset(record) - stream.close() + stream.flush() except ZerobusException as e: + print(f"Ingestion failed: {e}") + try: + stream.close() + except ZerobusException: + pass + unacked = list(stream.get_unacked_records()) - print(f"Stream failed: {e}. {len(unacked)} records unacknowledged.") + print(f"{len(unacked)} previously queued records were unacknowledged.") - # Preserve the record format and original batch grouping while retrying. new_stream = sdk.recreate_stream(stream) - new_stream.flush() - new_stream.close() + try: + new_stream.flush() + finally: + new_stream.close() +else: + stream.close() ``` Use `get_unacked_batches()` to inspect the original batch grouping after the stream closes: @@ -382,26 +394,33 @@ print(f"{len(unacked_batches)} batches remain unacknowledged") ## Performance Tips -The idiomatic flow is to ingest in a loop and `flush()` once — ingest calls queue -immediately and the SDK acknowledges records in the background, so a single `flush()` -confirms everything queued so far. The ack watermark is monotonic, so if you want a -durability checkpoint mid-stream, waiting on the last offset returned confirms every -prior record. In async code, an [`AckCallback`](#ackcallback) tracks durability without -blocking. Calling `wait_for_offset()` after every record in a tight loop limits +The reliable bulk path is `ingest_records_offset()` plus one `flush()`. That call +amortizes the Python-to-Rust crossing and returns an offset after the batch is queued. +For single records, use `ingest_record_offset()` in a loop and `flush()` once. Ingest +calls queue immediately and the SDK acknowledges records in the background, so a single +`flush()` confirms everything queued so far. The ack watermark is monotonic, so if you +want a durability checkpoint mid-stream, waiting on the last offset returned confirms +every prior record. In async code, an [`AckCallback`](#ackcallback) tracks durability +without blocking. Calling `wait_for_offset()` after every record in a tight loop limits throughput to one record per round-trip, so save it for confirming a specific record. -| Method | Throughput | Use case | -| ------------------------ | ----------- | -------------------------------------------------------------------------------------------------------------------- | -| `ingest_record_nowait()` | **Highest** | Fire-and-forget: no offset returned; maximum throughput when you do not need per-record ack tracking in the hot path | -| `ingest_record_offset()` | Medium | Recommended for most apps: returns an offset after queueing. Ingest in a loop, then `flush()` once | -| `ingest_record()` | Low | **Deprecated** — prefer offset-based APIs | +`ingest_record_nowait()` and `ingest_records_nowait()` spawn detached tasks and discard +enqueue errors. `flush()` can complete before those tasks allocate offsets, so they are +not a safe durability path. Prefer the offset APIs. + +| Method | Throughput | Use case | +| -------------------------- | ---------- | ------------------------------------------------------------------------------------------------- | +| `ingest_records_offset()` | Highest | Recommended bulk path: queue a batch, then `flush()` once | +| `ingest_record_offset()` | Medium | Recommended for single records: ingest in a loop, then `flush()` once | +| `ingest_record()` | Low | Deprecated; prefer offset-based APIs | +| `ingest_record_nowait()` | Unsafe | Detached fire-and-forget; enqueue errors can be lost and are not synchronized with `flush()` | +| `ingest_records_nowait()` | Unsafe | Detached batch fire-and-forget; same durability caveats as `ingest_record_nowait()` | -**Idiomatic flow:** +Idiomatic flow: ```python async def ingest_all(stream, records): - for record in records: - await stream.ingest_record_offset(record) # queues immediately, no round-trip + await stream.ingest_records_offset(records) # queues the batch, no round-trip await stream.flush() # one wait for everything ``` @@ -444,18 +463,18 @@ async def create_async_stream(sdk): **Single record ingestion:** -| Method | Sync | Async | Notes | -| ------------------------------ | ------------------------ | -------------------- | ----------------------------------- | -| `ingest_record_nowait(record)` | `→ None` | `→ None` (not async) | Fire-and-forget, highest throughput | -| `ingest_record_offset(record)` | `→ int` | `await → int` | Returns offset after queueing | -| `ingest_record(record)` | `→ RecordAcknowledgment` | `await → Awaitable` | **Deprecated** since v0.3.0 | +| Method | Sync | Async | Notes | +| ------------------------------ | ------------------------ | -------------------- | --------------------------------------------------------------------- | +| `ingest_record_offset(record)` | `→ int` | `await → int` | Recommended for single records; returns offset after queueing | +| `ingest_record(record)` | `→ RecordAcknowledgment` | `await → Awaitable` | Deprecated since v0.3.0 | +| `ingest_record_nowait(record)` | `→ None` | `→ None` (not async) | Detached fire-and-forget; enqueue errors are not synchronized with `flush()` | **Batch ingestion:** -| Method | Sync | Async | Notes | -| -------------------------------- | -------- | -------------------- | -------------------- | -| `ingest_records_nowait(records)` | `→ None` | `→ None` (not async) | Fire-and-forget | -| `ingest_records_offset(records)` | `→ int` | `await → int` | Returns final offset | +| Method | Sync | Async | Notes | +| -------------------------------- | -------- | -------------------- | --------------------------------------------------------------------- | +| `ingest_records_offset(records)` | `→ int` | `await → int` | Recommended bulk path; returns the batch's final offset | +| `ingest_records_nowait(records)` | `→ None` | `→ None` (not async) | Detached fire-and-forget; same durability caveats as `ingest_record_nowait()` | **Accepted record types:** diff --git a/python/examples/README.md b/python/examples/README.md index 864fcb55..907e3674 100644 --- a/python/examples/README.md +++ b/python/examples/README.md @@ -54,14 +54,14 @@ python examples/async_example_arrow.py # Arrow Flight ## Examples Overview -All examples demonstrate multiple ingestion methods: +All examples demonstrate the recommended offset APIs: -1. **`ingest_record_offset()`** - Single record with offset tracking -2. **`ingest_records_offset()`** - Batch ingestion with offset tracking -3. **`ingest_record_nowait()`** - Fire-and-forget single record -4. **`ingest_records_nowait()`** - Fire-and-forget batch (highest throughput) +1. `ingest_record_offset()` - Single record with offset tracking +2. `ingest_records_offset()` - Batch ingestion with offset tracking (preferred bulk path) -Each example includes detailed comments explaining when to use each method and their performance characteristics. +Queue records or batches, then call `flush()` once to confirm durability. The nowait APIs +are not shown because they spawn detached tasks and are not safely synchronized with +`flush()`. ### Serialization Formats @@ -80,11 +80,11 @@ More efficient over the wire. You can pass either: record = record_pb2.AirQuality(device_name="sensor-1", temp=25, humidity=60) table_properties = TableProperties(TABLE_NAME, record_pb2.AirQuality.DESCRIPTOR) -# Recommended: Use ingest_record_offset() for better performance +# Recommended: ingest_record_offset() then flush() once offset = stream.ingest_record_offset(record) -# Or fire-and-forget for maximum throughput -stream.ingest_record_nowait(record) +# Preferred bulk path: ingest_records_offset() then flush() once +# batch_offset = stream.ingest_records_offset([record]) # Option 2: Pass pre-serialized bytes (client controls serialization) # offset = stream.ingest_record_offset(record.SerializeToString()) @@ -102,11 +102,11 @@ Good for getting started. No protobuf schema required. You can pass either: record_dict = {"device_name": "sensor-1", "temp": 25, "humidity": 60} table_properties = TableProperties(TABLE_NAME) -# Recommended: Use ingest_record_offset() for better performance +# Recommended: ingest_record_offset() then flush() once offset = stream.ingest_record_offset(record_dict) -# Or fire-and-forget for maximum throughput -stream.ingest_record_nowait(record_dict) +# Preferred bulk path: ingest_records_offset() then flush() once +# batch_offset = stream.ingest_records_offset([record_dict]) # Option 2: Pass pre-serialized JSON string (client controls serialization) # offset = stream.ingest_record_offset(json.dumps(record_dict)) @@ -150,7 +150,7 @@ Both APIs provide the same functionality and performance. The key differences ar | Import | `from zerobus.sdk.sync import ZerobusSdk` | `from zerobus.sdk.aio import ZerobusSdk` | | Stream creation | `stream = sdk.create_stream(...)` | `stream = await sdk.create_stream(...)` | | Record ingestion (with offset) | `offset = stream.ingest_record_offset(record)` | `offset = await stream.ingest_record_offset(record)` | -| Record ingestion (fire-and-forget) | `stream.ingest_record_nowait(record)` | `stream.ingest_record_nowait(record)` | +| Batch ingestion (with offset) | `offset = stream.ingest_records_offset(records)` | `offset = await stream.ingest_records_offset(records)` | | Flush | `stream.flush()` | `await stream.flush()` | | Close | `stream.close()` | `await stream.close()` | | Execution context | Standard Python | Requires asyncio event loop | @@ -158,18 +158,16 @@ Both APIs provide the same functionality and performance. The key differences ar **Performance:** Both APIs offer equivalent throughput and durability. Choose based on your application's architecture, not performance needs. -**Recommended Methods:** +Recommended methods: -**Single Record Ingestion:** -- `ingest_record_offset()` - Returns offset immediately, use when you need to track offsets -- `ingest_record_nowait()` - Fire-and-forget, best for maximum throughput +- `ingest_records_offset()` - Preferred bulk path: queue a batch, then `flush()` once +- `ingest_record_offset()` - Single records: ingest in a loop, then `flush()` once -**Batch Ingestion:** -- `ingest_records_offset()` - Batch multiple records, returns final offset -- `ingest_records_nowait()` - Fire-and-forget batch ingestion, most efficient for bulk data +Deprecated: -**Deprecated:** -- `ingest_record()` - Use `ingest_record_offset()` instead (2-40x slower) +- `ingest_record()` - Use `ingest_record_offset()` instead + +The nowait APIs spawn detached tasks and are not safely synchronized with `flush()`. ### Serialization Format Comparison diff --git a/python/examples/async_example_json.py b/python/examples/async_example_json.py index 22807d6b..f8b11aaf 100644 --- a/python/examples/async_example_json.py +++ b/python/examples/async_example_json.py @@ -220,31 +220,11 @@ async def main(): logger.info(f" Batch {batch_num + 1}: {len(batch)} records, offset: {batch_offset}") success_count += len(batch) - # ======================================================================== - # Method 3: ingest_record_nowait() - Fire-and-forget for max throughput - # Best for high-throughput scenarios with callback-based ack tracking - # ======================================================================== - logger.info("\n3. Using ingest_record_nowait() - fire-and-forget") - remaining = NUM_RECORDS - success_count - if remaining > 0: - for i in range(min(100, remaining)): - idx = success_count + i - record_dict = create_sample_json_record(idx) - stream.ingest_record_nowait(record_dict) - - logger.info(f" Queued {min(100, remaining)} records (tracking via callback)") - success_count += min(100, remaining) - - # ======================================================================== - # Method 4: ingest_records_nowait() - Batch fire-and-forget - # Best for maximum throughput with batch efficiency - # ======================================================================== - logger.info("\n4. Using ingest_records_nowait() - batch fire-and-forget") remaining = NUM_RECORDS - success_count if remaining > 0: batch = [create_sample_json_record(success_count + i) for i in range(remaining)] - stream.ingest_records_nowait(batch) - logger.info(f" Queued {len(batch)} records in batch (tracking via callback)") + batch_offset = await stream.ingest_records_offset(batch) + logger.info(f" Remaining {len(batch)} records, offset: {batch_offset}") success_count += len(batch) submit_end_time = time.time() diff --git a/python/examples/async_example_proto.py b/python/examples/async_example_proto.py index 7711eafc..8dd7fb6d 100644 --- a/python/examples/async_example_proto.py +++ b/python/examples/async_example_proto.py @@ -217,28 +217,11 @@ async def main(): logger.info(f" Batch {batch_num + 1}: {len(batch)} records, offset: {batch_offset}") success_count += len(batch) - # ======================================================================== - # Method 3: ingest_record_nowait() - Fire-and-forget - # ======================================================================== - logger.info("\n3. Using ingest_record_nowait() - fire-and-forget") - remaining = NUM_RECORDS - success_count - if remaining > 0: - for i in range(min(100, remaining)): - record = create_sample_record(success_count + i) - stream.ingest_record_nowait(record) - - logger.info(f" Queued {min(100, remaining)} records (tracking via callback)") - success_count += min(100, remaining) - - # ======================================================================== - # Method 4: ingest_records_nowait() - Batch fire-and-forget - # ======================================================================== - logger.info("\n4. Using ingest_records_nowait() - batch fire-and-forget") remaining = NUM_RECORDS - success_count if remaining > 0: batch = [create_sample_record(success_count + i) for i in range(remaining)] - stream.ingest_records_nowait(batch) - logger.info(f" Queued {len(batch)} records in batch (tracking via callback)") + batch_offset = await stream.ingest_records_offset(batch) + logger.info(f" Remaining {len(batch)} records, offset: {batch_offset}") success_count += len(batch) submit_end_time = time.time() diff --git a/python/examples/sync_example_json.py b/python/examples/sync_example_json.py index ac0b14b1..2e95de51 100644 --- a/python/examples/sync_example_json.py +++ b/python/examples/sync_example_json.py @@ -195,30 +195,11 @@ def main(): logger.info(f" Batch {batch_num + 1}: {len(batch)} records, offset: {batch_offset}") success_count += len(batch) - # ======================================================================== - # Method 3: ingest_record_nowait() - Maximum throughput (fire-and-forget) - # Use when you don't need individual offsets and want maximum speed - # ======================================================================== - logger.info("\n3. Using ingest_record_nowait() - fire-and-forget") - remaining = NUM_RECORDS - success_count - if remaining > 0: - for i in range(min(10, remaining)): - idx = success_count + i - record_dict = create_sample_json_record(idx) - stream.ingest_record_nowait(record_dict) - logger.info(f" Queued {min(10, remaining)} records (no wait for ack)") - success_count += min(10, remaining) - - # ======================================================================== - # Method 4: ingest_records_nowait() - Batch fire-and-forget - # Combines batch efficiency with fire-and-forget speed - # ======================================================================== - logger.info("\n4. Using ingest_records_nowait() - batch fire-and-forget") remaining = NUM_RECORDS - success_count if remaining > 0: batch = [create_sample_json_record(success_count + i) for i in range(remaining)] - stream.ingest_records_nowait(batch) - logger.info(f" Queued {len(batch)} records in batch (no wait for ack)") + batch_offset = stream.ingest_records_offset(batch) + logger.info(f" Remaining {len(batch)} records, offset: {batch_offset}") success_count += len(batch) # ======================================================================== diff --git a/python/examples/sync_example_proto.py b/python/examples/sync_example_proto.py index dc521e95..cbfbf5a6 100644 --- a/python/examples/sync_example_proto.py +++ b/python/examples/sync_example_proto.py @@ -193,27 +193,11 @@ def main(): logger.info(f" Batch {batch_num + 1}: {len(batch)} records, offset: {batch_offset}") success_count += len(batch) - # ======================================================================== - # Method 3: ingest_record_nowait() - Fire-and-forget - # ======================================================================== - logger.info("\n3. Using ingest_record_nowait() - fire-and-forget") - remaining = NUM_RECORDS - success_count - if remaining > 0: - for i in range(min(10, remaining)): - record = create_sample_record(success_count + i) - stream.ingest_record_nowait(record) - logger.info(f" Queued {min(10, remaining)} records (no wait for ack)") - success_count += min(10, remaining) - - # ======================================================================== - # Method 4: ingest_records_nowait() - Batch fire-and-forget - # ======================================================================== - logger.info("\n4. Using ingest_records_nowait() - batch fire-and-forget") remaining = NUM_RECORDS - success_count if remaining > 0: batch = [create_sample_record(success_count + i) for i in range(remaining)] - stream.ingest_records_nowait(batch) - logger.info(f" Queued {len(batch)} records in batch (no wait for ack)") + batch_offset = stream.ingest_records_offset(batch) + logger.info(f" Remaining {len(batch)} records, offset: {batch_offset}") success_count += len(batch) # ======================================================================== diff --git a/python/zerobus/__init__.py b/python/zerobus/__init__.py index c86ffa07..fff1823f 100644 --- a/python/zerobus/__init__.py +++ b/python/zerobus/__init__.py @@ -10,7 +10,7 @@ >>> # Define a custom callback >>> class MyCallback(AckCallback): ... def on_ack(self, offset): - ... print(f"Record acknowledged at offset {offset}") + ... print(f"Submission acknowledged at offset {offset}") >>> >>> sdk = ZerobusSdk( ... host="https://your-shard-id.zerobus.region.cloud.databricks.com", diff --git a/python/zerobus/_zerobus_core.pyi b/python/zerobus/_zerobus_core.pyi index 346a2291..ffb71efa 100644 --- a/python/zerobus/_zerobus_core.pyi +++ b/python/zerobus/_zerobus_core.pyi @@ -45,8 +45,11 @@ class AckCallback: Base class for logical ingest submission acknowledgment callbacks. Subclass this in Python to create custom callbacks that are invoked - once per logical ingest submission. A batch submission produces one - callback, not one callback per record in the batch. + once per successfully queued logical ingest submission that later + acknowledges or fails. A batch that is accepted by the stream produces + one callback, not one callback per record in the batch. Pre-queue + validation, size, type, and closed-stream failures raise immediately + and do not generate a callback. Example: class MyCallback(AckCallback): @@ -115,7 +118,7 @@ class StreamConfigurationOptions: """Maximum time in milliseconds to wait for callbacks to finish after calling close() (default: 5000)""" ack_callback: Optional[AckCallback] - """Callback invoked once per logical ingest submission (default: None)""" + """Callback invoked once per successfully queued ingest submission that later acknowledges or fails (default: None)""" def __init__( self, @@ -145,7 +148,7 @@ class StreamConfigurationOptions: record_type: Serialization format (default: RecordType.PROTO) stream_paused_max_wait_time_ms: Max wait time during graceful close in ms (default: None) callback_max_wait_time_ms: Max wait time for callbacks after close in ms (default: 5000) - ack_callback: Callback invoked once per logical ingest submission (default: None) + ack_callback: Callback invoked once per successfully queued ingest submission that later acknowledges or fails (default: None) """ ... diff --git a/python/zerobus/sdk/aio/zerobus_sdk.py b/python/zerobus/sdk/aio/zerobus_sdk.py index f67b9452..077f7307 100644 --- a/python/zerobus/sdk/aio/zerobus_sdk.py +++ b/python/zerobus/sdk/aio/zerobus_sdk.py @@ -33,13 +33,6 @@ ... '{"value": "record2"}', ... ]) ... - ... # Fire-and-forget for maximum throughput - ... stream.ingest_record_nowait('{"value": "record3"}') # Not awaited! - ... stream.ingest_records_nowait([ - ... '{"value": "record4"}', - ... '{"value": "record5"}', - ... ]) # Not awaited! - ... ... await stream.flush() # Ensure all records are sent ... await stream.close() >>> @@ -117,9 +110,9 @@ async def ingest_record_offset(self, payload: Any): def ingest_record_nowait(self, payload: Any): """Submit record without waiting (fire-and-forget). - Highest-throughput single-record API: returns no offset and is not awaited. - Use when you do not need per-record offsets; track durability with an - ``AckCallback`` and ``await flush()`` before close. + Spawns a detached task and discards enqueue errors. ``flush()`` can complete + before the task allocates an offset, so this is not a safe durability path. + Prefer ``ingest_record_offset()`` or ``ingest_records_offset()``. """ return self._inner.ingest_record_nowait(payload) @@ -128,7 +121,11 @@ async def ingest_records_offset(self, payloads): return await self._inner.ingest_records_offset(payloads) def ingest_records_nowait(self, payloads): - """Submit batch of records without waiting.""" + """Submit batch of records without waiting. + + Same detached-task caveats as ``ingest_record_nowait()``. Prefer + ``ingest_records_offset()``. + """ return self._inner.ingest_records_nowait(payloads) async def wait_for_offset(self, offset: int): diff --git a/python/zerobus/sdk/shared/config.py b/python/zerobus/sdk/shared/config.py index a04d1bd1..243368af 100644 --- a/python/zerobus/sdk/shared/config.py +++ b/python/zerobus/sdk/shared/config.py @@ -14,8 +14,10 @@ Base class for logical ingest submission acknowledgment callbacks. Subclass this in Python to create custom callbacks that are invoked once per -logical ingest submission. A batch submission produces one callback, not one -callback per record in the batch. +successfully queued logical ingest submission that later acknowledges or fails. +A batch that is accepted by the stream produces one callback, not one callback +per record in the batch. Pre-queue validation, size, type, and closed-stream +failures raise immediately and do not generate a callback. Example: >>> class MyCallback(AckCallback): @@ -77,9 +79,10 @@ - None: Wait forever - x: Wait up to x milliseconds Default: 5000 - ack_callback: Callback invoked once per logical ingest submission when it is - acknowledged or encounters an error. A batch submission produces one callback. - Must be a class extending AckCallback. Default: None + ack_callback: Callback invoked once per successfully queued logical ingest + submission when it later acknowledges or fails. A batch that is accepted + by the stream produces one callback. Pre-queue failures do not generate a + callback. Must be a class extending AckCallback. Default: None Example: >>> from zerobus.sdk.shared import StreamConfigurationOptions, AckCallback diff --git a/python/zerobus/sdk/sync/zerobus_sdk.py b/python/zerobus/sdk/sync/zerobus_sdk.py index d4a8900a..9f142258 100644 --- a/python/zerobus/sdk/sync/zerobus_sdk.py +++ b/python/zerobus/sdk/sync/zerobus_sdk.py @@ -30,13 +30,6 @@ ... '{"value": "record2"}', ... ]) >>> - >>> # Fire-and-forget for maximum throughput - >>> stream.ingest_record_nowait('{"value": "record3"}') - >>> stream.ingest_records_nowait([ - ... '{"value": "record4"}', - ... '{"value": "record5"}', - ... ]) - >>> >>> stream.flush() # Ensure all records are sent >>> stream.close() """ @@ -79,9 +72,9 @@ def ingest_record_offset(self, payload): def ingest_record_nowait(self, payload): """Submit record without waiting (fire-and-forget). - Highest-throughput single-record API: returns no offset. Use when you do not - need per-record offsets; track durability with an ``AckCallback`` and call - ``flush()`` before close. + Spawns a detached task and discards enqueue errors. ``flush()`` can complete + before the task allocates an offset, so this is not a safe durability path. + Prefer ``ingest_record_offset()`` or ``ingest_records_offset()``. """ return self._inner.ingest_record_nowait(payload) @@ -90,7 +83,11 @@ def ingest_records_offset(self, payloads): return self._inner.ingest_records_offset(payloads) def ingest_records_nowait(self, payloads): - """Submit batch of records without waiting.""" + """Submit batch of records without waiting. + + Same detached-task caveats as ``ingest_record_nowait()``. Prefer + ``ingest_records_offset()``. + """ return self._inner.ingest_records_nowait(payloads) def wait_for_offset(self, offset: int): From 800b423d6a3d32f6ead14508e723458bf961ce98 Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Thu, 13 Aug 2026 14:23:38 +0000 Subject: [PATCH 03/36] Address TypeScript review comments --- typescript/NEXT_CHANGELOG.md | 5 +++ typescript/README.md | 54 +++++++++++++---------------- typescript/examples/json/README.md | 22 +++++------- typescript/examples/json/batch.ts | 16 ++++----- typescript/examples/proto/README.md | 28 +++++---------- typescript/examples/proto/batch.ts | 16 ++++----- typescript/src/headers_provider.ts | 44 ++++++++++------------- typescript/src/lib.rs | 9 ++--- typescript/test/unit.test.ts | 38 +++++++++----------- 9 files changed, 99 insertions(+), 133 deletions(-) diff --git a/typescript/NEXT_CHANGELOG.md b/typescript/NEXT_CHANGELOG.md index 43190dbf..d1f516c9 100644 --- a/typescript/NEXT_CHANGELOG.md +++ b/typescript/NEXT_CHANGELOG.md @@ -31,6 +31,11 @@ - Corrected README, example, and JSDoc snippets for CommonJS async entry points, generated Protobuf field names, variable declarations, stream recovery, and custom-header callbacks. +- Documented the `HeadersProvider` shape that `createStream()` actually accepts + (`getHeadersCallback` returning header tuples synchronously). +- Documented that omitted `descriptorProto` does not select JSON, that the + inherited inflight default is 1,000,000, and that `close()` is still required + to flush. README `main().catch` handlers now set a non-zero exit code. - Clarified the high-throughput ingestion pattern across the README, API reference, JSDoc doc comments (`ingestRecordOffset`, `ingestRecordsOffset`, `waitForOffset`, `flush`), and diff --git a/typescript/README.md b/typescript/README.md index 98ef8a97..14f7d85b 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -166,6 +166,7 @@ try { main().catch((error) => { console.error('Fatal error:', error); + process.exitCode = 1; }); ``` @@ -342,6 +343,7 @@ try { main().catch((error) => { console.error('Fatal error:', error); + process.exitCode = 1; }); ``` @@ -611,7 +613,7 @@ const stream = await sdk.createStream( | Option | Default | Description | |--------|---------|-------------| | `recordType` | `RecordType.Proto` | Serialization format: `RecordType.Json` or `RecordType.Proto` | -| `maxInflightRequests` | 10,000 | Maximum number of unacknowledged requests | +| `maxInflightRequests` | 1,000,000 | Maximum number of unacknowledged requests | | `recovery` | true | Enable automatic stream recovery | | `recoveryTimeoutMs` | 15,000 | Timeout for recovery operations (ms) | | `recoveryBackoffMs` | 2,000 | Delay between recovery attempts (ms) | @@ -689,42 +691,34 @@ const descriptorBase64 = loadDescriptorProto({ The SDK includes automatic recovery for transient failures (enabled by default with `recovery: true`). For permanent failures, use `recreateStream()` to automatically recover all unacknowledged batches. Always use try/finally blocks to ensure streams are properly closed: ```typescript +let replacement; try { const offset = await stream.ingestRecordOffset(record); - await stream.waitForOffset(offset); + await stream.flush(); console.log(`Success: offset ${offset}`); } catch (error) { console.error('Ingestion failed:', error); - // Optional: Inspect what needs recovery after a terminal stream failure. - const unackedBatches = await stream.getUnackedBatches(); - console.log(`Batches to recover: ${unackedBatches.length}`); - - // Recommended recovery approach: Use recreateStream() - // This method: - // 1. Gets all unacknowledged batches from the failed stream - // 2. Creates a new stream with the same configuration - // 3. Re-ingests all unacknowledged batches automatically - // 4. Returns the new stream ready for continued use + // Recreate only after a terminal stream failure. Enqueue errors leave the + // wrapper active, and close() releases the native handle needed by recreateStream(). try { - const newStream = await sdk.recreateStream(stream); - console.log(`Stream recreated with ${unackedBatches.length} batches re-ingested`); - - // Continue using newStream for further ingestion - try { - // Continue ingesting... - } finally { - await newStream.close(); - } + const unackedBatches = await stream.getUnackedBatches(); + console.log(`Batches to recover: ${unackedBatches.length}`); + replacement = await sdk.recreateStream(stream); + await replacement.flush(); + } catch (recoveryError) { + console.error('Recovery skipped or failed:', recoveryError); } finally { - // close() releases the failed wrapper's native handle even if it reports - // the terminal stream error again. - try { - await stream.close(); - } catch (closeError) { - console.error('Failed stream released:', closeError); + if (replacement) { + await replacement.close(); } } +} finally { + try { + await stream.close(); + } catch (closeError) { + console.error('Failed stream released:', closeError); + } } ``` @@ -1033,7 +1027,7 @@ Configuration options for stream behavior. ```typescript interface StreamConfigurationOptions { recordType?: RecordType; // RecordType.Json or RecordType.Proto. Default: RecordType.Proto - maxInflightRequests?: number; // Default: 10,000 + maxInflightRequests?: number; // Default: 1,000,000 recovery?: boolean; // Default: true recoveryTimeoutMs?: number; // Default: 15,000 recoveryBackoffMs?: number; // Default: 2,000 @@ -1053,7 +1047,7 @@ enum RecordType { 1. **Reuse SDK instances**: Create one `ZerobusSdk` instance per application 2. **Stream lifecycle**: Always close streams in a `finally` block to ensure all records are flushed -3. **Batch size**: Adjust `maxInflightRequests` based on your throughput requirements (default: 10,000) +3. **Batch size**: Adjust `maxInflightRequests` based on your throughput requirements (default: 1,000,000) 4. **Error handling**: The stream handles errors internally with automatic retry. Only use `recreateStream()` for persistent failures after internal retries are exhausted. 5. **Use Protocol Buffers for production**: Protocol Buffers (the default) provides better performance and schema validation. Use JSON only when you need schema flexibility or for quick prototyping. 6. **Store credentials securely**: Use environment variables, never hardcode credentials @@ -1105,7 +1099,7 @@ This SDK wraps the high-performance [Rust Zerobus SDK](https://github.com/databr **Benefits:** - **Native performance** - Rust implementation for high-throughput ingestion - **Native async/await support** - Rust futures become JavaScript Promises -- **Automatic memory management** - No manual cleanup required +- **Automatic memory management** for native objects. You still must `await stream.close()` to flush and release the stream. - **Type safety** - Compile-time checks on both sides ## Community and Contributing diff --git a/typescript/examples/json/README.md b/typescript/examples/json/README.md index 748e0530..53568247 100644 --- a/typescript/examples/json/README.md +++ b/typescript/examples/json/README.md @@ -65,22 +65,16 @@ Stream closed successfully **Offset-based API (Recommended):** ```typescript -// 1. Auto-serializing: pass object directly -const record = { device_name: 'sensor-001', temp: 22, humidity: 65 }; -const objectOffset = await stream.ingestRecordOffset(record); -await stream.waitForOffset(objectOffset); - -// 2. Pre-serialized: pass JSON string -const jsonString = JSON.stringify({ device_name: 'sensor-002', temp: 24, humidity: 70 }); -const stringOffset = await stream.ingestRecordOffset(jsonString); -await stream.waitForOffset(stringOffset); - -// 3. High-throughput: send many, wait once -let lastOffset: bigint; +// Queue records, then wait once. Immediate waitForOffset after a single ingest +// is valid for low-volume strict confirmation, not the default bulk pattern. +const records = [ + { device_name: 'sensor-001', temp: 22, humidity: 65 }, + { device_name: 'sensor-002', temp: 24, humidity: 70 } +]; for (const record of records) { - lastOffset = await stream.ingestRecordOffset(record); + await stream.ingestRecordOffset(record); } -await stream.waitForOffset(lastOffset); +await stream.flush(); ``` **Future-based API (Deprecated):** diff --git a/typescript/examples/json/batch.ts b/typescript/examples/json/batch.ts index 88a6aa03..138a32f7 100644 --- a/typescript/examples/json/batch.ts +++ b/typescript/examples/json/batch.ts @@ -100,9 +100,7 @@ async function main() { const offset1 = await stream.ingestRecordsOffset(batch1); if (offset1 !== null) { - console.log(`[Auto-serializing] Batch of 3 records sent with offset ID: ${offset1}`); - await stream.waitForOffset(offset1); - console.log(`[Auto-serializing] Batch acknowledged with offset ID: ${offset1}`); + console.log(`[Auto-serializing] Batch of 3 records queued with offset ID: ${offset1}`); } // 2. Pre-serialized: array of JSON strings @@ -114,13 +112,11 @@ async function main() { const offset2 = await stream.ingestRecordsOffset(batch2); if (offset2 !== null) { - console.log(`[Pre-serialized] Batch of 3 records sent with offset ID: ${offset2}`); - await stream.waitForOffset(offset2); - console.log(`[Pre-serialized] Batch acknowledged with offset ID: ${offset2}`); + console.log(`[Pre-serialized] Batch of 3 records queued with offset ID: ${offset2}`); } // 3. Large batch example - console.log('\n[Large batch] Sending batch of 100 records...'); + console.log('\n[Large batch] Queueing batch of 100 records...'); const largeBatch: AirQuality[] = Array.from({ length: 100 }, (_, i) => ({ device_name: `sensor-${i.toString().padStart(3, '0')}`, temp: 20 + (i % 15), @@ -129,10 +125,12 @@ async function main() { const offset3 = await stream.ingestRecordsOffset(largeBatch); if (offset3 !== null) { - await stream.waitForOffset(offset3); - console.log(`[Large batch] 100 records acknowledged with offset ID: ${offset3}`); + console.log(`[Large batch] 100 records queued with offset ID: ${offset3}`); } + await stream.flush(); + console.log('All offset-API batches acknowledged'); + // 4. Empty batch returns null const emptyOffset = await stream.ingestRecordsOffset([]); console.log(`[Empty batch] Returns: ${emptyOffset}`); diff --git a/typescript/examples/proto/README.md b/typescript/examples/proto/README.md index 17318c4a..6bb039db 100644 --- a/typescript/examples/proto/README.md +++ b/typescript/examples/proto/README.md @@ -106,19 +106,14 @@ const tableProperties: TableProperties = { ```typescript const AirQuality = airQuality.examples.AirQuality; -// 1. Auto-encoding: pass message directly -const record = AirQuality.create({ - deviceName: 'sensor-001', - temp: 22, - humidity: 65 -}); -const messageOffset = await stream.ingestRecordOffset(record); -await stream.waitForOffset(messageOffset); - -// 2. Pre-encoded: pass Buffer -const buffer = Buffer.from(AirQuality.encode(record).finish()); -const bufferOffset = await stream.ingestRecordOffset(buffer); -await stream.waitForOffset(bufferOffset); +const records = [ + AirQuality.create({ deviceName: 'sensor-001', temp: 22, humidity: 65 }), + AirQuality.create({ deviceName: 'sensor-002', temp: 24, humidity: 70 }) +]; +for (const record of records) { + await stream.ingestRecordOffset(record); +} +await stream.flush(); ``` ## Batch Example @@ -149,13 +144,8 @@ const messageBatch = [ AirQuality.create({ deviceName: 'sensor-003', temp: 24, humidity: 69 }) ]; const messageBatchOffset = await stream.ingestRecordsOffset(messageBatch); -if (messageBatchOffset !== null) { - await stream.waitForOffset(messageBatchOffset); -} - -// 2. Pre-encoded: array of Buffers -const bufferBatch = records.map(r => Buffer.from(AirQuality.encode(r).finish())); const bufferBatchOffset = await stream.ingestRecordsOffset(bufferBatch); +await stream.flush(); ``` ## Adapting for Your Custom Table diff --git a/typescript/examples/proto/batch.ts b/typescript/examples/proto/batch.ts index 97dc7ee9..f621cfe9 100644 --- a/typescript/examples/proto/batch.ts +++ b/typescript/examples/proto/batch.ts @@ -112,9 +112,7 @@ async function main() { const offset1 = await stream.ingestRecordsOffset(batch1); if (offset1 !== null) { - console.log(`[Auto-encoding] Batch of 3 records sent with offset ID: ${offset1}`); - await stream.waitForOffset(offset1); - console.log(`[Auto-encoding] Batch acknowledged with offset ID: ${offset1}`); + console.log(`[Auto-encoding] Batch of 3 records queued with offset ID: ${offset1}`); } // 2. Pre-encoded: array of Buffers @@ -126,13 +124,11 @@ async function main() { const offset2 = await stream.ingestRecordsOffset(batch2); if (offset2 !== null) { - console.log(`[Pre-encoded] Batch of 3 records sent with offset ID: ${offset2}`); - await stream.waitForOffset(offset2); - console.log(`[Pre-encoded] Batch acknowledged with offset ID: ${offset2}`); + console.log(`[Pre-encoded] Batch of 3 records queued with offset ID: ${offset2}`); } // 3. Large batch example - console.log('\n[Large batch] Sending batch of 100 records...'); + console.log('\n[Large batch] Queueing batch of 100 records...'); const largeBatch = Array.from({ length: 100 }, (_, i) => AirQuality.create({ deviceName: `sensor-${i.toString().padStart(3, '0')}`, @@ -143,10 +139,12 @@ async function main() { const offset3 = await stream.ingestRecordsOffset(largeBatch); if (offset3 !== null) { - await stream.waitForOffset(offset3); - console.log(`[Large batch] 100 records acknowledged with offset ID: ${offset3}`); + console.log(`[Large batch] 100 records queued with offset ID: ${offset3}`); } + await stream.flush(); + console.log('All offset-API batches acknowledged'); + // 4. Empty batch returns null const emptyOffset = await stream.ingestRecordsOffset([]); console.log(`[Empty batch] Returns: ${emptyOffset}`); diff --git a/typescript/src/headers_provider.ts b/typescript/src/headers_provider.ts index 678b930a..105afdfb 100644 --- a/typescript/src/headers_provider.ts +++ b/typescript/src/headers_provider.ts @@ -1,54 +1,46 @@ /** - * Interface for providing custom headers to Zerobus streams. + * Custom headers provider accepted by `createStream()`. * - * Implement this interface to use custom authentication beyond OAuth, - * such as Personal Access Tokens (PAT) or custom auth tokens. + * The native adapter invokes `getHeadersCallback` synchronously once during + * stream creation and stores the returned tuples. Returning a Promise, or + * passing a class with async `getHeaders()`, is not supported and can terminate + * the process. Token refresh is not currently wired through this callback. */ export interface HeadersProvider { /** - * Returns headers as array of [name, value] tuples. + * Returns headers as an array of [name, value] tuples. * * Required headers: * - ["authorization", "Bearer "] * - ["x-databricks-zerobus-table-name", ""] - * - * @returns Promise resolving to array of header name-value pairs */ - getHeaders(): Promise>; + getHeadersCallback: () => Array<[string, string]>; } /** * OAuth 2.0 Client Credentials headers provider. * - * **IMPORTANT: DO NOT instantiate this class directly.** + * Do not instantiate this class or pass it to `createStream()`. * * OAuth authentication is handled automatically by the Rust SDK when you call - * `createStream()` with clientId and clientSecret parameters (without providing - * a headers_provider). - * - * This class exists for: - * 1. Documentation purposes - showing the HeadersProvider pattern - * 2. API consistency with other Zerobus SDKs (Python, Java, Rust) + * `createStream()` with clientId and clientSecret and omit the headers provider. * - * **How to use OAuth authentication:** + * How to use OAuth authentication: * ```typescript - * // OAuth is the default - just pass clientId and clientSecret * const stream = await sdk.createStream( * tableProperties, - * clientId, // OAuth client ID - * clientSecret, // OAuth client secret + * clientId, + * clientSecret, * options - * // No headers_provider parameter = OAuth authentication * ); * ``` * - * The native `createStream` adapter currently requires its callback to return - * header tuples synchronously. Pass a callback object directly: + * How to use custom authentication (PAT or a static token): * ```typescript * const stream = await sdk.createStream( * tableProperties, - * '', // ignored - * '', // ignored + * '', + * '', * options, * { * getHeadersCallback: () => [ @@ -59,7 +51,7 @@ export interface HeadersProvider { * ); * ``` */ -export class OAuthHeadersProvider implements HeadersProvider { +export class OAuthHeadersProvider { constructor( private clientId: string, private clientSecret: string, @@ -71,8 +63,8 @@ export class OAuthHeadersProvider implements HeadersProvider { throw new Error( 'OAuthHeadersProvider should not be instantiated directly. ' + 'OAuth authentication is handled internally by the Rust SDK. ' + - 'To use OAuth: call createStream(tableProperties, clientId, clientSecret, options) without the headers_provider parameter. ' + - 'To use custom authentication: implement the HeadersProvider interface.' + 'To use OAuth: call createStream(tableProperties, clientId, clientSecret, options) without a headers provider. ' + + 'To use custom authentication: pass { getHeadersCallback: () => [...] } as the headers provider.' ); } } diff --git a/typescript/src/lib.rs b/typescript/src/lib.rs index 0abb81ed..ebc5f4a4 100644 --- a/typescript/src/lib.rs +++ b/typescript/src/lib.rs @@ -103,7 +103,8 @@ pub struct TableProperties { pub table_name: String, /// Optional Protocol Buffer descriptor as a base64-encoded string. - /// If not provided, JSON encoding will be used. + /// Omitting this does not select JSON. The stream defaults to Protocol Buffers + /// unless `record_type` is set to JSON. pub descriptor_proto: Option, } @@ -1135,13 +1136,13 @@ impl ZerobusSdk { /// /// ```typescript /// try { - /// await stream.ingestRecords(batch); + /// await stream.ingestRecordsOffset(batch); + /// await stream.flush(); /// } catch (error) { - /// // Recreate stream with all unacked batches re-ingested /// try { /// const newStream = await sdk.recreateStream(stream); /// try { - /// // Continue ingesting with newStream + /// await newStream.flush(); /// } finally { /// await newStream.close(); /// } diff --git a/typescript/test/unit.test.ts b/typescript/test/unit.test.ts index 20c0dee0..af557494 100644 --- a/typescript/test/unit.test.ts +++ b/typescript/test/unit.test.ts @@ -118,33 +118,27 @@ describe('ZerobusSdk', () => { describe('HeadersProvider', () => { it('should accept custom headers provider implementation', () => { - class TestHeadersProvider implements HeadersProvider { - async getHeaders(): Promise> { - return [ - ['authorization', 'Bearer test-token'], - ['x-databricks-zerobus-table-name', 'catalog.schema.table'], - ]; - } - } + const provider: HeadersProvider = { + getHeadersCallback: () => [ + ['authorization', 'Bearer test-token'], + ['x-databricks-zerobus-table-name', 'catalog.schema.table'], + ], + }; - const provider = new TestHeadersProvider(); assert.ok(provider); - assert.ok(typeof provider.getHeaders === 'function'); + assert.ok(typeof provider.getHeadersCallback === 'function'); }); - it('should return correct header format', async () => { - class TestHeadersProvider implements HeadersProvider { - async getHeaders(): Promise> { - return [ - ['authorization', 'Bearer test-token'], - ['x-databricks-zerobus-table-name', 'test-table'], - ['x-custom-header', 'custom-value'], - ]; - } - } + it('should return correct header format', () => { + const provider: HeadersProvider = { + getHeadersCallback: () => [ + ['authorization', 'Bearer test-token'], + ['x-databricks-zerobus-table-name', 'test-table'], + ['x-custom-header', 'custom-value'], + ], + }; - const provider = new TestHeadersProvider(); - const headers = await provider.getHeaders(); + const headers = provider.getHeadersCallback(); assert.strictEqual(headers.length, 3); assert.deepStrictEqual(headers[0], ['authorization', 'Bearer test-token']); From e027f5ecc19629448bf920a0e6df0bc11d7e2fd6 Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Thu, 13 Aug 2026 14:24:00 +0000 Subject: [PATCH 04/36] Fix Go documentation --- go/CONTRIBUTING.md | 2 +- go/NEXT_CHANGELOG.md | 7 +++ go/README.md | 72 ++++++++++++++--------------- go/examples/README.md | 18 ++++++++ go/version.go | 2 +- go/zerobus.go | 23 ++++----- purego/NEXT_CHANGELOG.md | 3 ++ purego/examples/json/batch/main.go | 20 ++++---- purego/examples/json/single/main.go | 31 ++++++++----- 9 files changed, 107 insertions(+), 71 deletions(-) diff --git a/go/CONTRIBUTING.md b/go/CONTRIBUTING.md index dfb3a357..ac5554bd 100644 --- a/go/CONTRIBUTING.md +++ b/go/CONTRIBUTING.md @@ -88,7 +88,7 @@ Available make targets: - `make build` - Build both Rust FFI and Go SDK - `make build-rust` - Build only the Rust FFI layer -- `make build-go` - Build only the Go SDK +- `make build-go` - Build the Go SDK. This target depends on `build-rust`. - `make clean` - Remove build artifacts - `make fmt` - Format all code (Go and Rust) - `make lint` - Run linters on all code diff --git a/go/NEXT_CHANGELOG.md b/go/NEXT_CHANGELOG.md index 3b2346f3..8f81c902 100644 --- a/go/NEXT_CHANGELOG.md +++ b/go/NEXT_CHANGELOG.md @@ -10,6 +10,13 @@ ### Documentation +- Corrected consumer installation so tagged releases do not require Rust or + `go generate`. Documented that `make build-go` depends on the Rust FFI build. +- Documented that `GetUnackedRecords()` must run before `Close()`, that + `RecordAck.Await()` waits for server durability, and that one stream can be + used from multiple goroutines. +- Added `Flush()` to copyable example snippets. + ### Internal Changes ### API Changes diff --git a/go/README.md b/go/README.md index 49541f62..32e8c72d 100644 --- a/go/README.md +++ b/go/README.md @@ -93,7 +93,7 @@ Before using the SDK, you need a Databricks workspace URL, a Delta table, and a go get github.com/databricks/zerobus-sdk/go@latest ``` -> **Note:** Tagged releases (e.g., `v1.0.0`) come with pre-built Rust libraries for Linux, macOS, and Windows. If you use `@main` or a commit hash, you will need to have Rust installed and run `go generate` to build the library yourself. +> Tagged releases (for example `v1.4.0`) include pre-built Rust libraries for Linux, macOS, and Windows. Consumers do not need Rust or `go generate`. Rust is required only when you build from `@main`, a commit hash, or a local checkout. **In your code:** @@ -113,7 +113,7 @@ func main() { } ``` -> **Note:** After the initial `go generate` step, regular `go build` works normally. The Rust library is statically linked into your binary. +> After `go get` of a tagged release, `go build` works normally. The pre-built Rust library is statically linked into your binary. ### Development Setup @@ -142,7 +142,7 @@ The SDK supports two serialization formats and two ingestion methods: - **Protocol Buffers** (Recommended for production): Type-safe approach with schema validation at compile time **Ingestion Methods:** -- **Single-record** (`IngestRecordOffset`): Ingest records one at a time with per-record acknowledgment +- **Single-record** (`IngestRecordOffset`): Queue records one at a time. The returned offset is an admission handle, not a durability confirmation. Call `Flush()` once to wait for server acknowledgments. - **Batch** (`IngestRecordsOffset`): Ingest multiple records at once with all-or-nothing semantics for higher throughput See [`examples/README.md`](examples/README.md) for detailed setup instructions and examples for all combinations. @@ -593,10 +593,12 @@ for i := 0; i < 100; i++ { wg.Wait() close(errCh) -// Check for errors for err := range errCh { log.Printf("Ingestion error: %v", err) } +if err := stream.Flush(); err != nil { + log.Fatal(err) +} ``` **Concurrent ingestion with multiple streams:** @@ -710,17 +712,19 @@ if err := stream.Close(); err != nil { } ``` -If the stream fails, retrieve unacknowledged records: +If Flush or ingest fails, inspect unacked records before Close(). Close() nils the stream handle and frees native resources, so GetUnackedRecords() cannot be called afterward. ```go -if err := stream.Close(); err != nil { - // Stream failed, get unacked records - unacked, err := stream.GetUnackedRecords() - if err != nil { - log.Fatal(err) +if err := stream.Flush(); err != nil { + unacked, unackedErr := stream.GetUnackedRecords() + if unackedErr != nil { + log.Printf("could not inspect unacked records: %v", unackedErr) + } else { + log.Printf("Failed to ack %d records", len(unacked)) } - log.Printf("Failed to ack %d records", len(unacked)) - // Retry with a new stream +} +if err := stream.Close(); err != nil { + log.Printf("close: %v", err) } ``` @@ -886,12 +890,12 @@ stream, err := sdk.CreateArrowStreamWithHeadersProvider( ### Unacked Batches -If the stream fails, retrieve unacknowledged batches to retry on a new stream: +If Flush fails, inspect unacked batches before Close(). Close() frees the handle. ```go -if err := stream.Close(); err != nil { +if err := stream.Flush(); err != nil { unacked, _ := stream.GetUnackedBatches() - // re-ingest unacked on a new stream + // re-ingest unacked on a new stream, then Close() the failed stream } ``` @@ -959,7 +963,7 @@ The test suite includes: 9. **Use Protocol Buffers for Production** - More efficient than JSON for high-volume scenarios 10. **Secure Credentials** - Never hardcode secrets; use environment variables or secret managers 11. **Test Recovery** - Simulate failures to verify your error handling logic -12. **One Stream Per Goroutine** - Don't share streams across goroutines; create separate streams for concurrent ingestion +12. **Concurrent ingestion** - One stream can be used from multiple goroutines. Create separate streams when you want independent tables, credentials, or failure isolation. ## Migration Guide @@ -1223,7 +1227,7 @@ func (st *ZerobusStream) GetUnackedRecords() ([]interface{}, error) Returns a snapshot of all records that have been sent to the server but not yet confirmed as durably written. -**IMPORTANT:** This method should **only be called after the stream has closed or failed**. Calling it on an active stream will return an error. +**IMPORTANT:** Call this on a failed stream before `Close()`. `Close()` nils the handle and frees native resources, so a later `GetUnackedRecords()` call fails with "Stream has been closed". An active stream also returns an error. **What you get:** - A copy of all pending records still waiting for server acknowledgment @@ -1231,28 +1235,24 @@ Returns a snapshot of all records that have been sent to the server but not yet - Each element is either `string` (JSON) or `[]byte` (protobuf) **When to use:** -- After stream failure to retrieve unacknowledged records for retry -- After `Close()` fails to see which records weren't durably written -- For implementing custom retry logic after stream errors +- After a terminal Flush or ingest failure, before `Close()` +- Not after `Close()`, which releases the stream handle **Note:** This creates a memory snapshot of pending data. For large numbers of unacked records, this can temporarily increase memory usage. **Example:** ```go -// Try to close the stream -if err := stream.Close(); err != nil { - // Stream failed to close, check for unacked records +if err := stream.Flush(); err != nil { unacked, err := stream.GetUnackedRecords() if err != nil { - log.Fatal(err) - } - log.Printf("%d records failed to be acknowledged", len(unacked)) - - // Retry with a new stream - for _, record := range unacked { - newStream.IngestRecordOffset(record) + log.Printf("could not inspect unacked records: %v", err) + } else { + log.Printf("%d records failed to be acknowledged", len(unacked)) } } +if closeErr := stream.Close(); closeErr != nil { + log.Printf("close: %v", closeErr) +} ``` ```go @@ -1269,7 +1269,7 @@ ack, err := stream.IngestRecord(`{"id": 1}`) if err != nil { log.Fatal(err) } -offset, err := ack.Await() // Returns immediately with cached offset +offset, err := ack.Await() // Blocks until the server acknowledges ``` ```go @@ -1322,15 +1322,15 @@ Represents an acknowledgment for an ingested record. The offset is available imm func (ack *RecordAck) Await() (int64, error) ``` -Returns the offset for the ingested record. Returns immediately since the offset is already available. +Returns the offset after waiting for server durability via `WaitForOffset()`. This is not immediate. Use `Offset()` when you only need the queued offset without waiting. **Example:** ```go -ack, _ := stream.IngestRecord(data) // Deprecated -offset, err := ack.Await() +ack, err := stream.IngestRecord(data) // Deprecated if err != nil { - log.Printf("Record failed: %v", err) + log.Fatal(err) } +offset, err := ack.Await() // Blocks until the server acknowledges ``` **Prefer the new API:** @@ -1483,7 +1483,7 @@ make build # Build only Rust FFI make build-rust -# Build only Go SDK +# Build the Go package. This target also builds the Rust FFI first. make build-go # Build examples diff --git a/go/examples/README.md b/go/examples/README.md index 92990c75..c066a85f 100644 --- a/go/examples/README.md +++ b/go/examples/README.md @@ -111,6 +111,12 @@ go run main.go ```go jsonRecord := `{"device_name": "sensor-001", "temp": 20, "humidity": 60}` offset, err := stream.IngestRecordOffset(jsonRecord) +if err != nil { + log.Fatal(err) +} +if err := stream.Flush(); err != nil { + log.Fatal(err) +} ``` **Protocol Buffers:** @@ -135,6 +141,12 @@ records := []interface{}{ `{"device_name": "sensor-002", "temp": 21, "humidity": 61}`, } batchOffset, err := stream.IngestRecordsOffset(records) +if err != nil { + log.Fatal(err) +} +if err := stream.Flush(); err != nil { + log.Fatal(err) +} ``` **Protocol Buffers:** @@ -147,6 +159,12 @@ for i := 0; i < 5; i++ { records = append(records, data) } batchOffset, err := stream.IngestRecordsOffset(records) +if err != nil { + log.Fatal(err) +} +if err := stream.Flush(); err != nil { + log.Fatal(err) +} ``` ### Fire-and-Forget Ingestion diff --git a/go/version.go b/go/version.go index 6e884ebc..e1d3a9da 100644 --- a/go/version.go +++ b/go/version.go @@ -1,7 +1,7 @@ package zerobus // sdkVersion must match the version in the next go/vX.Y.Z release tag. -const sdkVersion = "1.3.0" +const sdkVersion = "1.4.0" const sdkIdentifierPrefix = "zerobus-sdk-go" diff --git a/go/zerobus.go b/go/zerobus.go index c2f5e5b2..13a3be24 100644 --- a/go/zerobus.go +++ b/go/zerobus.go @@ -6,12 +6,13 @@ // // # Installation // -// This package requires a one-time build step to compile the Rust FFI layer: +// This package is a CGO wrapper around a Rust core. Tagged releases include +// pre-built libraries, so consumers can install with: // -// go get github.com/databricks/zerobus-sdk/go -// go generate github.com/databricks/zerobus-sdk/go +// go get github.com/databricks/zerobus-sdk/go@v1.4.0 // -// Prerequisites: Go 1.19+, Rust 1.70+, CGO enabled +// Prerequisites for consumers: Go 1.21+, CGO enabled, a C compiler. +// Rust and `go generate` are required only when building from source. // // # Quick Start // @@ -721,8 +722,8 @@ func (st *ZerobusStream) WaitForOffset(offset int64) error { // GetUnackedRecords retrieves all records that have not yet been acknowledged by the server. // -// IMPORTANT: This method should only be called AFTER the stream has closed or failed. -// Calling it on an active stream will return an error. +// IMPORTANT: Call this on a failed stream before Close(). Close() nils the +// handle and frees native resources, so a later GetUnackedRecords() call fails. // // Use this method to: // - Retrieve unacknowledged records after stream failure for retry logic @@ -737,15 +738,15 @@ func (st *ZerobusStream) WaitForOffset(offset int64) error { // // Example: // -// if err := stream.Close(); err != nil { -// // Stream failed, check for unacked records +// if err := stream.Flush(); err != nil { // unacked, err := stream.GetUnackedRecords() // if err != nil { -// log.Fatal(err) +// log.Printf("could not inspect unacked records: %v", err) +// } else { +// log.Printf("Failed to acknowledge %d records", len(unacked)) // } -// log.Printf("Failed to acknowledge %d records", len(unacked)) -// // Retry with a new stream // } +// _ = stream.Close() func (st *ZerobusStream) GetUnackedRecords() ([]interface{}, error) { if st.ptr == nil { return nil, &ZerobusError{Message: "Stream has been closed", IsRetryable: false} diff --git a/purego/NEXT_CHANGELOG.md b/purego/NEXT_CHANGELOG.md index f20760c3..aa3f69ae 100644 --- a/purego/NEXT_CHANGELOG.md +++ b/purego/NEXT_CHANGELOG.md @@ -8,6 +8,9 @@ ### Documentation +- Flush recovery no longer treats every flush error as terminal. Batch examples + expect one callback per batch and wait for that callback before exit. + ### Internal Changes ### Breaking Changes diff --git a/purego/examples/json/batch/main.go b/purego/examples/json/batch/main.go index 04df07b3..5adf4049 100644 --- a/purego/examples/json/batch/main.go +++ b/purego/examples/json/batch/main.go @@ -20,6 +20,7 @@ import ( "context" "log" "sync/atomic" + "time" "github.com/databricks/zerobus-sdk/purego/examples/config" "github.com/databricks/zerobus-sdk/purego/examples/internal/exampleutil" @@ -71,20 +72,19 @@ func main() { } log.Printf("Batch of %d records queued; batch offset ID: %d", len(batch), batchOffset) - // Confirm the batch. - if batchOffset >= 0 { - if err := stream.WaitForOffset(batchOffset); err != nil { - log.Fatalf("wait for offset %d: %v", batchOffset, err) - } - log.Printf("Batch acknowledged at offset ID: %d", batchOffset) - } - - // Flush pending records, then close. if err := stream.Flush(); err != nil { log.Fatalf("flush: %v", err) } + + // A batch produces one callback event, not one per record. Callback delivery + // can still be running when Close() returns, so wait for it before exit. + deadline := time.Now().Add(5 * time.Second) + for obs.acked.Load() < 1 && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if err := stream.Close(); err != nil { log.Fatalf("close: %v", err) } - log.Printf("Stream closed successfully. Callback observed %d acknowledgements.", obs.acked.Load()) + log.Printf("Stream closed. Callback observed %d acknowledgements (expected 1 for the batch).", obs.acked.Load()) } diff --git a/purego/examples/json/single/main.go b/purego/examples/json/single/main.go index 3013021a..7083a660 100644 --- a/purego/examples/json/single/main.go +++ b/purego/examples/json/single/main.go @@ -63,10 +63,23 @@ func main() { log.Printf("Record %d queued with offset ID: %d", i+1, offset) } - // 4. Flush once, then close. On failure, recover unacked records. + // 4. Flush once, then close. A flush timeout can leave the stream active, so + // only recover when unacked retrieval succeeds. if err := stream.Flush(); err != nil { - log.Printf("stream failed: %v", err) - recoverUnacked(sdk, cfg, stream) + log.Printf("flush failed: %v", err) + unacked, unackedErr := stream.GetUnackedRecords() + if unackedErr != nil { + log.Printf("stream still active or unacked retrieval failed: %v", unackedErr) + if closeErr := stream.Close(); closeErr != nil { + log.Printf("close: %v", closeErr) + } + return + } + if len(unacked) == 0 { + _ = stream.Close() + return + } + recoverUnacked(sdk, cfg, stream, unacked) return } if err := stream.Close(); err != nil { @@ -75,16 +88,10 @@ func main() { log.Println("All records acknowledged. Stream closed successfully.") } -// recoverUnacked re-ingests records from a failed stream on a new stream. -func recoverUnacked(sdk *zerobus.SDK, cfg config.Settings, failed *zerobus.Stream) { - unacked, err := failed.GetUnackedRecords() - if err != nil { - log.Fatalf("get unacked records: %v", err) - } +// recoverUnacked re-ingests previously retrieved records on a new stream. +func recoverUnacked(sdk *zerobus.SDK, cfg config.Settings, failed *zerobus.Stream, unacked [][]byte) { + defer failed.Close() log.Printf("Recovering %d unacknowledged records on a fresh stream.", len(unacked)) - if len(unacked) == 0 { - return - } retry, err := openStream(sdk, cfg) if err != nil { log.Fatalf("reopen stream: %v", err) From bdc21c20edb25311338091afe7b933b00a723422 Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Thu, 13 Aug 2026 14:24:18 +0000 Subject: [PATCH 05/36] Fix Java documentation --- java/CONTRIBUTING.md | 2 +- java/NEXT_CHANGELOG.md | 5 +++++ java/README.md | 37 +++++++++++++++++++++++-------------- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/java/CONTRIBUTING.md b/java/CONTRIBUTING.md index 10446e87..3bf0acef 100644 --- a/java/CONTRIBUTING.md +++ b/java/CONTRIBUTING.md @@ -9,7 +9,7 @@ This document covers Java-specific development setup and workflow. ### Prerequisites - Git -- Java 8 or higher - [Download Java](https://adoptium.net/) +- Java 11 or higher to build and test from source. Published JARs remain compatible with Java 8. - Maven 3.6 or higher - [Download Maven](https://maven.apache.org/download.cgi) - Protocol Buffers compiler (`protoc`) 33.0 - [Download protoc](https://github.com/protocolbuffers/protobuf/releases/tag/v33.0) diff --git a/java/NEXT_CHANGELOG.md b/java/NEXT_CHANGELOG.md index 15dc1f58..4fed9586 100644 --- a/java/NEXT_CHANGELOG.md +++ b/java/NEXT_CHANGELOG.md @@ -15,6 +15,11 @@ durability barrier after queued ingestion. Clarified that acknowledgment callbacks fire once per logical ingest submission, including one callback per batch ingest call. +- Documented that published JARs support Java 8 while source builds need JDK 11, + that macOS JNI artifacts are not in the current release set, that + `recoveryRetries` defaults to 4, and that `flush()` waits for durability rather + than callback completion. `recreateStream()` is no longer presented as a safe + production recovery path. ### Internal Changes diff --git a/java/README.md b/java/README.md index 735d054a..3f2d1665 100644 --- a/java/README.md +++ b/java/README.md @@ -68,7 +68,7 @@ The Java SDK uses JNI (Java Native Interface) to call a high-performance Rust im ### Runtime Requirements -- **Java**: 8 or higher - [Download Java](https://adoptium.net/) +- **Java**: 8 or higher to run a published JAR. Building and testing from source requires JDK 11 or higher. - **Databricks workspace** with Zerobus access enabled ### Supported Platforms @@ -82,8 +82,8 @@ This SDK includes native libraries for the following platforms: | Linux (musl / Alpine) | x86_64 | Supported | | Linux (musl / Alpine) | aarch64 | Supported | | Windows | x86_64 | Supported | -| macOS | x86_64 | Supported | -| macOS | aarch64 (Apple Silicon) | Supported | +| macOS | x86_64 | Source-build only; the published JNI artifacts currently include Linux and Windows | +| macOS | aarch64 (Apple Silicon) | Source-build only; the published JNI artifacts currently include Linux and Windows | Linux glibc builds support glibc 2.26 and newer, including Amazon Linux 2. On Linux, the libc flavor (glibc vs musl) is detected at runtime. To override detection, set @@ -119,7 +119,7 @@ On Linux, the libc flavor (glibc vs musl) is detected at runtime. To override de ### Build Requirements (only for building from source) -- **Java**: 8 or higher - [Download Java](https://adoptium.net/) +- **Java**: 8 or higher to run a published JAR. Building and testing from source requires JDK 11 or higher. - **Maven**: 3.6 or higher - [Download Maven](https://maven.apache.org/download.cgi) - **Protocol Buffers Compiler** (`protoc`): 33.0 - [Download protoc](https://github.com/protocolbuffers/protobuf/releases/tag/v33.0) (for compiling your own `.proto` schemas) @@ -793,16 +793,24 @@ try { **Migration:** ```java -// Before (deprecated ZerobusStream): -stream.ingestRecord(record).join(); +// Before (deprecated ZerobusStream): wait once after the loop +CompletableFuture last = null; +for (AirQuality record : records) { + last = stream.ingestRecord(record); +} +if (last != null) { + last.join(); +} -// After (recommended ZerobusProtoStream): -long offset = stream.ingestRecordOffset(record); -stream.waitForOffset(offset); +// After (recommended ZerobusProtoStream): queue, then flush once +for (AirQuality record : records) { + stream.ingestRecordOffset(record); +} +stream.flush(); // Batch ingestion: -Optional batchOffset = stream.ingestRecordsOffset(batch); -batchOffset.ifPresent(o -> { try { stream.waitForOffset(o); } catch (Exception e) { throw new RuntimeException(e); } }); +stream.ingestRecordsOffset(batch); +stream.flush(); ``` --- @@ -877,7 +885,7 @@ ZerobusJsonStream stream = sdk.streamBuilder() | `recovery` | true | Enable automatic stream recovery | | `recoveryTimeoutMs` | 15000 | Timeout for recovery operations (ms) | | `recoveryBackoffMs` | 2000 | Delay between recovery attempts (ms) | -| `recoveryRetries` | 3 | Maximum number of recovery attempts | +| `recoveryRetries` | 4 | Maximum number of recovery attempts | | `flushTimeoutMs` | 300000 | Timeout for flush operations (ms) | | `serverLackOfAckTimeoutMs` | 60000 | Server acknowledgment timeout (ms) | | `ackCallback` | None | Callback invoked on record acknowledgment | @@ -1595,7 +1603,7 @@ ZerobusProtoStream stream = sdk.streamBuilder() for (AirQuality record : records) { stream.ingestRecordOffset(record); } -stream.flush(); // drain remaining acks before close +stream.flush(); // wait for durability; callbacks may still be running until close() ``` Implementations must be thread-safe and lightweight (callbacks run on internal @@ -1616,7 +1624,8 @@ processing threads). - `ingestRecordOffset()` + final `flush()` / `waitForOffset(lastOffset)` → High throughput (recommended) - `ingestRecordOffset()` + `waitForOffset()` per record → When a specific record must be confirmed before continuing - `ingestRecord().join()` → Deprecated; prefer the offset-based API -11. **Recovery pattern**: Use `sdk.recreateStream(closedStream)` to automatically re-ingest unacknowledged records, or manually use `getUnackedBatches()` after stream close +11. **Thread safety**: `ZerobusSdk` and streams are not thread-safe. Synchronize externally if more than one thread uses the same instance. +12. **Recovery**: `recreateStream()` currently requires a closed stream, and a failed `close()` can drop unacked payloads before they are cached. Do not rely on it for production recovery until that is fixed. Prefer inspecting `getUnackedBatches()` only after a successful close. ## Community and Contributing From c42594149cb9a104619e0b28b1c781355ca67b8d Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Thu, 13 Aug 2026 14:24:43 +0000 Subject: [PATCH 06/36] Fix remaining SDK docs --- cpp/NEXT_CHANGELOG.md | 2 ++ cpp/examples/json/single.cpp | 10 +++++++-- dotnet/CONTRIBUTING.md | 2 +- dotnet/NEXT_CHANGELOG.md | 3 +++ dotnet/examples/JsonSingle/Program.cs | 20 +++++++++++++++--- dotnet/src/Zerobus/ZerobusStream.cs | 18 +++++++++++++---- rust/NEXT_CHANGELOG.md | 3 +++ rust/examples/json/batch.rs | 23 +++++---------------- rust/examples/proto/compiled/batch.rs | 23 +++++---------------- rust/ffi/NEXT_CHANGELOG.md | 2 ++ rust/ffi/README.md | 29 +++++++++++++++++++++------ rust/sdk/src/sdk.rs | 6 ++---- rust/sdk/src/stream/grpc/acks.rs | 11 ++++------ rust/sdk/src/stream/grpc/ingest.rs | 10 ++++----- 14 files changed, 93 insertions(+), 69 deletions(-) diff --git a/cpp/NEXT_CHANGELOG.md b/cpp/NEXT_CHANGELOG.md index 4a6df1e1..a1051e39 100644 --- a/cpp/NEXT_CHANGELOG.md +++ b/cpp/NEXT_CHANGELOG.md @@ -20,6 +20,8 @@ - Corrected custom-header examples and clarified that acknowledgment callbacks run once per logical ingest submission rather than once per record in a batch. +- Recovery after a flush timeout now treats unacked retrieval failure as an + active stream rather than assuming the stream is terminal. ### Internal Changes diff --git a/cpp/examples/json/single.cpp b/cpp/examples/json/single.cpp index 6362e08e..4e0ffa5c 100644 --- a/cpp/examples/json/single.cpp +++ b/cpp/examples/json/single.cpp @@ -141,8 +141,14 @@ int main() { } catch (const zerobus::ZerobusException& e) { std::cerr << "Stream failed: " << e.what() << "\n"; - std::vector unacked = - stream.get_unacked_records(); + std::vector unacked; + try { + unacked = stream.get_unacked_records(); + } catch (const zerobus::ZerobusException& retrieval) { + std::cerr << "Could not inspect unacked records (stream may still be active): " + << retrieval.what() << "\n"; + return 1; + } std::cout << "Recovering " << unacked.size() << " unacknowledged records on a fresh stream.\n"; diff --git a/dotnet/CONTRIBUTING.md b/dotnet/CONTRIBUTING.md index cee926e7..c9b19981 100644 --- a/dotnet/CONTRIBUTING.md +++ b/dotnet/CONTRIBUTING.md @@ -98,7 +98,7 @@ When making FFI-related changes: 1. Update Rust code in `../rust/ffi/src/` 2. Update exported C API in `../rust/ffi/zerobus.h` if needed -3. Update .NET interop bindings in `src/Zerobus/Interop/` +3. Update .NET interop bindings in `src/Zerobus/Native/` 4. Rebuild native artifacts: ```bash ./build_native.sh diff --git a/dotnet/NEXT_CHANGELOG.md b/dotnet/NEXT_CHANGELOG.md index b645defd..e3f2216f 100644 --- a/dotnet/NEXT_CHANGELOG.md +++ b/dotnet/NEXT_CHANGELOG.md @@ -24,6 +24,9 @@ - Corrected installation and source-build prerequisites, separated JSON and Protobuf stream examples, added a runnable generated-message example, and replaced per-record waits with one final flush in bulk-ingestion examples. +- Documented that `GetUnackedRecords()` can fail while the stream is still + active after a flush timeout, and stopped reporting success after ingest + failures. Pointed CONTRIBUTING at `src/Zerobus/Native/`. ### Internal Changes diff --git a/dotnet/examples/JsonSingle/Program.cs b/dotnet/examples/JsonSingle/Program.cs index 340bc2c5..515a4530 100644 --- a/dotnet/examples/JsonSingle/Program.cs +++ b/dotnet/examples/JsonSingle/Program.cs @@ -32,6 +32,7 @@ options); Console.WriteLine("Ingesting records..."); +int failed = 0; for (int i = 0; i < 5; i++) { @@ -47,20 +48,33 @@ try { long offset = stream.IngestRecord(jsonRecord); - Console.WriteLine($"Ingested record {i} at offset {offset}"); + Console.WriteLine($"Queued record {i} at offset {offset}"); } catch (ZerobusException ex) when (ex.IsRetryable) { + failed++; Console.WriteLine($"Failed to ingest record {i} (retryable): {ex.RawMessage}"); } catch (ZerobusException ex) { + failed++; Console.WriteLine($"Failed to ingest record {i}: {ex.RawMessage}"); } } -// Confirm every successfully queued record with one durability barrier. +if (failed == 5) +{ + throw new InvalidOperationException("No records were queued"); +} + Console.WriteLine("Waiting for acknowledgments..."); stream.Flush(); -Console.WriteLine("All records successfully ingested and acknowledged!"); +if (failed > 0) +{ + Console.WriteLine($"{5 - failed} records flushed; {failed} ingest calls failed."); +} +else +{ + Console.WriteLine("All records successfully ingested and acknowledged!"); +} diff --git a/dotnet/src/Zerobus/ZerobusStream.cs b/dotnet/src/Zerobus/ZerobusStream.cs index 8777b4ef..adf4bfec 100644 --- a/dotnet/src/Zerobus/ZerobusStream.cs +++ b/dotnet/src/Zerobus/ZerobusStream.cs @@ -69,10 +69,12 @@ public bool IsClosed() /// /// // JSON stream /// long jsonOffset = jsonStream.IngestRecord("{\"id\": 1, \"message\": \"Hello\"}"); + /// jsonStream.Flush(); /// /// // Protobuf stream /// byte[] protoBytes = SerializeMyProto(myMessage); /// long protoOffset = protoStream.IngestRecord(protoBytes); + /// protoStream.Flush(); /// /// public long IngestRecord(string payload) @@ -127,6 +129,7 @@ public long IngestRecord(ReadOnlySpan payload) /// "{\"device\": \"sensor-002\", \"temp\": 21}", /// ]; /// long batchOffset = stream.IngestRecords(records); + /// stream.Flush(); /// /// public long IngestRecords(string[] records) @@ -239,10 +242,17 @@ public Task FlushAsync() /// } /// catch (ZerobusException) /// { - /// var unacked = stream.GetUnackedRecords(); - /// Console.WriteLine($"Failed to acknowledge {unacked.Length} records"); - /// foreach (var payload in unacked) - /// Console.WriteLine($"{payload.Length} bytes"); + /// // A flush timeout can leave the stream active. GetUnackedRecords + /// // requires a closed or failed stream. + /// try + /// { + /// var unacked = stream.GetUnackedRecords(); + /// Console.WriteLine($"Failed to acknowledge {unacked.Length} records"); + /// } + /// catch (ZerobusException retrieval) + /// { + /// Console.WriteLine($"Could not inspect unacked records: {retrieval.Message}"); + /// } /// } /// /// diff --git a/rust/NEXT_CHANGELOG.md b/rust/NEXT_CHANGELOG.md index 5f2f3b66..ec29eaaf 100644 --- a/rust/NEXT_CHANGELOG.md +++ b/rust/NEXT_CHANGELOG.md @@ -38,6 +38,9 @@ - Corrected README and rustdoc examples so their dependencies, feature flags, imports, and mutable stream bindings compile as shown. +- Batch examples and primary rustdoc now queue all records and wait once with + `flush()` or the last offset, and no longer refer to removed `ingest_record()` + / `ingest_records()` methods. ### Internal Changes diff --git a/rust/examples/json/batch.rs b/rust/examples/json/batch.rs index c4e7a4f4..6ef45fce 100644 --- a/rust/examples/json/batch.rs +++ b/rust/examples/json/batch.rs @@ -98,12 +98,7 @@ async fn ingest_with_offset_api(stream: &mut ZerobusStream) -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), ZerobusError> { /// # let my_record = vec![1, 2, 3]; - /// // Ingest multiple records and collect their offsets - /// let mut offsets = Vec::new(); + /// // Queue records, then wait once on the last offset (or call flush()). + /// let mut last_offset = None; /// for i in 0..100 { - /// let offset = stream.ingest_record_offset(vec![i as u8]).await?; - /// offsets.push(offset); + /// last_offset = Some(stream.ingest_record_offset(vec![i as u8]).await?); /// } - /// - /// // Wait for specific offsets - /// for offset in offsets { + /// if let Some(offset) = last_offset { /// stream.wait_for_offset(offset).await?; /// } /// println!("All records acknowledged"); diff --git a/rust/sdk/src/stream/grpc/ingest.rs b/rust/sdk/src/stream/grpc/ingest.rs index ca4c9e89..e328b645 100644 --- a/rust/sdk/src/stream/grpc/ingest.rs +++ b/rust/sdk/src/stream/grpc/ingest.rs @@ -16,9 +16,8 @@ use crate::{EncodedBatch, EncodedRecord, OffsetId, ZerobusError, ZerobusResult}; impl ZerobusStream { /// Ingests a single record and returns its logical offset directly. /// - /// This is an alternative to `ingest_record()` that returns the logical offset directly - /// as an integer (after queuing) instead of wrapping it in a Future. Use `wait_for_offset()` - /// to explicitly wait for server acknowledgment of this offset when needed. + /// Returns the logical offset after the record is queued. Use `wait_for_offset()` + /// or `flush()` to wait for server acknowledgment. /// /// # Arguments /// @@ -65,9 +64,8 @@ impl ZerobusStream { /// Ingests a batch of records and returns the logical offset directly. /// - /// This is an alternative to `ingest_records()` that returns the logical offset directly - /// (after queuing) instead of wrapping it in a Future. Use `wait_for_offset()` to explicitly - /// wait for server acknowledgment when needed. + /// Returns the logical offset after the batch is queued. Use `wait_for_offset()` + /// or `flush()` to wait for server acknowledgment. /// /// # Arguments /// From 2fb6296dcba51f48bcaf16d8f9f03075e9324a26 Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Thu, 13 Aug 2026 15:03:33 +0000 Subject: [PATCH 07/36] Fix leftover Java docs --- java/NEXT_CHANGELOG.md | 5 + java/README.md | 236 ++++++++---------- java/examples/README.md | 77 +++--- java/examples/arrow/README.md | 40 +-- java/examples/json/README.md | 28 ++- java/examples/legacy/README.md | 36 +-- java/examples/proto/README.md | 19 +- .../zerobus/tools/GenerateProto.java | 4 +- java/tools/README.md | 20 +- 9 files changed, 233 insertions(+), 232 deletions(-) diff --git a/java/NEXT_CHANGELOG.md b/java/NEXT_CHANGELOG.md index 4fed9586..07e0f889 100644 --- a/java/NEXT_CHANGELOG.md +++ b/java/NEXT_CHANGELOG.md @@ -20,6 +20,11 @@ `recoveryRetries` defaults to 4, and that `flush()` waits for durability rather than callback completion. `recreateStream()` is no longer presented as a safe production recovery path. +- Maven Central snippets no longer tell users to redeclare compile-scope + transitives (`protobuf-java`, `slf4j-api`). Stream-builder examples use + try-with-resources. GenerateProto docs no longer claim STRUCT support, JAR + examples use `zerobus-ingest-sdk` 1.3.0, and proto examples generate + `AirQualityProto.java` with `protoc` instead of treating it as checked in. ### Internal Changes diff --git a/java/README.md b/java/README.md index 3f2d1665..019cdd5b 100644 --- a/java/README.md +++ b/java/README.md @@ -91,10 +91,14 @@ On Linux, the libc flavor (glibc vs musl) is detected at runtime. To override de ### Dependencies -**When using the fat JAR** (recommended for most users): +**When using Maven or Gradle** (regular JAR, recommended): +- `protobuf-java` and `slf4j-api` are pulled transitively from the published POM. Do not redeclare them unless you need a different version. +- Add an SLF4J implementation such as [`slf4j-simple` 2.0.17](https://mvnrepository.com/artifact/org.slf4j/slf4j-simple/2.0.17) or [`logback-classic` 1.4.14](https://mvnrepository.com/artifact/ch.qos.logback/logback-classic/1.4.14). The SDK depends on the SLF4J API only. + +**When using the fat JAR** (standalone scripts or CLI tools without a build system): - No additional dependencies required - all dependencies are bundled -**When using the regular JAR**: +**When using the regular JAR as a file on the classpath** (without Maven or Gradle): - [`protobuf-java` 4.33.0](https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java/4.33.0) - [`slf4j-api` 2.0.17](https://mvnrepository.com/artifact/org.slf4j/slf4j-api/2.0.17) - An SLF4J implementation such as [`slf4j-simple` 2.0.17](https://mvnrepository.com/artifact/org.slf4j/slf4j-simple/2.0.17) or [`logback-classic` 1.4.14](https://mvnrepository.com/artifact/ch.qos.logback/logback-classic/1.4.14) @@ -155,35 +159,14 @@ dependencies { } ``` -**Important**: You must also add the required dependencies manually, as they are not automatically included: +Add an SLF4J implementation (the SDK depends on `slf4j-api` only): ```xml - - - - - com.databricks - zerobus-ingest-sdk - 1.3.0 - - - - - com.google.protobuf - protobuf-java - 4.33.0 - - - org.slf4j - slf4j-api - 2.0.17 - - - org.slf4j - slf4j-simple - 2.0.17 - - + + org.slf4j + slf4j-simple + 2.0.17 + ``` **Fat JAR (with all dependencies bundled):** @@ -278,13 +261,12 @@ Create `pom.xml`: 1.3.0 - + - com.google.protobuf - protobuf-java - 4.33.0 + org.slf4j + slf4j-simple + 2.0.17 - ``` @@ -415,12 +397,13 @@ The tool automatically maps Unity Catalog types to proto2 types: | TIMESTAMP | int64 | | ARRAY\ | repeated type | | MAP\ | map\ | -| STRUCT\ | nested message | + +`STRUCT` columns are not generated. Map those fields by hand if needed. **Benefits:** - No manual schema creation required - Ensures schema consistency between your table and protobuf definitions -- Automatically handles complex types (arrays, maps, structs) +- Automatically handles ARRAY and MAP columns (`STRUCT` is not generated) - Reduces errors from manual type mapping - No need to clone the repository - runs directly from the SDK JAR @@ -565,29 +548,35 @@ API for all stream types and mirrors the Rust SDK's `stream_builder()`: ```java // Protocol Buffer -ZerobusProtoStream protoStream = sdk.streamBuilder() - .table("catalog.schema.table") - .oauth(clientId, clientSecret) - .compiledProto(MyProto.getDescriptor().toProto()) - .build() - .join(); +try (ZerobusProtoStream protoStream = sdk.streamBuilder() + .table("catalog.schema.table") + .oauth(clientId, clientSecret) + .compiledProto(MyProto.getDescriptor().toProto()) + .build() + .join()) { + // ingest... +} // JSON -ZerobusJsonStream jsonStream = sdk.streamBuilder() - .table("catalog.schema.table") - .oauth(clientId, clientSecret) - .json() - .build() - .join(); +try (ZerobusJsonStream jsonStream = sdk.streamBuilder() + .table("catalog.schema.table") + .oauth(clientId, clientSecret) + .json() + .build() + .join()) { + // ingest... +} // Arrow Flight (Beta) -ZerobusArrowStream arrowStream = sdk.streamBuilder() - .table("catalog.schema.table") - .oauth(clientId, clientSecret) - .arrow(schema) - .ipcCompression(IPCCompressionType.ZSTD) - .build() - .join(); +try (ZerobusArrowStream arrowStream = sdk.streamBuilder() + .table("catalog.schema.table") + .oauth(clientId, clientSecret) + .arrow(schema) + .ipcCompression(IPCCompressionType.ZSTD) + .build() + .join()) { + // ingest... +} ``` Stream configuration is set directly on the builder (for example `.maxInflightRecords(50000)`, @@ -628,14 +617,14 @@ java -cp "../../target/zerobus-ingest-sdk-*-jar-with-dependencies.jar:." \ **Clean JSON API** - use the stream builder for a simplified experience: ```java -// No proto types or configuration needed! -ZerobusJsonStream stream = sdk.streamBuilder() - .table(tableName) - .oauth(clientId, clientSecret) - .json() - .build() - .join(); -stream.ingestRecordOffset("{\"field\": \"value\"}"); +try (ZerobusJsonStream stream = sdk.streamBuilder() + .table(tableName) + .oauth(clientId, clientSecret) + .json() + .build() + .join()) { + stream.ingestRecordOffset("{\"field\": \"value\"}"); +} ``` See [`examples/README.md`](https://github.com/databricks/zerobus-sdk/blob/main/java/examples/README.md) for detailed documentation. @@ -661,21 +650,20 @@ Schema schema = new Schema(Arrays.asList( Field.nullable("device_name", ArrowType.LargeUtf8.INSTANCE), Field.nullable("temp", new ArrowType.Int(32, true)))); -ZerobusArrowStream stream = sdk.streamBuilder() - .table(tableName) - .oauth(clientId, clientSecret) - .arrow(schema) - .build() - .join(); - -try (VectorSchemaRoot batch = VectorSchemaRoot.create(schema, allocator)) { +try (ZerobusArrowStream stream = sdk.streamBuilder() + .table(tableName) + .oauth(clientId, clientSecret) + .arrow(schema) + .build() + .join(); + VectorSchemaRoot batch = VectorSchemaRoot.create(schema, allocator)) { // populate batch... Optional offset = stream.ingestBatch(batch); if (offset.isPresent()) { stream.waitForOffset(offset.get()); } + stream.flush(); } -stream.close(); ``` > **Beta.** Arrow Flight ingestion is in Beta. The API is stabilising but may still change before reaching GA. @@ -695,12 +683,14 @@ HeadersProvider provider = () -> { return headers; }; -ZerobusJsonStream stream = sdk.streamBuilder() - .table(tableName) - .headersProvider(provider) - .json() - .build() - .join(); +try (ZerobusJsonStream stream = sdk.streamBuilder() + .table(tableName) + .headersProvider(provider) + .json() + .build() + .join()) { + // ingest... +} ``` The same provider works with the builder's `compiledProto()` and `arrow()` format selectors. @@ -732,14 +722,12 @@ Use `ZerobusProtoStream` or `ZerobusJsonStream` for all new code. They use offse > [Acknowledgments and throughput](#acknowledgments-and-throughput) for the full picture. ```java -ZerobusProtoStream stream = sdk.streamBuilder() - .table(tableName) - .oauth(clientId, clientSecret) - .compiledProto(AirQuality.getDescriptor().toProto()) - .build() - .join(); - -try { +try (ZerobusProtoStream stream = sdk.streamBuilder() + .table(tableName) + .oauth(clientId, clientSecret) + .compiledProto(AirQuality.getDescriptor().toProto()) + .build() + .join()) { long lastOffset = -1; // Ingest in a loop @@ -756,9 +744,6 @@ try { // Confirm all records are acknowledged stream.waitForOffset(lastOffset); -} finally { - stream.close(); - sdk.close(); } ``` @@ -828,51 +813,41 @@ stream.flush(); Use the stream builder for a clean API that doesn't require Protocol Buffer types: ```java -// Create JSON stream - no proto types needed! -ZerobusJsonStream stream = sdk.streamBuilder() - .table("catalog.schema.table") - .oauth(clientId, clientSecret) - .json() - .build() - .join(); +try (ZerobusJsonStream stream = sdk.streamBuilder() + .table("catalog.schema.table") + .oauth(clientId, clientSecret) + .json() + .build() + .join()) { + stream.ingestRecordOffset("{\"device_name\": \"sensor-1\", \"temp\": 25}"); -try { - // Ingest JSON string directly - long offset = stream.ingestRecordOffset("{\"device_name\": \"sensor-1\", \"temp\": 25}"); - stream.waitForOffset(offset); - - // Or use objects with a serializer (Gson, Jackson, etc.) Gson gson = new Gson(); Map data = new HashMap<>(); data.put("device_name", "sensor-2"); data.put("temp", 26); - offset = stream.ingestRecordOffset(data, gson::toJson); + stream.ingestRecordOffset(data, gson::toJson); - // Batch ingestion List batch = Arrays.asList( "{\"device_name\": \"sensor-1\", \"temp\": 25}", "{\"device_name\": \"sensor-2\", \"temp\": 26}" ); - Optional batchOffset = stream.ingestRecordsOffset(batch); - if (batchOffset.isPresent()) { - stream.waitForOffset(batchOffset.get()); - } -} finally { - stream.close(); - sdk.close(); + stream.ingestRecordsOffset(batch); + stream.flush(); } ``` With custom configuration set directly on the builder: ```java -ZerobusJsonStream stream = sdk.streamBuilder() - .table(tableName) - .oauth(clientId, clientSecret) - .maxInflightRecords(50000) - .json() - .build() - .join(); +try (ZerobusJsonStream stream = sdk.streamBuilder() + .table(tableName) + .oauth(clientId, clientSecret) + .maxInflightRecords(50000) + .json() + .build() + .join()) { + // ingest... +} ``` ## Configuration @@ -1591,19 +1566,18 @@ AckCallback callback = new AckCallback() { } }; -ZerobusProtoStream stream = sdk.streamBuilder() - .table(tableName) - .oauth(clientId, clientSecret) - .ackCallback(callback) - .compiledProto(descriptor) - .build() - .join(); - -// Ingest without blocking; the callback fires as acks arrive. -for (AirQuality record : records) { - stream.ingestRecordOffset(record); +try (ZerobusProtoStream stream = sdk.streamBuilder() + .table(tableName) + .oauth(clientId, clientSecret) + .ackCallback(callback) + .compiledProto(descriptor) + .build() + .join()) { + for (AirQuality record : records) { + stream.ingestRecordOffset(record); + } + stream.flush(); // wait for durability; callbacks may still be running until close() } -stream.flush(); // wait for durability; callbacks may still be running until close() ``` Implementations must be thread-safe and lightweight (callbacks run on internal @@ -1612,7 +1586,7 @@ processing threads). ## Best Practices 1. **Reuse SDK instances**: Create one `ZerobusSdk` instance per application -2. **Stream lifecycle**: Always close streams in a `finally` block or use try-with-resources +2. **Stream lifecycle**: Wrap the SDK and each stream in try-with-resources. Streams hold native resources that the garbage collector does not release. 3. **Use offset-based API for high throughput**: `ingestRecordOffset()` avoids `CompletableFuture` overhead 4. **Ingest in a loop, then `flush()`**: Confirm durability once after a batch with `flush()` (or `waitForOffset()` on the last offset, since acks are ordered). Use per-record waits only when a specific record must be confirmed before continuing. 5. **Batch records when possible**: Use `ingestRecordsOffset()` for multiple records diff --git a/java/examples/README.md b/java/examples/README.md index 15061967..1adc2c25 100644 --- a/java/examples/README.md +++ b/java/examples/README.md @@ -19,7 +19,7 @@ examples/ ├── README.md (this file) ├── proto/ (Protocol Buffer examples - ZerobusProtoStream) │ ├── README.md -│ ├── AirQualityProto.java (generated proto) +│ ├── air_quality.proto (compile with protoc --java_out=proto) │ ├── SingleRecordExample.java │ └── BatchIngestionExample.java ├── json/ (JSON examples - ZerobusJsonStream) @@ -51,35 +51,33 @@ Each example demonstrates: single ingestion + wait, batch ingestion + wait for l ### ZerobusProtoStream (Recommended for Protocol Buffers) ```java -ZerobusProtoStream stream = sdk.streamBuilder() - .table(tableName) - .oauth(clientId, clientSecret) - .compiledProto(MyProto.getDescriptor().toProto()) - .build() - .join(); - -// Method-level generics - flexible typing -stream.ingestRecordOffset(myProtoMessage); // Message -stream.ingestRecordOffset(preEncodedBytes); // byte[] -stream.ingestRecordsOffset(listOfMessages); // batch -stream.ingestRecordsOffset(listOfByteArrays); // batch +try (ZerobusProtoStream stream = sdk.streamBuilder() + .table(tableName) + .oauth(clientId, clientSecret) + .compiledProto(MyProto.getDescriptor().toProto()) + .build() + .join()) { + stream.ingestRecordOffset(myProtoMessage); // Message + stream.ingestRecordOffset(preEncodedBytes); // byte[] + stream.ingestRecordsOffset(listOfMessages); // batch + stream.ingestRecordsOffset(listOfByteArrays); // batch +} ``` ### ZerobusJsonStream (Recommended for JSON) ```java -ZerobusJsonStream stream = sdk.streamBuilder() - .table(tableName) - .oauth(clientId, clientSecret) - .json() - .build() - .join(); - -// Method-level generics - flexible typing -stream.ingestRecordOffset(object, gson::toJson); // Object + serializer -stream.ingestRecordOffset(jsonString); // String -stream.ingestRecordsOffset(objects, gson::toJson);// batch -stream.ingestRecordsOffset(jsonStrings); // batch +try (ZerobusJsonStream stream = sdk.streamBuilder() + .table(tableName) + .oauth(clientId, clientSecret) + .json() + .build() + .join()) { + stream.ingestRecordOffset(object, gson::toJson); // Object + serializer + stream.ingestRecordOffset(jsonString); // String + stream.ingestRecordsOffset(objects, gson::toJson);// batch + stream.ingestRecordsOffset(jsonStrings); // batch +} ``` ### ZerobusArrowStream (Beta - Arrow Flight) @@ -90,17 +88,16 @@ Schema schema = new Schema(Arrays.asList( Field.nullable("temp", new ArrowType.Int(32, true)) )); -ZerobusArrowStream stream = sdk.streamBuilder() - .table(tableName) - .oauth(clientId, clientSecret) - .arrow(schema) - .build() - .join(); - -// Columnar batch ingestion -Optional offset = stream.ingestBatch(vectorSchemaRoot); -if (offset.isPresent()) { - stream.waitForOffset(offset.get()); +try (ZerobusArrowStream stream = sdk.streamBuilder() + .table(tableName) + .oauth(clientId, clientSecret) + .arrow(schema) + .build() + .join()) { + Optional offset = stream.ingestBatch(vectorSchemaRoot); + if (offset.isPresent()) { + stream.waitForOffset(offset.get()); + } } ``` @@ -156,6 +153,9 @@ mvn package -DskipTests -Dzerobus.skipNativeLibCheck=true ```bash cd examples +# Generate AirQualityProto.java from the proto schema (not checked in) +protoc --java_out=proto proto/air_quality.proto + # Compile examples javac -d . -cp "../target/classes:$(cd .. && mvn dependency:build-classpath -q -DincludeScope=runtime -Dmdep.outputFile=/dev/stdout)" \ proto/com/databricks/zerobus/examples/proto/AirQualityProto.java \ @@ -195,7 +195,10 @@ java -cp ".:../target/classes:$(cd .. && mvn dependency:build-classpath -q -Dinc ```bash cd examples -# Compile (requires proto for AirQuality) +# Generate AirQualityProto.java if you have not already (not checked in) +protoc --java_out=proto proto/air_quality.proto + +# Compile javac -d . -cp "../target/classes:$(cd .. && mvn dependency:build-classpath -q -DincludeScope=runtime -Dmdep.outputFile=/dev/stdout)" \ proto/com/databricks/zerobus/examples/proto/AirQualityProto.java \ legacy/LegacyStreamExample.java diff --git a/java/examples/arrow/README.md b/java/examples/arrow/README.md index 2bdfd3a4..4c3bd03c 100644 --- a/java/examples/arrow/README.md +++ b/java/examples/arrow/README.md @@ -89,12 +89,14 @@ Schema schema = new Schema(Arrays.asList( Field.nullable("humidity", new ArrowType.Int(64, true)) )); -ZerobusArrowStream stream = sdk.streamBuilder() - .table(tableName) - .oauth(clientId, clientSecret) - .arrow(schema) - .build() - .join(); +try (ZerobusArrowStream stream = sdk.streamBuilder() + .table(tableName) + .oauth(clientId, clientSecret) + .arrow(schema) + .build() + .join()) { + // ingest... +} ``` ### Ingesting Batches @@ -118,18 +120,20 @@ Set shared and Arrow-specific options directly on the builder. Arrow-specific kn calling `.arrow(...)`: ```java -ZerobusArrowStream stream = sdk.streamBuilder() - .table(tableName) - .oauth(clientId, clientSecret) - .flushTimeoutMs(600000) - .recovery(true) - .recoveryRetries(5) - .arrow(schema) - .maxInflightBatches(2000) - .ipcCompression(IPCCompressionType.ZSTD) - .streamPausedMaxWaitTimeMs(5000) - .build() - .join(); +try (ZerobusArrowStream stream = sdk.streamBuilder() + .table(tableName) + .oauth(clientId, clientSecret) + .flushTimeoutMs(600000) + .recovery(true) + .recoveryRetries(5) + .arrow(schema) + .maxInflightBatches(2000) + .ipcCompression(IPCCompressionType.ZSTD) + .streamPausedMaxWaitTimeMs(5000) + .build() + .join()) { + // ingest... +} ``` ### Recovering Unacknowledged Batches diff --git a/java/examples/json/README.md b/java/examples/json/README.md index 8c72157c..259565a4 100644 --- a/java/examples/json/README.md +++ b/java/examples/json/README.md @@ -41,12 +41,14 @@ java -cp ".:../target/classes:$(cd .. && mvn dependency:build-classpath -q -Dinc ### Creating a JSON Stream ```java -ZerobusJsonStream stream = sdk.streamBuilder() - .table(tableName) - .oauth(clientId, clientSecret) - .json() - .build() - .join(); +try (ZerobusJsonStream stream = sdk.streamBuilder() + .table(tableName) + .oauth(clientId, clientSecret) + .json() + .build() + .join()) { + // ingest... +} ``` For custom authentication, provide the authorization and table name headers: @@ -59,12 +61,14 @@ HeadersProvider provider = () -> { return headers; }; -ZerobusJsonStream stream = sdk.streamBuilder() - .table(tableName) - .headersProvider(provider) - .json() - .build() - .join(); +try (ZerobusJsonStream stream = sdk.streamBuilder() + .table(tableName) + .headersProvider(provider) + .json() + .build() + .join()) { + // ingest... +} ``` ### Single Record Ingestion diff --git a/java/examples/legacy/README.md b/java/examples/legacy/README.md index 5d686272..72d18c69 100644 --- a/java/examples/legacy/README.md +++ b/java/examples/legacy/README.md @@ -16,7 +16,10 @@ This directory contains examples using the deprecated `ZerobusStream` class. ```bash cd examples -# Compile (requires AirQualityProto from proto folder) +# Generate AirQualityProto.java from the proto schema (not checked in) +protoc --java_out=proto proto/air_quality.proto + +# Compile javac -d . -cp "../target/classes:$(cd .. && mvn dependency:build-classpath -q -DincludeScope=runtime -Dmdep.outputFile=/dev/stdout)" \ proto/com/databricks/zerobus/examples/proto/AirQualityProto.java \ legacy/LegacyStreamExample.java @@ -74,21 +77,22 @@ stream.ingestRecord(record).join(); ### After (ZerobusProtoStream) ```java -ZerobusProtoStream stream = sdk.streamBuilder() - .table(tableName) - .oauth(clientId, clientSecret) - .compiledProto(AirQuality.getDescriptor().toProto()) - .build() - .join(); - -// Non-blocking, returns offset -long offset = stream.ingestRecordOffset(record); - -// Wait when needed -stream.waitForOffset(offset); - -// Batch support -Optional batchOffset = stream.ingestRecordsOffset(records); +try (ZerobusProtoStream stream = sdk.streamBuilder() + .table(tableName) + .oauth(clientId, clientSecret) + .compiledProto(AirQuality.getDescriptor().toProto()) + .build() + .join()) { + for (AirQuality record : records) { + stream.ingestRecordOffset(record); + } + stream.flush(); + + Optional batchOffset = stream.ingestRecordsOffset(records); + if (batchOffset.isPresent()) { + stream.waitForOffset(batchOffset.get()); + } +} ``` ## Why Migrate? diff --git a/java/examples/proto/README.md b/java/examples/proto/README.md index 693632f5..287e6b94 100644 --- a/java/examples/proto/README.md +++ b/java/examples/proto/README.md @@ -15,7 +15,10 @@ This directory contains examples for ingesting data using `ZerobusProtoStream`. ```bash cd examples -# Compile (AirQualityProto.java is pre-generated) +# Generate AirQualityProto.java from the proto schema (not checked in) +protoc --java_out=proto proto/air_quality.proto + +# Compile javac -d . -cp "../target/classes:$(cd .. && mvn dependency:build-classpath -q -DincludeScope=runtime -Dmdep.outputFile=/dev/stdout)" \ proto/com/databricks/zerobus/examples/proto/AirQualityProto.java \ proto/SingleRecordExample.java \ @@ -42,12 +45,14 @@ java -cp ".:../target/classes:$(cd .. && mvn dependency:build-classpath -q -Dinc ### Creating a Proto Stream ```java -ZerobusProtoStream stream = sdk.streamBuilder() - .table(tableName) - .oauth(clientId, clientSecret) - .compiledProto(AirQuality.getDescriptor().toProto()) - .build() - .join(); +try (ZerobusProtoStream stream = sdk.streamBuilder() + .table(tableName) + .oauth(clientId, clientSecret) + .compiledProto(AirQuality.getDescriptor().toProto()) + .build() + .join()) { + // ingest... +} ``` ### Single Record Ingestion diff --git a/java/src/main/java/com/databricks/zerobus/tools/GenerateProto.java b/java/src/main/java/com/databricks/zerobus/tools/GenerateProto.java index 55975669..c503a469 100644 --- a/java/src/main/java/com/databricks/zerobus/tools/GenerateProto.java +++ b/java/src/main/java/com/databricks/zerobus/tools/GenerateProto.java @@ -19,8 +19,8 @@ * Generate proto2 file from Unity Catalog table schema. * *

This tool fetches table schema from Unity Catalog and generates a corresponding proto2 - * definition file. It supports all Delta data types and maps them to appropriate Protocol Buffer - * types. + * definition file. It supports scalar Delta types plus ARRAY and MAP, and maps + * them to Protocol Buffer types. STRUCT columns are not generated. * *

Usage: java GenerateProto --uc-endpoint <endpoint> --client-id <id> * --client-secret <secret> --table <catalog.schema.table> --output <output.proto> diff --git a/java/tools/README.md b/java/tools/README.md index 4a4bea3e..30ab2a7c 100644 --- a/java/tools/README.md +++ b/java/tools/README.md @@ -11,9 +11,9 @@ The tool is **packaged within the Zerobus SDK JAR**, so users can run it directl ## Features - Fetches table schema directly from Unity Catalog -- Supports all standard Delta data types +- Supports scalar Delta types plus ARRAY and MAP - Generates proto2 format files -- Handles complex types (arrays and maps) +- Handles ARRAY and MAP columns (STRUCT / nested messages are not generated) - Uses OAuth 2.0 client credentials authentication - No external dependencies beyond Java standard library - Packaged in SDK JAR for easy distribution @@ -51,7 +51,7 @@ If you have downloaded the SDK JAR without the source code: ```bash # Using the shaded JAR (includes all dependencies) -java -cp databricks-zerobus-ingest-sdk-0.1.0-jar-with-dependencies.jar \ +java -cp zerobus-ingest-sdk-1.3.0-jar-with-dependencies.jar \ com.databricks.zerobus.tools.GenerateProto \ --uc-endpoint "https://your-workspace.cloud.databricks.com" \ --client-id "your-client-id" \ @@ -65,7 +65,7 @@ Or, if the JAR has a Main-Class manifest entry (which it does): ```bash # Even simpler - just use -jar flag -java -jar databricks-zerobus-ingest-sdk-0.1.0-jar-with-dependencies.jar \ +java -jar zerobus-ingest-sdk-1.3.0-jar-with-dependencies.jar \ --uc-endpoint "https://your-workspace.cloud.databricks.com" \ --client-id "your-client-id" \ --client-secret "your-client-secret" \ @@ -103,6 +103,8 @@ The tool automatically maps Delta/Unity Catalog types to Protocol Buffer types: | `ARRAY` | `repeated type` | | `MAP` | `map` | +`STRUCT` columns are not generated. The tool throws `Unsupported column type` for them; map those fields by hand if needed. + ## Examples ### Basic Usage @@ -111,7 +113,7 @@ Generate a proto file for a simple table: **From the SDK JAR:** ```bash -java -jar databricks-zerobus-ingest-sdk-0.1.0-jar-with-dependencies.jar \ +java -jar zerobus-ingest-sdk-1.3.0-jar-with-dependencies.jar \ --uc-endpoint "https://myworkspace.cloud.databricks.com" \ --client-id "abc123" \ --client-secret "secret123" \ @@ -147,7 +149,7 @@ message users { Specify a custom message name: ```bash -java -jar databricks-zerobus-ingest-sdk-0.1.0-jar-with-dependencies.jar \ +java -jar zerobus-ingest-sdk-1.3.0-jar-with-dependencies.jar \ --uc-endpoint "https://myworkspace.cloud.databricks.com" \ --client-id "abc123" \ --client-secret "secret123" \ @@ -161,7 +163,7 @@ java -jar databricks-zerobus-ingest-sdk-0.1.0-jar-with-dependencies.jar \ The tool handles complex types like arrays and maps: ```bash -java -jar databricks-zerobus-ingest-sdk-0.1.0-jar-with-dependencies.jar \ +java -jar zerobus-ingest-sdk-1.3.0-jar-with-dependencies.jar \ --uc-endpoint "https://myworkspace.cloud.databricks.com" \ --client-id "abc123" \ --client-secret "secret123" \ @@ -236,14 +238,14 @@ If you encounter unsupported type errors: ## Distribution -The tool is distributed as part of the Zerobus SDK JAR. When you download or build the SDK, the `GenerateProto` tool is automatically included in the shaded JAR file (`databricks-zerobus-ingest-sdk-*-jar-with-dependencies.jar`). +The tool is distributed as part of the Zerobus SDK JAR. When you download or build the SDK, the `GenerateProto` tool is automatically included in the shaded JAR file (`zerobus-ingest-sdk-*-jar-with-dependencies.jar`). Users can run the tool directly from the JAR without needing access to the source code: ```bash # Download the SDK JAR (or build it with mvn package -Dzerobus.skipNativeLibCheck=true) # Then simply run: -java -jar databricks-zerobus-ingest-sdk-0.1.0-jar-with-dependencies.jar \ +java -jar zerobus-ingest-sdk-1.3.0-jar-with-dependencies.jar \ --uc-endpoint "..." \ --client-id "..." \ --client-secret "..." \ From d8dbfc33a43b02a1dff63be857ba22187b34e541 Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Thu, 13 Aug 2026 15:03:48 +0000 Subject: [PATCH 08/36] Fix leftover Rust docs --- rust/NEXT_CHANGELOG.md | 3 +++ rust/examples/arrow/README.md | 4 +++- rust/examples/json/README.md | 4 ++-- rust/examples/proto/README.md | 4 ++-- rust/sdk/src/stream/grpc/acks.rs | 6 +++--- rust/tools/generate_files/README.md | 2 +- 6 files changed, 14 insertions(+), 9 deletions(-) diff --git a/rust/NEXT_CHANGELOG.md b/rust/NEXT_CHANGELOG.md index ec29eaaf..89b562d6 100644 --- a/rust/NEXT_CHANGELOG.md +++ b/rust/NEXT_CHANGELOG.md @@ -41,6 +41,9 @@ - Batch examples and primary rustdoc now queue all records and wait once with `flush()` or the last offset, and no longer refer to removed `ingest_record()` / `ingest_records()` methods. +- Example READMEs and `get_unacked_*` rustdoc now name `ingest_record_offset()` / + `ingest_records_offset()`. The generate-files tool README quoting is valid shell. + Arrow example docs place schema validation at stream creation, not the first batch. ### Internal Changes diff --git a/rust/examples/arrow/README.md b/rust/examples/arrow/README.md index 839e2f3a..f9b022ed 100644 --- a/rust/examples/arrow/README.md +++ b/rust/examples/arrow/README.md @@ -154,4 +154,6 @@ let batch = RecordBatch::try_new( **3. Update table name and credentials** in the constants at the top of `main.rs`. -> **Tip.** When in doubt about the Arrow type for a given Delta column type, the SDK validates the schema on the first batch — a mismatch fails fast with a descriptive error. +> **Tip.** When in doubt about the Arrow type for a given Delta column type, the SDK +> validates the schema when the stream is created. A mismatch fails fast with a +> descriptive error. diff --git a/rust/examples/json/README.md b/rust/examples/json/README.md index c8dcd41b..23f5f15a 100644 --- a/rust/examples/json/README.md +++ b/rust/examples/json/README.md @@ -24,8 +24,8 @@ JSON examples are recommended for getting started - they're simpler and don't re - Great for quick prototyping **Available examples:** -- **`single.rs`** - Ingest records one at a time using `ingest_record_offset()` / `ingest_record()` -- **`batch.rs`** - Ingest multiple records at once using `ingest_records_offset()` / `ingest_records()` +- **`single.rs`** - Ingest records one at a time using `ingest_record_offset()` +- **`batch.rs`** - Ingest multiple records at once using `ingest_records_offset()` ## Three Ways to Pass Data diff --git a/rust/examples/proto/README.md b/rust/examples/proto/README.md index 163823d5..a00c9ea6 100644 --- a/rust/examples/proto/README.md +++ b/rust/examples/proto/README.md @@ -37,8 +37,8 @@ The examples are grouped by how the protobuf schema is obtained: - **`compiled/`** — the schema is known ahead of time and compiled into Rust structs. **No schema generation needed to run these** — the files under `compiled/output/` are already included. - - **`compiled/single.rs`** - Ingest records one at a time using `ingest_record_offset()` / `ingest_record()` - - **`compiled/batch.rs`** - Ingest multiple records at once using `ingest_records_offset()` / `ingest_records()` + - **`compiled/single.rs`** - Ingest records one at a time using `ingest_record_offset()` + - **`compiled/batch.rs`** - Ingest multiple records at once using `ingest_records_offset()` - **`dynamic/`** — the schema is known only at runtime (no compiled `.proto`), and records are built field-by-field with `DynamicRecord`. - **`dynamic/single.rs`** - Build the descriptor in code and ingest dynamic records one at a time diff --git a/rust/sdk/src/stream/grpc/acks.rs b/rust/sdk/src/stream/grpc/acks.rs index 4916fe23..bb444209 100644 --- a/rust/sdk/src/stream/grpc/acks.rs +++ b/rust/sdk/src/stream/grpc/acks.rs @@ -224,7 +224,7 @@ impl ZerobusStream { /// /// An iterator over individual `EncodedRecord` items. All unacknowledged records are /// flattened into a single sequence, regardless of how they were originally ingested - /// (via `ingest_record()` or `ingest_records()`). + /// (via `ingest_record_offset()` or `ingest_records_offset()`). /// /// # Errors /// @@ -265,8 +265,8 @@ impl ZerobusStream { /// /// **Note:** This method returns the unacknowledged records as a vector of `EncodedBatch` items, /// where each batch corresponds to how records were ingested: - /// - Each `ingest_record()` call creates a single batch containing one record - /// - Each `ingest_records()` call creates a single batch containing multiple records + /// - Each `ingest_record_offset()` call creates a single batch containing one record + /// - Each `ingest_records_offset()` call creates a single batch containing multiple records /// /// For alternatives, see `ZerobusStream::get_unacked_records()` and `ZerobusSdk::recreate_stream()`. /// diff --git a/rust/tools/generate_files/README.md b/rust/tools/generate_files/README.md index 2f0345a4..f856c5e9 100644 --- a/rust/tools/generate_files/README.md +++ b/rust/tools/generate_files/README.md @@ -51,7 +51,7 @@ cargo run -- \ ```bash cargo run -- \ - --uc-endpoint """ \ + --uc-endpoint "" \ --client-id "your-client-id" \ --client-secret "your-client-secret" \ --table "catalog.schema.table_name" \ From 7da7400adeeebdf20ad8569e8b1cfc8086fb1b33 Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Thu, 13 Aug 2026 15:03:49 +0000 Subject: [PATCH 09/36] Fix TypeScript inflight default --- typescript/NEXT_CHANGELOG.md | 1 + typescript/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/typescript/NEXT_CHANGELOG.md b/typescript/NEXT_CHANGELOG.md index d1f516c9..a2537538 100644 --- a/typescript/NEXT_CHANGELOG.md +++ b/typescript/NEXT_CHANGELOG.md @@ -36,6 +36,7 @@ - Documented that omitted `descriptorProto` does not select JSON, that the inherited inflight default is 1,000,000, and that `close()` is still required to flush. README `main().catch` handlers now set a non-zero exit code. +- Binding rustdoc for `maxInflightRequests` now matches that 1,000,000 default. - Clarified the high-throughput ingestion pattern across the README, API reference, JSDoc doc comments (`ingestRecordOffset`, `ingestRecordsOffset`, `waitForOffset`, `flush`), and diff --git a/typescript/src/lib.rs b/typescript/src/lib.rs index ebc5f4a4..ddbe2e7f 100644 --- a/typescript/src/lib.rs +++ b/typescript/src/lib.rs @@ -50,7 +50,7 @@ pub enum RecordType { #[napi(object)] pub struct StreamConfigurationOptions { /// Maximum number of unacknowledged requests that can be in flight. - /// Default: 10,000 + /// Default: 1,000,000 pub max_inflight_requests: Option, /// Enable automatic stream recovery on transient failures. From d4641f7f9eec4e6f3c963748eb215de779a00fa3 Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Thu, 13 Aug 2026 15:03:51 +0000 Subject: [PATCH 10/36] Fix leftover Go docs --- go/NEXT_CHANGELOG.md | 3 +++ go/examples/arrow/go.mod | 2 +- go/examples/json/batch/go.mod | 2 +- go/examples/json/batch/main.go | 8 ++++---- go/examples/json/single/go.mod | 2 +- go/examples/proto/batch/go.mod | 2 +- go/examples/proto/batch/main.go | 8 ++++---- go/examples/proto/go.mod | 2 +- go/examples/proto/single/go.mod | 2 +- go/tests/go.mod | 2 +- 10 files changed, 18 insertions(+), 15 deletions(-) diff --git a/go/NEXT_CHANGELOG.md b/go/NEXT_CHANGELOG.md index 8f81c902..521dc2f0 100644 --- a/go/NEXT_CHANGELOG.md +++ b/go/NEXT_CHANGELOG.md @@ -16,6 +16,9 @@ `RecordAck.Await()` waits for server durability, and that one stream can be used from multiple goroutines. - Added `Flush()` to copyable example snippets. +- Batch examples name the offset returned by `IngestRecordsOffset` `batchOffset`. + JSON and protobuf example modules declare `go 1.21` to match the SDK minimum. + The Arrow example and tests declare `go 1.22.0`, which is what `arrow-go` v18 requires. ### Internal Changes diff --git a/go/examples/arrow/go.mod b/go/examples/arrow/go.mod index da778195..f3111066 100644 --- a/go/examples/arrow/go.mod +++ b/go/examples/arrow/go.mod @@ -1,6 +1,6 @@ module zerobus-examples/arrow -go 1.24.0 +go 1.22.0 require ( github.com/apache/arrow-go/v18 v18.0.0 diff --git a/go/examples/json/batch/go.mod b/go/examples/json/batch/go.mod index f1904c49..85dcfcf7 100644 --- a/go/examples/json/batch/go.mod +++ b/go/examples/json/batch/go.mod @@ -1,6 +1,6 @@ module zerobus-examples/json-batch -go 1.25.3 +go 1.21 require github.com/databricks/zerobus-sdk/go v0.1.0 diff --git a/go/examples/json/batch/main.go b/go/examples/json/batch/main.go index a99c8608..b66d9533 100644 --- a/go/examples/json/batch/main.go +++ b/go/examples/json/batch/main.go @@ -60,14 +60,14 @@ func main() { `{"device_name": "sensor-005", "temp": 24, "humidity": 64}`, } - lastOffset, err := stream.IngestRecordsOffset(batchRecords) + batchOffset, err := stream.IngestRecordsOffset(batchRecords) if err != nil { log.Fatalf("Failed to ingest batch: %v", err) } - log.Printf("Batch of %d records ingested, last offset: %d", len(batchRecords), lastOffset) + log.Printf("Batch of %d records ingested, batch offset: %d", len(batchRecords), batchOffset) - // Wait for the last offset to ensure the entire batch is acknowledged. - if err := stream.WaitForOffset(lastOffset); err != nil { + // Wait for the batch offset to ensure the entire batch is acknowledged. + if err := stream.WaitForOffset(batchOffset); err != nil { log.Fatalf("Failed to wait for batch acknowledgment: %v", err) } log.Println("Batch acknowledged!") diff --git a/go/examples/json/single/go.mod b/go/examples/json/single/go.mod index 0c6c870d..b94a3e30 100644 --- a/go/examples/json/single/go.mod +++ b/go/examples/json/single/go.mod @@ -1,6 +1,6 @@ module zerobus-examples/json-single -go 1.25.3 +go 1.21 require github.com/databricks/zerobus-sdk/go v0.1.0 diff --git a/go/examples/proto/batch/go.mod b/go/examples/proto/batch/go.mod index ae5c9e5b..1fea6b09 100644 --- a/go/examples/proto/batch/go.mod +++ b/go/examples/proto/batch/go.mod @@ -1,6 +1,6 @@ module zerobus-examples/proto-batch -go 1.25.3 +go 1.21 require ( github.com/databricks/zerobus-sdk/go v0.1.0 diff --git a/go/examples/proto/batch/main.go b/go/examples/proto/batch/main.go index a03a77e0..734fe99f 100644 --- a/go/examples/proto/batch/main.go +++ b/go/examples/proto/batch/main.go @@ -84,14 +84,14 @@ func main() { batchRecords = append(batchRecords, data) } - lastOffset, err := stream.IngestRecordsOffset(batchRecords) + batchOffset, err := stream.IngestRecordsOffset(batchRecords) if err != nil { log.Fatalf("Failed to ingest batch: %v", err) } - log.Printf("Batch of %d records ingested, last offset: %d", len(batchRecords), lastOffset) + log.Printf("Batch of %d records ingested, batch offset: %d", len(batchRecords), batchOffset) - // Wait for the last offset to ensure the entire batch is acknowledged. - if err := stream.WaitForOffset(lastOffset); err != nil { + // Wait for the batch offset to ensure the entire batch is acknowledged. + if err := stream.WaitForOffset(batchOffset); err != nil { log.Fatalf("Failed to wait for batch acknowledgment: %v", err) } log.Println("Batch acknowledged!") diff --git a/go/examples/proto/go.mod b/go/examples/proto/go.mod index 09f4360e..d917a560 100644 --- a/go/examples/proto/go.mod +++ b/go/examples/proto/go.mod @@ -1,6 +1,6 @@ module zerobus-examples -go 1.25.3 +go 1.21 require ( google.golang.org/protobuf v1.36.10 diff --git a/go/examples/proto/single/go.mod b/go/examples/proto/single/go.mod index f6fade20..24838854 100644 --- a/go/examples/proto/single/go.mod +++ b/go/examples/proto/single/go.mod @@ -1,6 +1,6 @@ module zerobus-examples/proto-single -go 1.25.3 +go 1.21 require ( github.com/databricks/zerobus-sdk/go v0.1.0 diff --git a/go/tests/go.mod b/go/tests/go.mod index ebb30171..4beced83 100644 --- a/go/tests/go.mod +++ b/go/tests/go.mod @@ -1,6 +1,6 @@ module github.com/databricks/zerobus-sdk/go/tests -go 1.24.0 +go 1.22.0 require ( github.com/apache/arrow-go/v18 v18.0.0 From 2875180a8bf55774dc286951a4d106c8fc7c528f Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Thu, 13 Aug 2026 15:03:53 +0000 Subject: [PATCH 11/36] Fix C++ Arrow schema docs --- cpp/NEXT_CHANGELOG.md | 2 ++ cpp/examples/arrow/README.md | 3 ++- cpp/examples/arrow/arrow_ingest.cpp | 5 +++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/cpp/NEXT_CHANGELOG.md b/cpp/NEXT_CHANGELOG.md index a1051e39..19dd253d 100644 --- a/cpp/NEXT_CHANGELOG.md +++ b/cpp/NEXT_CHANGELOG.md @@ -22,6 +22,8 @@ run once per logical ingest submission rather than once per record in a batch. - Recovery after a flush timeout now treats unacked retrieval failure as an active stream rather than assuming the stream is terminal. +- Arrow Flight schema validation is documented at stream creation (the schema IPC + bytes passed to `create_arrow_stream`), not on the first ingested batch. ### Internal Changes diff --git a/cpp/examples/arrow/README.md b/cpp/examples/arrow/README.md index c0b75ae6..e067dcd1 100644 --- a/cpp/examples/arrow/README.md +++ b/cpp/examples/arrow/README.md @@ -94,7 +94,8 @@ stream.close(); - **All-or-nothing per `RecordBatch`** — a batch is acknowledged as a unit. - **Single acknowledgment** — one offset ID for the whole `RecordBatch`. - **Schema validation** — the `RecordBatch` schema must match the schema - configured on the stream. The server validates on the first batch and fails + configured on the stream. The server validates that schema when the stream is + created, from the schema IPC bytes passed to `create_arrow_stream`, and fails fast with a descriptive error on a mismatch. ## IPC Compression diff --git a/cpp/examples/arrow/arrow_ingest.cpp b/cpp/examples/arrow/arrow_ingest.cpp index 9d065545..02d9ba0a 100644 --- a/cpp/examples/arrow/arrow_ingest.cpp +++ b/cpp/examples/arrow/arrow_ingest.cpp @@ -73,8 +73,9 @@ std::int64_t now_micros() { // type. This mirrors the canonical Arrow schema the Databricks Arrow Flight // server derives from a Delta table: Delta STRING -> large_utf8, INT -> int32, // DOUBLE -> float64, TIMESTAMP -> timestamp(microsecond, "UTC"). The server -// validates the record-batch schema on the first batch and fails fast with a -// descriptive error on a mismatch. +// validates the schema when the stream is created, from the schema IPC bytes +// passed to create_arrow_stream, and fails fast with a descriptive error on a +// mismatch. std::shared_ptr orders_schema() { auto utc_micros = arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"); return arrow::schema({ From f62152606e926ea028a410fefe6199e0acbcf5de Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Thu, 13 Aug 2026 17:17:57 +0000 Subject: [PATCH 12/36] Fix Python recovery snippet --- python/README.md | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/python/README.md b/python/README.md index f189a814..548ab73b 100644 --- a/python/README.md +++ b/python/README.md @@ -349,10 +349,11 @@ except ZerobusException as e: ## Handling Stream Failures The SDK automatically handles retries for transient errors. Enqueue, flush, and close -failures all surface as `ZerobusException`. An enqueue failure can leave the stream -active, and both `get_unacked_records()` and `recreate_stream()` require a closed -stream. Close first, then inspect or recreate. `recreate_stream()` re-queues records -that were already accepted; it does not retry a payload that failed to enqueue. +failures all surface as `ZerobusException`. `get_unacked_records()` and +`recreate_stream()` succeed only after the stream has already closed, which a terminal +failure does. An enqueue failure leaves the stream active, so those calls fail; raise +the original error and keep the stream. `recreate_stream()` re-queues records that were +already accepted; it does not retry a payload that failed to enqueue. ```python from zerobus.sdk.shared import ZerobusException @@ -364,18 +365,18 @@ try: except ZerobusException as e: print(f"Ingestion failed: {e}") try: - stream.close() + unacked = list(stream.get_unacked_records()) except ZerobusException: - pass - - unacked = list(stream.get_unacked_records()) + raise print(f"{len(unacked)} previously queued records were unacknowledged.") - - new_stream = sdk.recreate_stream(stream) try: - new_stream.flush() - finally: - new_stream.close() + new_stream = sdk.recreate_stream(stream) + try: + new_stream.flush() + finally: + new_stream.close() + except ZerobusException: + raise e else: stream.close() ``` From f94fb74b9697b7752b92a02c8e08cde7d14d0249 Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Thu, 13 Aug 2026 17:17:58 +0000 Subject: [PATCH 13/36] Fix TypeScript recovery docs --- typescript/README.md | 15 +++++++-------- typescript/examples/json/README.md | 16 +++++++--------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/typescript/README.md b/typescript/README.md index 14f7d85b..6d26df70 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -688,7 +688,7 @@ const descriptorBase64 = loadDescriptorProto({ ## Error Handling -The SDK includes automatic recovery for transient failures (enabled by default with `recovery: true`). For permanent failures, use `recreateStream()` to automatically recover all unacknowledged batches. Always use try/finally blocks to ensure streams are properly closed: +The SDK includes automatic recovery for transient failures (enabled by default with `recovery: true`). `getUnackedBatches()` and `recreateStream()` succeed only after a terminal native-stream failure, which already closes the stream. An enqueue failure leaves the wrapper active, so those calls reject; rethrow the original error. Do not call `stream.close()` before `recreateStream()`, because close releases the native handle. ```typescript let replacement; @@ -698,16 +698,14 @@ try { console.log(`Success: offset ${offset}`); } catch (error) { console.error('Ingestion failed:', error); - - // Recreate only after a terminal stream failure. Enqueue errors leave the - // wrapper active, and close() releases the native handle needed by recreateStream(). try { const unackedBatches = await stream.getUnackedBatches(); console.log(`Batches to recover: ${unackedBatches.length}`); replacement = await sdk.recreateStream(stream); await replacement.flush(); } catch (recoveryError) { - console.error('Recovery skipped or failed:', recoveryError); + console.error('Stream was not terminal or recovery failed:', recoveryError); + throw error; } finally { if (replacement) { await replacement.close(); @@ -788,11 +786,12 @@ This method is the **recommended approach** for recovering from stream failures. **Example:** ```typescript try { - await stream.ingestRecords(batch); + await stream.ingestRecordsOffset(batch); + await stream.flush(); } catch (error) { - // Automatically recreate stream and recover all unacked batches + // recreateStream() rejects unless the native stream already failed closed. const newStream = await sdk.recreateStream(stream); - // Continue ingesting with newStream + await newStream.flush(); } ``` diff --git a/typescript/examples/json/README.md b/typescript/examples/json/README.md index 53568247..285b9125 100644 --- a/typescript/examples/json/README.md +++ b/typescript/examples/json/README.md @@ -110,13 +110,12 @@ JSON Batch Ingestion Example Stream created === Offset-based API (Recommended) === -[Auto-serializing] Batch of 3 records sent with offset ID: 0 -[Auto-serializing] Batch acknowledged with offset ID: 0 -[Pre-serialized] Batch of 3 records sent with offset ID: 1 -[Pre-serialized] Batch acknowledged with offset ID: 1 +[Auto-serializing] Batch of 3 records queued with offset ID: 0 +[Pre-serialized] Batch of 3 records queued with offset ID: 1 -[Large batch] Sending batch of 100 records... -[Large batch] 100 records acknowledged with offset ID: 2 +[Large batch] Queueing batch of 100 records... +[Large batch] 100 records queued with offset ID: 2 +All offset-API batches acknowledged [Empty batch] Returns: null === Future-based API (Deprecated) === @@ -137,9 +136,6 @@ const objectBatch = [ { device_name: 'sensor-003', temp: 24, humidity: 69 } ]; const objectBatchOffset = await stream.ingestRecordsOffset(objectBatch); -if (objectBatchOffset !== null) { - await stream.waitForOffset(objectBatchOffset); -} // 2. Pre-serialized: array of JSON strings const stringBatch = [ @@ -147,6 +143,8 @@ const stringBatch = [ JSON.stringify({ device_name: 'sensor-005', temp: 26, humidity: 73 }) ]; const stringBatchOffset = await stream.ingestRecordsOffset(stringBatch); + +await stream.flush(); ``` **Batch semantics:** From 17985386c8579fe571afdb97c8d022c71d0211f9 Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Thu, 13 Aug 2026 17:18:00 +0000 Subject: [PATCH 14/36] Fix Go protobuf flush --- go/examples/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/go/examples/README.md b/go/examples/README.md index c066a85f..3113a908 100644 --- a/go/examples/README.md +++ b/go/examples/README.md @@ -129,6 +129,12 @@ message := &pb.AirQuality{ } data, _ := proto.Marshal(message) offset, err := stream.IngestRecordOffset(data) +if err != nil { + log.Fatal(err) +} +if err := stream.Flush(); err != nil { + log.Fatal(err) +} ``` ### Batch Ingestion From 090413729a4317eeb43369b7c375e91195d5074e Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Thu, 13 Aug 2026 17:18:01 +0000 Subject: [PATCH 15/36] Fix Java source-build docs --- java/README.md | 15 ++++++++------- java/examples/README.md | 8 +++++++- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/java/README.md b/java/README.md index 019cdd5b..e452b821 100644 --- a/java/README.md +++ b/java/README.md @@ -204,7 +204,12 @@ cd zerobus-sdk/java mvn clean package -Dzerobus.skipNativeLibCheck=true ``` -This generates two JAR files in the `target/` directory: +`-Dzerobus.skipNativeLibCheck=true` compiles the Java sources without staging JNI +libraries. The JARs in `target/` do not include native libraries and cannot ingest +until you either install a published artifact from Maven Central, or stage the JNI +libraries under `src/main/resources/native/` and run Maven without that flag. + +A release build (natives staged, no skip flag) generates two JAR files in `target/`: - **Regular JAR**: `zerobus-ingest-sdk-1.3.0.jar` (~12MB, includes native libraries) - Contains only the SDK classes @@ -214,10 +219,6 @@ This generates two JAR files in the `target/` directory: - Contains SDK classes plus all dependencies bundled - Self-contained, easier to deploy -The skip flag is for local Java-only builds. Release builds must stage the JNI -libraries under `src/main/resources/native/` and run Maven without that flag so -the packaged JAR includes native libraries. - **Which JAR to use?** - **Regular JAR**: When using Maven/Gradle (recommended) - **Fat JAR**: For standalone scripts or CLI tools without a build system @@ -1401,7 +1402,7 @@ Builder for creating `StreamConfigurationOptions`. ```java StreamConfigurationOptionsBuilder setMaxInflightRecords(int maxInflightRecords) ``` -Sets the maximum number of unacknowledged records (default: 50000). +Sets the maximum number of unacknowledged records (default: 1000000). ```java StreamConfigurationOptionsBuilder setRecovery(boolean recovery) @@ -1421,7 +1422,7 @@ Sets the delay between recovery attempts in milliseconds (default: 2000). ```java StreamConfigurationOptionsBuilder setRecoveryRetries(int recoveryRetries) ``` -Sets the maximum number of recovery attempts (default: 3). +Sets the maximum number of recovery attempts (default: 4). ```java StreamConfigurationOptionsBuilder setFlushTimeoutMs(int flushTimeoutMs) diff --git a/java/examples/README.md b/java/examples/README.md index 1adc2c25..12bffcd3 100644 --- a/java/examples/README.md +++ b/java/examples/README.md @@ -141,9 +141,15 @@ export DATABRICKS_CLIENT_SECRET="your-client-secret" ### 4. Build the SDK +The example `java` commands below load JNI libraries from the packaged SDK. +`-Dzerobus.skipNativeLibCheck=true` compiles Java sources only and those commands +will fail at native load. Either install the published artifact from Maven Central, +or stage JNI libraries under `src/main/resources/native/` and package without the +skip flag: + ```bash cd .. # Go to SDK root -mvn package -DskipTests -Dzerobus.skipNativeLibCheck=true +mvn package -DskipTests ``` ## Running Examples From adb083b6c8dc8e2535542749b39a93a42efc4609 Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Thu, 13 Aug 2026 17:18:03 +0000 Subject: [PATCH 16/36] Fix FFI snippet errors --- rust/ffi/README.md | 61 +++++++++++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/rust/ffi/README.md b/rust/ffi/README.md index 98e15e56..7bf37dc5 100644 --- a/rust/ffi/README.md +++ b/rust/ffi/README.md @@ -92,35 +92,56 @@ CResult r = {0}; /* init: fetch GET /api/2.1/unity-catalog/tables/{name} and pass its JSON body */ CZerobusProtoSchema *schema = zerobus_proto_schema_from_uc_json(uc_table_json, &r); -/* on error schema == NULL; read r.error_message then zerobus_free_error_message(r.error_message) */ +if (schema == NULL) { + zerobus_free_error_message(r.error_message); + return; +} -uintptr_t dlen; +uintptr_t dlen = 0; const uint8_t *desc = zerobus_proto_schema_descriptor_bytes(schema, &dlen); -CZerobusStream *stream = zerobus_sdk_create_stream(sdk, table_name, desc, dlen, - client_id, client_secret, &opts, &r); +if (desc == NULL) { + zerobus_proto_schema_free(schema); + return; +} + +r = (CResult){0}; +CZerobusStream *stream = zerobus_sdk_create_stream( + sdk, table_name, desc, dlen, client_id, client_secret, &opts, &r); +if (stream == NULL) { + zerobus_free_error_message(r.error_message); + zerobus_proto_schema_free(schema); + return; +} -/* encode, ingest, flush, close, and free every C allocation */ -uint8_t *buf; uintptr_t len; +uint8_t *buf = NULL; +uintptr_t len = 0; +r = (CResult){0}; if (!zerobus_proto_schema_encode_json(schema, record_json, &buf, &len, &r)) { - /* handle r.error_message, then zerobus_free_error_message(r.error_message) */ + zerobus_free_error_message(r.error_message); + zerobus_stream_free(stream); + zerobus_proto_schema_free(schema); + return; } + const uint8_t *records[] = { buf }; const uintptr_t record_lens[] = { len }; -CResult ingest = {0}; -zerobus_stream_ingest_proto_records(stream, records, record_lens, 1, &ingest); -zerobus_free_proto_bytes(buf, len); -if (ingest.error_message) { - zerobus_free_error_message(ingest.error_message); +r = (CResult){0}; +if (zerobus_stream_ingest_proto_records(stream, records, record_lens, 1, &r) < 0) { + zerobus_free_error_message(r.error_message); + zerobus_free_proto_bytes(buf, len); + zerobus_stream_free(stream); + zerobus_proto_schema_free(schema); + return; } -CResult flush = {0}; -zerobus_stream_flush(stream, &flush); -if (flush.error_message) { - zerobus_free_error_message(flush.error_message); +zerobus_free_proto_bytes(buf, len); + +r = (CResult){0}; +if (!zerobus_stream_flush(stream, &r)) { + zerobus_free_error_message(r.error_message); } -CResult close_r = {0}; -zerobus_stream_close(stream, &close_r); -if (close_r.error_message) { - zerobus_free_error_message(close_r.error_message); +r = (CResult){0}; +if (!zerobus_stream_close(stream, &r)) { + zerobus_free_error_message(r.error_message); } zerobus_stream_free(stream); zerobus_sdk_free(sdk); From 181e597d920abbd6c2158be03f274eb719eedba9 Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Thu, 13 Aug 2026 18:31:57 +0000 Subject: [PATCH 17/36] Fix Python flush and callback docs --- python/README.md | 4 ++++ python/examples/README.md | 4 ++++ python/zerobus/_zerobus_core.pyi | 5 ++++- python/zerobus/sdk/shared/config.py | 4 +++- 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/python/README.md b/python/README.md index 548ab73b..59a6b2b7 100644 --- a/python/README.md +++ b/python/README.md @@ -556,6 +556,10 @@ class MyCallback(AckCallback): pass ``` +`close()` waits at most `callback_max_wait_time_ms` (default 5000 ms) for +in-flight callbacks. A callback per queued submission is not guaranteed if that +budget expires. + ### `HeadersProvider` For custom authentication (e.g. custom token providers), implement `HeadersProvider` and pass it to `create_stream()`. Must include both `authorization` and `x-databricks-zerobus-table-name` headers. See [`examples/`](examples/) for implementation details. diff --git a/python/examples/README.md b/python/examples/README.md index 907e3674..6ec75a50 100644 --- a/python/examples/README.md +++ b/python/examples/README.md @@ -82,9 +82,11 @@ table_properties = TableProperties(TABLE_NAME, record_pb2.AirQuality.DESCRIPTOR) # Recommended: ingest_record_offset() then flush() once offset = stream.ingest_record_offset(record) +stream.flush() # Preferred bulk path: ingest_records_offset() then flush() once # batch_offset = stream.ingest_records_offset([record]) +# stream.flush() # Option 2: Pass pre-serialized bytes (client controls serialization) # offset = stream.ingest_record_offset(record.SerializeToString()) @@ -104,9 +106,11 @@ table_properties = TableProperties(TABLE_NAME) # Recommended: ingest_record_offset() then flush() once offset = stream.ingest_record_offset(record_dict) +stream.flush() # Preferred bulk path: ingest_records_offset() then flush() once # batch_offset = stream.ingest_records_offset([record_dict]) +# stream.flush() # Option 2: Pass pre-serialized JSON string (client controls serialization) # offset = stream.ingest_record_offset(json.dumps(record_dict)) diff --git a/python/zerobus/_zerobus_core.pyi b/python/zerobus/_zerobus_core.pyi index ffb71efa..7eed9e65 100644 --- a/python/zerobus/_zerobus_core.pyi +++ b/python/zerobus/_zerobus_core.pyi @@ -49,7 +49,10 @@ class AckCallback: acknowledges or fails. A batch that is accepted by the stream produces one callback, not one callback per record in the batch. Pre-queue validation, size, type, and closed-stream failures raise immediately - and do not generate a callback. + and do not generate a callback. ``close()`` waits at most + ``callback_max_wait_time_ms`` (default 5000) for in-flight callbacks, + so a callback per queued submission is not guaranteed if that budget + expires. Example: class MyCallback(AckCallback): diff --git a/python/zerobus/sdk/shared/config.py b/python/zerobus/sdk/shared/config.py index 243368af..83af7dbb 100644 --- a/python/zerobus/sdk/shared/config.py +++ b/python/zerobus/sdk/shared/config.py @@ -17,7 +17,9 @@ successfully queued logical ingest submission that later acknowledges or fails. A batch that is accepted by the stream produces one callback, not one callback per record in the batch. Pre-queue validation, size, type, and closed-stream -failures raise immediately and do not generate a callback. +failures raise immediately and do not generate a callback. ``close()`` waits at +most ``callback_max_wait_time_ms`` (default 5000) for in-flight callbacks, so a +callback per queued submission is not guaranteed if that budget expires. Example: >>> class MyCallback(AckCallback): From 5d4487ef5b1219df63aff51a4d83188a0c6f2917 Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Thu, 13 Aug 2026 18:31:58 +0000 Subject: [PATCH 18/36] Fix leftover Go docs --- go/CLAUDE.md | 2 +- go/Makefile | 2 +- go/README.md | 9 +++++---- go/arrow_stream.go | 5 ++++- go/build.go | 7 ++++--- go/zerobus.go | 9 +++++---- 6 files changed, 20 insertions(+), 14 deletions(-) diff --git a/go/CLAUDE.md b/go/CLAUDE.md index 61461b96..c3902d20 100644 --- a/go/CLAUDE.md +++ b/go/CLAUDE.md @@ -49,7 +49,7 @@ Run from `go/`: - `make build` — Build Rust FFI lib + Go SDK - `make build-rust` — Build only Rust FFI layer -- `make build-go` — Build only Go SDK (requires pre-built FFI lib) +- `make build-go` — Build Rust FFI (via `build-rust`) then the Go SDK - `make test` — Run tests - `make lint` — go vet + cargo clippy - `make fmt` — gofmt + cargo fmt diff --git a/go/Makefile b/go/Makefile index 0558b863..f7632dd2 100644 --- a/go/Makefile +++ b/go/Makefile @@ -6,7 +6,7 @@ help: @echo "Available targets:" @echo " make build - Build both Rust FFI and Go SDK" @echo " make build-rust - Build only the Rust FFI layer" - @echo " make build-go - Build only the Go SDK" + @echo " make build-go - Build Rust FFI, then the Go SDK" @echo " make clean - Remove build artifacts" @echo " make fmt - Format all code (Go and Rust)" @echo " make fmt-go - Format Go code" diff --git a/go/README.md b/go/README.md index 32e8c72d..35a3e0bb 100644 --- a/go/README.md +++ b/go/README.md @@ -93,7 +93,7 @@ Before using the SDK, you need a Databricks workspace URL, a Delta table, and a go get github.com/databricks/zerobus-sdk/go@latest ``` -> Tagged releases (for example `v1.4.0`) include pre-built Rust libraries for Linux, macOS, and Windows. Consumers do not need Rust or `go generate`. Rust is required only when you build from `@main`, a commit hash, or a local checkout. +> Tagged releases and checkouts that include `go/lib/` archives (including `@main` and commit hashes) do not need Rust or `go generate`. Rebuild the FFI only when you change `rust/ffi` or the archive for your platform is missing. **In your code:** @@ -127,8 +127,9 @@ func main() { # Clone the repository git clone https://github.com/databricks/zerobus-sdk.git cd zerobus-sdk/go -go generate # Builds Rust FFI -make build # Builds everything +# Archives under lib/ are committed. Rebuild the FFI only if you change rust/ffi: +# go generate +make build ``` See [Building from Source](#building-from-source) for more build options and [Community and Contributing](#community-and-contributing) for contribution guidelines. @@ -963,7 +964,7 @@ The test suite includes: 9. **Use Protocol Buffers for Production** - More efficient than JSON for high-volume scenarios 10. **Secure Credentials** - Never hardcode secrets; use environment variables or secret managers 11. **Test Recovery** - Simulate failures to verify your error handling logic -12. **Concurrent ingestion** - One stream can be used from multiple goroutines. Create separate streams when you want independent tables, credentials, or failure isolation. +12. **Concurrent ingestion** - One stream can be used from multiple goroutines for ingest. Wait for those workers to finish before `Close()`, `Flush()`, or recreating the stream; concurrent Close/ingest can race native-handle destruction. Create separate streams when you want independent tables, credentials, or failure isolation. ## Migration Guide diff --git a/go/arrow_stream.go b/go/arrow_stream.go index 307ab85e..76336a9e 100644 --- a/go/arrow_stream.go +++ b/go/arrow_stream.go @@ -204,7 +204,10 @@ func (st *ZerobusArrowStream) Close() error { // GetUnackedBatches returns all unacknowledged batches as Arrow IPC bytes. // Each []byte is a self-contained IPC stream (schema + one RecordBatch) that can -// be re-ingested into a new stream. Only call after the stream has closed or failed. +// be re-ingested into a new stream. +// +// IMPORTANT: Call this on a failed stream before Close(). Close() nils the +// handle and frees native resources, so a later GetUnackedBatches() call fails. func (st *ZerobusArrowStream) GetUnackedBatches() ([][]byte, error) { if st.ptr == nil { return nil, &ZerobusError{Message: "Arrow stream has been closed", IsRetryable: false} diff --git a/go/build.go b/go/build.go index 80f4dec7..5ffe0276 100644 --- a/go/build.go +++ b/go/build.go @@ -10,9 +10,10 @@ import ( "runtime" ) -// This file provides utilities for building the Rust FFI library. -// Users must run: go generate github.com/databricks/zerobus-sdk/go -// before building their application. +// This file provides utilities for rebuilding the Rust FFI library. +// Tagged releases and checkouts that include go/lib/ archives do not need +// `go generate`. Run it, or set ZEROBUS_BUILD_RUST=1, only when rebuilding +// the FFI from source. func init() { if _, exists := os.LookupEnv("ZEROBUS_BUILD_RUST"); !exists { diff --git a/go/zerobus.go b/go/zerobus.go index 13a3be24..9a5aeea3 100644 --- a/go/zerobus.go +++ b/go/zerobus.go @@ -6,13 +6,14 @@ // // # Installation // -// This package is a CGO wrapper around a Rust core. Tagged releases include -// pre-built libraries, so consumers can install with: +// This package is a CGO wrapper around a Rust core. Tagged releases and +// checkouts that include lib/ archives do not need Rust. Consumers can install +// with: // // go get github.com/databricks/zerobus-sdk/go@v1.4.0 // // Prerequisites for consumers: Go 1.21+, CGO enabled, a C compiler. -// Rust and `go generate` are required only when building from source. +// Rust and `go generate` are required only when rebuilding the FFI. // // # Quick Start // @@ -98,7 +99,7 @@ // // Errors are categorized as retryable or non-retryable: // -// ack, err := stream.IngestRecord(data) +// _, err := stream.IngestRecordOffset(data) // if err != nil { // if zbErr, ok := err.(*zerobus.ZerobusError); ok { // if zbErr.Retryable() { From dff48e467a90ccad4ddd50c40d9f68dbbe161c39 Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Thu, 13 Aug 2026 18:32:00 +0000 Subject: [PATCH 19/36] Fail Pure-Go batch on callback error --- purego/examples/json/batch/main.go | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/purego/examples/json/batch/main.go b/purego/examples/json/batch/main.go index 5adf4049..b60ab244 100644 --- a/purego/examples/json/batch/main.go +++ b/purego/examples/json/batch/main.go @@ -28,11 +28,19 @@ import ( ) // ackObserver counts acknowledgements from callback hooks. -type ackObserver struct{ acked atomic.Int64 } +type ackObserver struct { + acked atomic.Int64 + offset atomic.Int64 + failed atomic.Bool +} -func (o *ackObserver) OnAck(offset int64) { o.acked.Add(1) } +func (o *ackObserver) OnAck(offset int64) { + o.offset.Store(offset) + o.acked.Add(1) +} func (o *ackObserver) OnError(offset int64, err error) { + o.failed.Store(true) log.Printf("record at offset %d failed: %v", offset, err) } @@ -79,9 +87,18 @@ func main() { // A batch produces one callback event, not one per record. Callback delivery // can still be running when Close() returns, so wait for it before exit. deadline := time.Now().Add(5 * time.Second) - for obs.acked.Load() < 1 && time.Now().Before(deadline) { + for obs.acked.Load() < 1 && !obs.failed.Load() && time.Now().Before(deadline) { time.Sleep(10 * time.Millisecond) } + if obs.failed.Load() { + log.Fatal("batch callback reported an error") + } + if obs.acked.Load() < 1 { + log.Fatal("timed out waiting for batch callback") + } + if got := obs.offset.Load(); got != batchOffset { + log.Fatalf("callback offset %d != batch offset %d", got, batchOffset) + } if err := stream.Close(); err != nil { log.Fatalf("close: %v", err) From 9045006331429c304b78926135ae1020294e7c1f Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Thu, 13 Aug 2026 18:32:02 +0000 Subject: [PATCH 20/36] Fix Java GenerateProto docs --- java/README.md | 9 +++------ .../databricks/zerobus/StreamConfigurationOptions.java | 1 - 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/java/README.md b/java/README.md index e452b821..0c79898a 100644 --- a/java/README.md +++ b/java/README.md @@ -364,11 +364,6 @@ Running the generation tool will create `src/main/proto/record.proto`: ```protobuf syntax = "proto2"; -package com.example; - -option java_package = "com.example.proto"; -option java_outer_classname = "Record"; - message AirQuality { optional string device_name = 1; optional int32 temp = 2; @@ -376,7 +371,9 @@ message AirQuality { } ``` -After generating the proto file, compile it as shown above: +The tool writes a proto2 message only. It does not emit `package`, `java_package`, +or `java_outer_classname`. Add those yourself if you need a Java package, then +compile as shown above: ```bash protoc --java_out=src/main/java src/main/proto/record.proto ``` diff --git a/java/src/main/java/com/databricks/zerobus/StreamConfigurationOptions.java b/java/src/main/java/com/databricks/zerobus/StreamConfigurationOptions.java index e19a8fa2..1700a4b6 100644 --- a/java/src/main/java/com/databricks/zerobus/StreamConfigurationOptions.java +++ b/java/src/main/java/com/databricks/zerobus/StreamConfigurationOptions.java @@ -290,7 +290,6 @@ public StreamConfigurationOptionsBuilder setRecoveryTimeoutMs(int recoveryTimeou * * @param recoveryBackoffMs the recovery backoff delay in milliseconds * @return this builder for method chaining - * @throws IllegalArgumentException if recoveryBackoffMs is less than 0 */ public StreamConfigurationOptionsBuilder setRecoveryBackoffMs(int recoveryBackoffMs) { this.recoveryBackoffMs = recoveryBackoffMs; From 369a597a0479ee53faf565fa2de42b4dba324280 Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Thu, 13 Aug 2026 18:32:03 +0000 Subject: [PATCH 21/36] Fix Rust batch README flush --- rust/examples/json/README.md | 47 +++++++++++------------------------ rust/examples/proto/README.md | 23 ++++++----------- 2 files changed, 22 insertions(+), 48 deletions(-) diff --git a/rust/examples/json/README.md b/rust/examples/json/README.md index 23f5f15a..86b8d66f 100644 --- a/rust/examples/json/README.md +++ b/rust/examples/json/README.md @@ -133,47 +133,28 @@ Stream closed successfully ```rust use databricks_zerobus_ingest_sdk::{JsonValue, JsonString}; -// 1. Auto-serializing: Vec of wrapped structs -let batch: Vec> = vec![ +let batch1: Vec> = vec![ JsonValue(Order { id: 1, /* ... */ }), JsonValue(Order { id: 2, /* ... */ }), JsonValue(Order { id: 3, /* ... */ }), ]; -if let Some(offset) = stream.ingest_records_offset(batch).await? { - stream.wait_for_offset(offset).await?; -} +stream.ingest_records_offset(batch1).await?; -// 2. Pre-serialized: Vec of wrapped strings -let batch: Vec = vec![ - JsonString(r#"{ - "id": 4 - }"#.to_string()), - JsonString(r#"{ - "id": 5 - }"#.to_string()), - JsonString(r#"{ - "id": 6 - }"#.to_string()), +let batch2: Vec = vec![ + JsonString(r#"{ "id": 4 }"#.to_string()), + JsonString(r#"{ "id": 5 }"#.to_string()), + JsonString(r#"{ "id": 6 }"#.to_string()), ]; -if let Some(offset) = stream.ingest_records_offset(batch).await? { - stream.wait_for_offset(offset).await?; -} +stream.ingest_records_offset(batch2).await?; -// 3. Backward-compatible: Vec of raw strings -let batch: Vec = vec![ - r#"{ - "id": 7 - }"#.to_string(), - r#"{ - "id": 8 - }"#.to_string(), - r#"{ - "id": 9 - }"#.to_string(), +let batch3: Vec = vec![ + r#"{ "id": 7 }"#.to_string(), + r#"{ "id": 8 }"#.to_string(), + r#"{ "id": 9 }"#.to_string(), ]; -if let Some(offset) = stream.ingest_records_offset(batch).await? { - stream.wait_for_offset(offset).await?; -} +stream.ingest_records_offset(batch3).await?; + +stream.flush().await?; ``` **Batch semantics:** diff --git a/rust/examples/proto/README.md b/rust/examples/proto/README.md index a00c9ea6..d2b1e7bd 100644 --- a/rust/examples/proto/README.md +++ b/rust/examples/proto/README.md @@ -158,35 +158,28 @@ Stream closed successfully use databricks_zerobus_ingest_sdk::{ProtoMessage, ProtoBytes}; use prost::Message; -// 1. Auto-encoding: Vec of wrapped messages -let batch: Vec> = vec![ +let batch1: Vec> = vec![ ProtoMessage(TableOrders { id: Some(1), /* ... */ }), ProtoMessage(TableOrders { id: Some(2), /* ... */ }), ProtoMessage(TableOrders { id: Some(3), /* ... */ }), ]; -if let Some(offset) = stream.ingest_records_offset(batch).await? { - stream.wait_for_offset(offset).await?; -} +stream.ingest_records_offset(batch1).await?; -// 2. Pre-encoded: Vec of wrapped bytes -let batch: Vec = vec![ +let batch2: Vec = vec![ ProtoBytes(TableOrders { id: Some(4), /* ... */ }.encode_to_vec()), ProtoBytes(TableOrders { id: Some(5), /* ... */ }.encode_to_vec()), ProtoBytes(TableOrders { id: Some(6), /* ... */ }.encode_to_vec()), ]; -if let Some(offset) = stream.ingest_records_offset(batch).await? { - stream.wait_for_offset(offset).await?; -} +stream.ingest_records_offset(batch2).await?; -// 3. Backward-compatible: Vec of raw bytes -let batch: Vec> = vec![ +let batch3: Vec> = vec![ TableOrders { id: Some(7), /* ... */ }.encode_to_vec(), TableOrders { id: Some(8), /* ... */ }.encode_to_vec(), TableOrders { id: Some(9), /* ... */ }.encode_to_vec(), ]; -if let Some(offset) = stream.ingest_records_offset(batch).await? { - stream.wait_for_offset(offset).await?; -} +stream.ingest_records_offset(batch3).await?; + +stream.flush().await?; ``` **Batch semantics:** From cd637c0e03236bd5b988ddf17ae6aca9770c1b99 Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Thu, 13 Aug 2026 18:32:05 +0000 Subject: [PATCH 22/36] Fix C++ recovery README --- cpp/examples/json/README.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/cpp/examples/json/README.md b/cpp/examples/json/README.md index 2d19f664..45550815 100644 --- a/cpp/examples/json/README.md +++ b/cpp/examples/json/README.md @@ -92,15 +92,23 @@ zerobus::Stream stream = transient disconnects. If a stream fails *terminally*, `flush()`/`close()` throws — and a failed `close()` keeps the handle alive so you can drain whatever was never acknowledged with `get_unacked_records()` and re-ingest it on a fresh -stream. (After a *successful* `close()` the handle is freed, so that call would -throw instead — recovery belongs on the failure path only.) +stream. A flush timeout can leave the stream active; retrieval then throws, and +those records cannot be recovered until the stream has actually closed. After a +*successful* `close()` the handle is freed, so that call would throw instead — +recovery belongs on the failure path only. ```cpp try { stream.flush(); stream.close(); } catch (const zerobus::ZerobusException& e) { - std::vector unacked = stream.get_unacked_records(); + std::vector unacked; + try { + unacked = stream.get_unacked_records(); + } catch (const zerobus::ZerobusException& retrieval) { + // Stream may still be active (for example a flush timeout). + throw; + } zerobus::Stream retry = open_stream(...); for (const auto& record : unacked) { retry.ingest_json_record(record.as_string()); // loop — no per-record wait From be1bf5a33d5f5df9aa9565ce0735780f408743f5 Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Thu, 13 Aug 2026 18:32:06 +0000 Subject: [PATCH 23/36] Fix FFI batch offset docs --- rust/ffi/src/stream.rs | 12 ++++++++---- rust/ffi/zerobus.h | 12 ++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/rust/ffi/src/stream.rs b/rust/ffi/src/stream.rs index c0754e04..160d4bbc 100644 --- a/rust/ffi/src/stream.rs +++ b/rust/ffi/src/stream.rs @@ -1039,7 +1039,8 @@ pub extern "C" fn zerobus_stream_ingest_json_record_async( } /// Ingest a batch of protobuf records -/// Returns the offset of the last record in the batch, or -1 on error +/// Returns the logical offset of the batch submission, or -1 on error. +/// The core assigns one offset to the entire batch, not one per record. /// Returns -2 if batch is empty #[no_mangle] pub extern "C" fn zerobus_stream_ingest_proto_records( @@ -1111,7 +1112,8 @@ pub extern "C" fn zerobus_stream_ingest_proto_records( }) } -/// Ingest a batch of protobuf records on a background task and report the last offset via callback. +/// Ingest a batch of protobuf records on a background task and report the batch +/// submission offset via callback. #[no_mangle] pub extern "C" fn zerobus_stream_ingest_proto_records_async( stream: *mut CZerobusStream, @@ -1189,7 +1191,8 @@ pub extern "C" fn zerobus_stream_ingest_proto_records_async( } /// Ingest a batch of JSON records -/// Returns the offset of the last record in the batch, or -1 on error +/// Returns the logical offset of the batch submission, or -1 on error. +/// The core assigns one offset to the entire batch, not one per record. /// Returns -2 if batch is empty #[no_mangle] pub extern "C" fn zerobus_stream_ingest_json_records( @@ -1259,7 +1262,8 @@ pub extern "C" fn zerobus_stream_ingest_json_records( }) } -/// Ingest a batch of JSON records on a background task and report the last offset via callback. +/// Ingest a batch of JSON records on a background task and report the batch +/// submission offset via callback. #[no_mangle] pub extern "C" fn zerobus_stream_ingest_json_records_async( stream: *mut CZerobusStream, diff --git a/rust/ffi/zerobus.h b/rust/ffi/zerobus.h index a194e9e0..26865f57 100644 --- a/rust/ffi/zerobus.h +++ b/rust/ffi/zerobus.h @@ -631,7 +631,8 @@ bool zerobus_stream_ingest_json_record_async(struct CZerobusStream *stream, /** * Ingest a batch of protobuf records - * Returns the offset of the last record in the batch, or -1 on error + * Returns the logical offset of the batch submission, or -1 on error. + * The core assigns one offset to the entire batch, not one per record. * Returns -2 if batch is empty */ int64_t zerobus_stream_ingest_proto_records(struct CZerobusStream *stream, @@ -641,7 +642,8 @@ int64_t zerobus_stream_ingest_proto_records(struct CZerobusStream *stream, struct CResult *result); /** - * Ingest a batch of protobuf records on a background task and report the last offset via callback. + * Ingest a batch of protobuf records on a background task and report the batch + * submission offset via callback. */ bool zerobus_stream_ingest_proto_records_async(struct CZerobusStream *stream, const uint8_t *const *records, @@ -653,7 +655,8 @@ bool zerobus_stream_ingest_proto_records_async(struct CZerobusStream *stream, /** * Ingest a batch of JSON records - * Returns the offset of the last record in the batch, or -1 on error + * Returns the logical offset of the batch submission, or -1 on error. + * The core assigns one offset to the entire batch, not one per record. * Returns -2 if batch is empty */ int64_t zerobus_stream_ingest_json_records(struct CZerobusStream *stream, @@ -662,7 +665,8 @@ int64_t zerobus_stream_ingest_json_records(struct CZerobusStream *stream, struct CResult *result); /** - * Ingest a batch of JSON records on a background task and report the last offset via callback. + * Ingest a batch of JSON records on a background task and report the batch + * submission offset via callback. */ bool zerobus_stream_ingest_json_records_async(struct CZerobusStream *stream, const char *const *json_records, From 1d85ce546058d2e0cbd11bcfd6af904a1880ad8c Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Thu, 13 Aug 2026 18:32:08 +0000 Subject: [PATCH 24/36] Fail .NET example on partial loss --- dotnet/examples/JsonSingle/Program.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dotnet/examples/JsonSingle/Program.cs b/dotnet/examples/JsonSingle/Program.cs index 515a4530..5b669d4d 100644 --- a/dotnet/examples/JsonSingle/Program.cs +++ b/dotnet/examples/JsonSingle/Program.cs @@ -72,7 +72,8 @@ if (failed > 0) { - Console.WriteLine($"{5 - failed} records flushed; {failed} ingest calls failed."); + throw new InvalidOperationException( + $"{5 - failed} records flushed; {failed} ingest calls failed."); } else { From 1cc28960945d0753011ff8df0ecba8dc4e170635 Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Fri, 14 Aug 2026 08:04:08 +0000 Subject: [PATCH 25/36] Remove Java recreateStream warning --- java/NEXT_CHANGELOG.md | 3 +-- java/README.md | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/java/NEXT_CHANGELOG.md b/java/NEXT_CHANGELOG.md index 07e0f889..967b618b 100644 --- a/java/NEXT_CHANGELOG.md +++ b/java/NEXT_CHANGELOG.md @@ -18,8 +18,7 @@ - Documented that published JARs support Java 8 while source builds need JDK 11, that macOS JNI artifacts are not in the current release set, that `recoveryRetries` defaults to 4, and that `flush()` waits for durability rather - than callback completion. `recreateStream()` is no longer presented as a safe - production recovery path. + than callback completion. - Maven Central snippets no longer tell users to redeclare compile-scope transitives (`protobuf-java`, `slf4j-api`). Stream-builder examples use try-with-resources. GenerateProto docs no longer claim STRUCT support, JAR diff --git a/java/README.md b/java/README.md index 0c79898a..6ffba3b9 100644 --- a/java/README.md +++ b/java/README.md @@ -1597,7 +1597,7 @@ processing threads). - `ingestRecordOffset()` + `waitForOffset()` per record → When a specific record must be confirmed before continuing - `ingestRecord().join()` → Deprecated; prefer the offset-based API 11. **Thread safety**: `ZerobusSdk` and streams are not thread-safe. Synchronize externally if more than one thread uses the same instance. -12. **Recovery**: `recreateStream()` currently requires a closed stream, and a failed `close()` can drop unacked payloads before they are cached. Do not rely on it for production recovery until that is fixed. Prefer inspecting `getUnackedBatches()` only after a successful close. +12. **Recovery**: Use `recreateStream()` on a closed stream to re-ingest unacknowledged records. Inspect `getUnackedBatches()` after close when you need the payloads yourself. ## Community and Contributing From 7392e173b6229eb9a97a2f9c1910234c0105f214 Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Fri, 14 Aug 2026 08:07:17 +0000 Subject: [PATCH 26/36] Fix leftover snippet docs --- java/README.md | 5 +++-- python/README.md | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/java/README.md b/java/README.md index 6ffba3b9..6680a574 100644 --- a/java/README.md +++ b/java/README.md @@ -1574,12 +1574,13 @@ try (ZerobusProtoStream stream = sdk.streamBuilder() for (AirQuality record : records) { stream.ingestRecordOffset(record); } - stream.flush(); // wait for durability; callbacks may still be running until close() + stream.flush(); // wait for durability; callbacks may still be running } ``` Implementations must be thread-safe and lightweight (callbacks run on internal -processing threads). +processing threads). `close()` waits at most five seconds for in-flight +callbacks; remaining callbacks may still be pending when it returns. ## Best Practices diff --git a/python/README.md b/python/README.md index 59a6b2b7..970217e7 100644 --- a/python/README.md +++ b/python/README.md @@ -367,7 +367,7 @@ except ZerobusException as e: try: unacked = list(stream.get_unacked_records()) except ZerobusException: - raise + raise e print(f"{len(unacked)} previously queued records were unacknowledged.") try: new_stream = sdk.recreate_stream(stream) From 39f966d55b507ad7a93074b9f0401bc91198d69f Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Fri, 14 Aug 2026 08:36:03 +0000 Subject: [PATCH 27/36] Fix CI format failures --- cpp/examples/json/single.cpp | 3 ++- go/examples/arrow/go.mod | 2 +- go/examples/json/batch/go.mod | 2 +- go/examples/json/single/go.mod | 2 +- go/examples/proto/batch/go.mod | 2 +- go/examples/proto/go.mod | 2 +- go/examples/proto/single/go.mod | 2 +- go/tests/go.mod | 2 +- .../main/java/com/databricks/zerobus/tools/GenerateProto.java | 4 ++-- 9 files changed, 11 insertions(+), 10 deletions(-) diff --git a/cpp/examples/json/single.cpp b/cpp/examples/json/single.cpp index 4e0ffa5c..1f718141 100644 --- a/cpp/examples/json/single.cpp +++ b/cpp/examples/json/single.cpp @@ -145,7 +145,8 @@ int main() { try { unacked = stream.get_unacked_records(); } catch (const zerobus::ZerobusException& retrieval) { - std::cerr << "Could not inspect unacked records (stream may still be active): " + std::cerr << "Could not inspect unacked records (stream may still be " + "active): " << retrieval.what() << "\n"; return 1; } diff --git a/go/examples/arrow/go.mod b/go/examples/arrow/go.mod index f3111066..da778195 100644 --- a/go/examples/arrow/go.mod +++ b/go/examples/arrow/go.mod @@ -1,6 +1,6 @@ module zerobus-examples/arrow -go 1.22.0 +go 1.24.0 require ( github.com/apache/arrow-go/v18 v18.0.0 diff --git a/go/examples/json/batch/go.mod b/go/examples/json/batch/go.mod index 85dcfcf7..f1904c49 100644 --- a/go/examples/json/batch/go.mod +++ b/go/examples/json/batch/go.mod @@ -1,6 +1,6 @@ module zerobus-examples/json-batch -go 1.21 +go 1.25.3 require github.com/databricks/zerobus-sdk/go v0.1.0 diff --git a/go/examples/json/single/go.mod b/go/examples/json/single/go.mod index b94a3e30..0c6c870d 100644 --- a/go/examples/json/single/go.mod +++ b/go/examples/json/single/go.mod @@ -1,6 +1,6 @@ module zerobus-examples/json-single -go 1.21 +go 1.25.3 require github.com/databricks/zerobus-sdk/go v0.1.0 diff --git a/go/examples/proto/batch/go.mod b/go/examples/proto/batch/go.mod index 1fea6b09..ae5c9e5b 100644 --- a/go/examples/proto/batch/go.mod +++ b/go/examples/proto/batch/go.mod @@ -1,6 +1,6 @@ module zerobus-examples/proto-batch -go 1.21 +go 1.25.3 require ( github.com/databricks/zerobus-sdk/go v0.1.0 diff --git a/go/examples/proto/go.mod b/go/examples/proto/go.mod index d917a560..09f4360e 100644 --- a/go/examples/proto/go.mod +++ b/go/examples/proto/go.mod @@ -1,6 +1,6 @@ module zerobus-examples -go 1.21 +go 1.25.3 require ( google.golang.org/protobuf v1.36.10 diff --git a/go/examples/proto/single/go.mod b/go/examples/proto/single/go.mod index 24838854..f6fade20 100644 --- a/go/examples/proto/single/go.mod +++ b/go/examples/proto/single/go.mod @@ -1,6 +1,6 @@ module zerobus-examples/proto-single -go 1.21 +go 1.25.3 require ( github.com/databricks/zerobus-sdk/go v0.1.0 diff --git a/go/tests/go.mod b/go/tests/go.mod index 4beced83..ebb30171 100644 --- a/go/tests/go.mod +++ b/go/tests/go.mod @@ -1,6 +1,6 @@ module github.com/databricks/zerobus-sdk/go/tests -go 1.22.0 +go 1.24.0 require ( github.com/apache/arrow-go/v18 v18.0.0 diff --git a/java/src/main/java/com/databricks/zerobus/tools/GenerateProto.java b/java/src/main/java/com/databricks/zerobus/tools/GenerateProto.java index c503a469..a3fd803f 100644 --- a/java/src/main/java/com/databricks/zerobus/tools/GenerateProto.java +++ b/java/src/main/java/com/databricks/zerobus/tools/GenerateProto.java @@ -19,8 +19,8 @@ * Generate proto2 file from Unity Catalog table schema. * *

This tool fetches table schema from Unity Catalog and generates a corresponding proto2 - * definition file. It supports scalar Delta types plus ARRAY and MAP, and maps - * them to Protocol Buffer types. STRUCT columns are not generated. + * definition file. It supports scalar Delta types plus ARRAY and MAP, and maps them to Protocol + * Buffer types. STRUCT columns are not generated. * *

Usage: java GenerateProto --uc-endpoint <endpoint> --client-id <id> * --client-secret <secret> --table <catalog.schema.table> --output <output.proto> From cb605238ec221c7f65f1853860cd8ed4965c637f Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Fri, 14 Aug 2026 09:16:48 +0000 Subject: [PATCH 28/36] Address comments --- java/NEXT_CHANGELOG.md | 7 ++- java/README.md | 20 ++++----- java/examples/README.md | 10 ++--- java/examples/arrow/README.md | 6 +-- java/examples/json/README.md | 20 ++++----- java/examples/json/SingleRecordExample.java | 39 +++++------------ java/examples/legacy/README.md | 6 +-- java/examples/proto/README.md | 20 ++++----- java/examples/proto/SingleRecordExample.java | 46 +++++--------------- java/tools/README.md | 12 ++--- purego/NEXT_CHANGELOG.md | 6 ++- purego/examples/json/single/main.go | 12 +++-- typescript/NEXT_CHANGELOG.md | 1 + typescript/README.md | 6 ++- 14 files changed, 83 insertions(+), 128 deletions(-) diff --git a/java/NEXT_CHANGELOG.md b/java/NEXT_CHANGELOG.md index 967b618b..7fb17b27 100644 --- a/java/NEXT_CHANGELOG.md +++ b/java/NEXT_CHANGELOG.md @@ -10,7 +10,7 @@ ### Documentation -- Updated dependency snippets to version 1.3.0 and corrected README and example +- Updated dependency snippets to version 1.4.0 and corrected README and example code for stream cleanup, recreation, unique local variables, and a single durability barrier after queued ingestion. Clarified that acknowledgment callbacks fire once per logical ingest submission, including one callback per @@ -22,8 +22,11 @@ - Maven Central snippets no longer tell users to redeclare compile-scope transitives (`protobuf-java`, `slf4j-api`). Stream-builder examples use try-with-resources. GenerateProto docs no longer claim STRUCT support, JAR - examples use `zerobus-ingest-sdk` 1.3.0, and proto examples generate + examples use `zerobus-ingest-sdk` 1.4.0, and proto examples generate `AirQualityProto.java` with `protoc` instead of treating it as checked in. + Example README snippets and the JSON/proto `SingleRecordExample` programs + queue records or batches and call `flush()` once instead of + `waitForOffset()` after a single ingest. ### Internal Changes diff --git a/java/README.md b/java/README.md index 6680a574..1670f033 100644 --- a/java/README.md +++ b/java/README.md @@ -146,7 +146,7 @@ Add the SDK as a dependency in your `pom.xml`: com.databricks zerobus-ingest-sdk - 1.3.0 + 1.4.0 ``` @@ -155,7 +155,7 @@ Or with Gradle (`build.gradle`): ```groovy dependencies { - implementation 'com.databricks:zerobus-ingest-sdk:1.3.0' + implementation 'com.databricks:zerobus-ingest-sdk:1.4.0' } ``` @@ -178,7 +178,7 @@ If you prefer the self-contained fat JAR with all dependencies included: com.databricks zerobus-ingest-sdk - 1.3.0 + 1.4.0 jar-with-dependencies @@ -188,7 +188,7 @@ Or with Gradle: ```groovy dependencies { - implementation 'com.databricks:zerobus-ingest-sdk:1.3.0:jar-with-dependencies' + implementation 'com.databricks:zerobus-ingest-sdk:1.4.0:jar-with-dependencies' } ``` @@ -211,11 +211,11 @@ libraries under `src/main/resources/native/` and run Maven without that flag. A release build (natives staged, no skip flag) generates two JAR files in `target/`: -- **Regular JAR**: `zerobus-ingest-sdk-1.3.0.jar` (~12MB, includes native libraries) +- **Regular JAR**: `zerobus-ingest-sdk-1.4.0.jar` (~12MB, includes native libraries) - Contains only the SDK classes - Requires all dependencies on the classpath -- **Fat JAR**: `zerobus-ingest-sdk-1.3.0-jar-with-dependencies.jar` (~19MB, includes native libraries + all dependencies) +- **Fat JAR**: `zerobus-ingest-sdk-1.4.0-jar-with-dependencies.jar` (~19MB, includes native libraries + all dependencies) - Contains SDK classes plus all dependencies bundled - Self-contained, easier to deploy @@ -259,7 +259,7 @@ Create `pom.xml`: com.databricks zerobus-ingest-sdk - 1.3.0 + 1.4.0 @@ -322,16 +322,16 @@ The proto generation tool requires the fat JAR (all dependencies included): ```bash # Download from Maven Central -wget https://repo1.maven.org/maven2/com/databricks/zerobus-ingest-sdk/1.3.0/zerobus-ingest-sdk-1.3.0-jar-with-dependencies.jar +wget https://repo1.maven.org/maven2/com/databricks/zerobus-ingest-sdk/1.4.0/zerobus-ingest-sdk-1.4.0-jar-with-dependencies.jar # Or if you built from source, it's in target/ -# cp target/zerobus-ingest-sdk-1.3.0-jar-with-dependencies.jar . +# cp target/zerobus-ingest-sdk-1.4.0-jar-with-dependencies.jar . ``` **Run the tool:** ```bash -java -jar zerobus-ingest-sdk-1.3.0-jar-with-dependencies.jar \ +java -jar zerobus-ingest-sdk-1.4.0-jar-with-dependencies.jar \ --uc-endpoint "https://dbc-a1b2c3d4-e5f6.cloud.databricks.com" \ --client-id "your-service-principal-application-id" \ --client-secret "your-service-principal-secret" \ diff --git a/java/examples/README.md b/java/examples/README.md index 12bffcd3..73bdc886 100644 --- a/java/examples/README.md +++ b/java/examples/README.md @@ -41,10 +41,10 @@ examples/ | `proto/BatchIngestionExample` | `ZerobusProtoStream` | Batch ingestion | | `json/SingleRecordExample` | `ZerobusJsonStream` | Single record ingestion (Object + String) | | `json/BatchIngestionExample` | `ZerobusJsonStream` | Batch ingestion | -| `arrow/ArrowIngestionExample` | `ZerobusArrowStream` | Three streams demonstrating each IPC compression codec (NONE, LZ4_FRAME, ZSTD); 10 batches per stream, waitForOffset + flush + close (Beta) | +| `arrow/ArrowIngestionExample` | `ZerobusArrowStream` | Three streams demonstrating each IPC compression codec (NONE, LZ4_FRAME, ZSTD); 10 batches per stream, then flush + close (Beta) | | `legacy/LegacyStreamExample` | `ZerobusStream` | Legacy Future-based API | -Each example demonstrates: single ingestion + wait, batch ingestion + wait for last, and recreateStream. +Each example demonstrates: queue then flush, batch ingestion, and recreateStream. ## Stream Classes @@ -94,10 +94,8 @@ try (ZerobusArrowStream stream = sdk.streamBuilder() .arrow(schema) .build() .join()) { - Optional offset = stream.ingestBatch(vectorSchemaRoot); - if (offset.isPresent()) { - stream.waitForOffset(offset.get()); - } + stream.ingestBatch(vectorSchemaRoot); + stream.flush(); } ``` diff --git a/java/examples/arrow/README.md b/java/examples/arrow/README.md index 4c3bd03c..03bb1b53 100644 --- a/java/examples/arrow/README.md +++ b/java/examples/arrow/README.md @@ -106,10 +106,8 @@ try (VectorSchemaRoot batch = VectorSchemaRoot.create(schema, allocator)) { // Populate the batch... batch.setRowCount(rowCount); - Optional offset = stream.ingestBatch(batch); - if (offset.isPresent()) { - stream.waitForOffset(offset.get()); - } + stream.ingestBatch(batch); + stream.flush(); } ``` diff --git a/java/examples/json/README.md b/java/examples/json/README.md index 259565a4..276b5a65 100644 --- a/java/examples/json/README.md +++ b/java/examples/json/README.md @@ -75,15 +75,15 @@ try (ZerobusJsonStream stream = sdk.streamBuilder() ```java // Method 1: Raw JSON string -long offset = stream.ingestRecordOffset("{\"device_name\": \"sensor-1\", \"temp\": 25}"); +stream.ingestRecordOffset("{\"device_name\": \"sensor-1\", \"temp\": 25}"); // Method 2: Object with serializer (Gson, Jackson, or custom) Map data = new HashMap<>(); data.put("device_name", "sensor-1"); data.put("temp", 25); -offset = stream.ingestRecordOffset(data, gson::toJson); +stream.ingestRecordOffset(data, gson::toJson); -stream.waitForOffset(offset); +stream.flush(); ``` ### Batch Ingestion @@ -94,15 +94,13 @@ List jsonBatch = Arrays.asList( "{\"device_name\": \"s1\", \"temp\": 20}", "{\"device_name\": \"s2\", \"temp\": 21}" ); -Optional offset = stream.ingestRecordsOffset(jsonBatch); +stream.ingestRecordsOffset(jsonBatch); // Method 2: List of objects with serializer List> objectBatch = ...; -offset = stream.ingestRecordsOffset(objectBatch, gson::toJson); +stream.ingestRecordsOffset(objectBatch, gson::toJson); -if (offset.isPresent()) { - stream.waitForOffset(offset.get()); -} +stream.flush(); ``` ### Getting Unacknowledged Records @@ -120,11 +118,11 @@ List unackedObjects = stream.getUnackedRecords(json -> gson.fromJson(jso ### SingleRecordExample Demonstrates both single-record ingestion methods plus recreateStream: -1. **Object with serializer** - 11 records (1 + 10) -2. **String directly** - 11 records (1 + 10) +1. **Object with serializer** - 10 records, then flush +2. **String directly** - 10 records, then flush 3. **RecreateStream demo** - 3 records -**Total: 25 records** +**Total: 23 records** ### BatchIngestionExample diff --git a/java/examples/json/SingleRecordExample.java b/java/examples/json/SingleRecordExample.java index 987b9196..469faf55 100644 --- a/java/examples/json/SingleRecordExample.java +++ b/java/examples/json/SingleRecordExample.java @@ -48,49 +48,30 @@ public static void main(String[] args) throws Exception { // === Auto-serialized: Object with serializer === System.out.println("Auto-serialized (Object + serializer):"); - Map data1 = new HashMap<>(); - data1.put("device_name", "json-main-single"); - data1.put("temp", 20); - data1.put("humidity", 50); - long offset = stream.ingestRecordOffset(data1, SingleRecordExample::toJson); - stream.waitForOffset(offset); - totalRecords++; - System.out.println(" 1 record ingested and acknowledged (offset: " + offset + ")"); - - // Idiomatic flow: ingest in a loop, then confirm durability once on the last - // offset. Acks are ordered, so waiting on the last offset confirms every prior - // record. - long lastOffset = -1; for (int i = 0; i < 10; i++) { Map data = new HashMap<>(); data.put("device_name", "json-main-loop-" + i); - data.put("temp", 21 + i); - data.put("humidity", 51 + i); - lastOffset = stream.ingestRecordOffset(data, SingleRecordExample::toJson); // returns immediately + data.put("temp", 20 + i); + data.put("humidity", 50 + i); + stream.ingestRecordOffset(data, SingleRecordExample::toJson); totalRecords++; } - stream.waitForOffset(lastOffset); // confirm durability once - System.out.println(" 10 records ingested, last acknowledged (offset: " + lastOffset + ")"); + stream.flush(); + System.out.println(" 10 records queued and flushed"); // === Pre-serialized: Raw JSON string === System.out.println("\nPre-serialized (String):"); - String json1 = "{\"device_name\": \"json-alt-single\", \"temp\": 30, \"humidity\": 60}"; - offset = stream.ingestRecordOffset(json1); - stream.waitForOffset(offset); - totalRecords++; - System.out.println(" 1 record ingested and acknowledged (offset: " + offset + ")"); - for (int i = 0; i < 10; i++) { String json = String.format( "{\"device_name\": \"json-alt-loop-%d\", \"temp\": %d, \"humidity\": %d}", - i, 31 + i, 61 + i + i, 30 + i, 60 + i ); - lastOffset = stream.ingestRecordOffset(json); + stream.ingestRecordOffset(json); totalRecords++; } - stream.waitForOffset(lastOffset); - System.out.println(" 10 records ingested, last acknowledged (offset: " + lastOffset + ")"); + stream.flush(); + System.out.println(" 10 records queued and flushed"); System.out.println("\n=== Complete: " + totalRecords + " records ingested ==="); @@ -130,7 +111,7 @@ public static void main(String[] args) throws Exception { "{\"device_name\": \"json-recreate-%d\", \"temp\": %d, \"humidity\": %d}", i, 40 + i, 70 + i ); - long newOffset = newStream.ingestRecordOffset(json); + newStream.ingestRecordOffset(json); newRecords++; } newStream.flush(); diff --git a/java/examples/legacy/README.md b/java/examples/legacy/README.md index 72d18c69..2ba80642 100644 --- a/java/examples/legacy/README.md +++ b/java/examples/legacy/README.md @@ -88,10 +88,8 @@ try (ZerobusProtoStream stream = sdk.streamBuilder() } stream.flush(); - Optional batchOffset = stream.ingestRecordsOffset(records); - if (batchOffset.isPresent()) { - stream.waitForOffset(batchOffset.get()); - } + stream.ingestRecordsOffset(records); + stream.flush(); } ``` diff --git a/java/examples/proto/README.md b/java/examples/proto/README.md index 287e6b94..fd1fdbad 100644 --- a/java/examples/proto/README.md +++ b/java/examples/proto/README.md @@ -64,13 +64,13 @@ AirQuality record = AirQuality.newBuilder() .setTemp(25) .setHumidity(65) .build(); -long offset = stream.ingestRecordOffset(record); +stream.ingestRecordOffset(record); // Method 2: Pre-encoded bytes byte[] encodedBytes = record.toByteArray(); -offset = stream.ingestRecordOffset(encodedBytes); +stream.ingestRecordOffset(encodedBytes); -stream.waitForOffset(offset); +stream.flush(); ``` ### Batch Ingestion @@ -78,15 +78,13 @@ stream.waitForOffset(offset); ```java // Method 1: List of messages List messages = Arrays.asList(record1, record2, record3); -Optional offset = stream.ingestRecordsOffset(messages); +stream.ingestRecordsOffset(messages); // Method 2: List of pre-encoded bytes List encodedRecords = Arrays.asList(bytes1, bytes2, bytes3); -offset = stream.ingestRecordsOffset(encodedRecords); +stream.ingestRecordsOffset(encodedRecords); -if (offset.isPresent()) { - stream.waitForOffset(offset.get()); -} +stream.flush(); ``` ### Getting Unacknowledged Records @@ -104,11 +102,11 @@ List unackedMessages = stream.getUnackedRecords(AirQuality.parser()) ### SingleRecordExample Demonstrates both single-record ingestion methods plus recreateStream: -1. **Message directly** - 11 records (1 + 10) -2. **Pre-encoded bytes** - 11 records (1 + 10) +1. **Message directly** - 10 records, then flush +2. **Pre-encoded bytes** - 10 records, then flush 3. **RecreateStream demo** - 3 records -**Total: 25 records** +**Total: 23 records** ### BatchIngestionExample diff --git a/java/examples/proto/SingleRecordExample.java b/java/examples/proto/SingleRecordExample.java index b2640c89..93726bbc 100644 --- a/java/examples/proto/SingleRecordExample.java +++ b/java/examples/proto/SingleRecordExample.java @@ -47,56 +47,32 @@ public static void main(String[] args) throws Exception { // === Auto-encoded: Message objects === System.out.println("Auto-encoded (Message):"); - AirQuality record1 = AirQuality.newBuilder() - .setDeviceName("proto-main-single") - .setTemp(20) - .setHumidity(50) - .build(); - long offset = stream.ingestRecordOffset(record1); - stream.waitForOffset(offset); - totalRecords++; - System.out.println(" 1 record ingested and acknowledged (offset: " + offset + ")"); - - // Idiomatic flow: ingest in a loop, then confirm durability once on the last - // offset. Acks are ordered, so waiting on the last offset confirms every prior - // record. - long lastOffset = -1; for (int i = 0; i < 10; i++) { AirQuality record = AirQuality.newBuilder() .setDeviceName("proto-main-loop-" + i) - .setTemp(21 + i) - .setHumidity(51 + i) + .setTemp(20 + i) + .setHumidity(50 + i) .build(); - lastOffset = stream.ingestRecordOffset(record); // returns immediately + stream.ingestRecordOffset(record); totalRecords++; } - stream.waitForOffset(lastOffset); // confirm durability once - System.out.println(" 10 records ingested, last acknowledged (offset: " + lastOffset + ")"); + stream.flush(); + System.out.println(" 10 records queued and flushed"); // === Pre-encoded: byte arrays === System.out.println("\nPre-encoded (byte[]):"); - AirQuality record2 = AirQuality.newBuilder() - .setDeviceName("proto-alt-single") - .setTemp(30) - .setHumidity(60) - .build(); - offset = stream.ingestRecordOffset(record2.toByteArray()); - stream.waitForOffset(offset); - totalRecords++; - System.out.println(" 1 record ingested and acknowledged (offset: " + offset + ")"); - for (int i = 0; i < 10; i++) { AirQuality record = AirQuality.newBuilder() .setDeviceName("proto-alt-loop-" + i) - .setTemp(31 + i) - .setHumidity(61 + i) + .setTemp(30 + i) + .setHumidity(60 + i) .build(); - lastOffset = stream.ingestRecordOffset(record.toByteArray()); + stream.ingestRecordOffset(record.toByteArray()); totalRecords++; } - stream.waitForOffset(lastOffset); - System.out.println(" 10 records ingested, last acknowledged (offset: " + lastOffset + ")"); + stream.flush(); + System.out.println(" 10 records queued and flushed"); System.out.println("\n=== Complete: " + totalRecords + " records ingested ==="); @@ -136,7 +112,7 @@ public static void main(String[] args) throws Exception { .setTemp(40 + i) .setHumidity(70 + i) .build(); - long newOffset = newStream.ingestRecordOffset(record); + newStream.ingestRecordOffset(record); newRecords++; } newStream.flush(); diff --git a/java/tools/README.md b/java/tools/README.md index 30ab2a7c..01b8f460 100644 --- a/java/tools/README.md +++ b/java/tools/README.md @@ -51,7 +51,7 @@ If you have downloaded the SDK JAR without the source code: ```bash # Using the shaded JAR (includes all dependencies) -java -cp zerobus-ingest-sdk-1.3.0-jar-with-dependencies.jar \ +java -cp zerobus-ingest-sdk-1.4.0-jar-with-dependencies.jar \ com.databricks.zerobus.tools.GenerateProto \ --uc-endpoint "https://your-workspace.cloud.databricks.com" \ --client-id "your-client-id" \ @@ -65,7 +65,7 @@ Or, if the JAR has a Main-Class manifest entry (which it does): ```bash # Even simpler - just use -jar flag -java -jar zerobus-ingest-sdk-1.3.0-jar-with-dependencies.jar \ +java -jar zerobus-ingest-sdk-1.4.0-jar-with-dependencies.jar \ --uc-endpoint "https://your-workspace.cloud.databricks.com" \ --client-id "your-client-id" \ --client-secret "your-client-secret" \ @@ -113,7 +113,7 @@ Generate a proto file for a simple table: **From the SDK JAR:** ```bash -java -jar zerobus-ingest-sdk-1.3.0-jar-with-dependencies.jar \ +java -jar zerobus-ingest-sdk-1.4.0-jar-with-dependencies.jar \ --uc-endpoint "https://myworkspace.cloud.databricks.com" \ --client-id "abc123" \ --client-secret "secret123" \ @@ -149,7 +149,7 @@ message users { Specify a custom message name: ```bash -java -jar zerobus-ingest-sdk-1.3.0-jar-with-dependencies.jar \ +java -jar zerobus-ingest-sdk-1.4.0-jar-with-dependencies.jar \ --uc-endpoint "https://myworkspace.cloud.databricks.com" \ --client-id "abc123" \ --client-secret "secret123" \ @@ -163,7 +163,7 @@ java -jar zerobus-ingest-sdk-1.3.0-jar-with-dependencies.jar \ The tool handles complex types like arrays and maps: ```bash -java -jar zerobus-ingest-sdk-1.3.0-jar-with-dependencies.jar \ +java -jar zerobus-ingest-sdk-1.4.0-jar-with-dependencies.jar \ --uc-endpoint "https://myworkspace.cloud.databricks.com" \ --client-id "abc123" \ --client-secret "secret123" \ @@ -245,7 +245,7 @@ Users can run the tool directly from the JAR without needing access to the sourc ```bash # Download the SDK JAR (or build it with mvn package -Dzerobus.skipNativeLibCheck=true) # Then simply run: -java -jar zerobus-ingest-sdk-1.3.0-jar-with-dependencies.jar \ +java -jar zerobus-ingest-sdk-1.4.0-jar-with-dependencies.jar \ --uc-endpoint "..." \ --client-id "..." \ --client-secret "..." \ diff --git a/purego/NEXT_CHANGELOG.md b/purego/NEXT_CHANGELOG.md index aa3f69ae..3494231f 100644 --- a/purego/NEXT_CHANGELOG.md +++ b/purego/NEXT_CHANGELOG.md @@ -8,8 +8,10 @@ ### Documentation -- Flush recovery no longer treats every flush error as terminal. Batch examples - expect one callback per batch and wait for that callback before exit. +- Flush recovery no longer treats every flush error as terminal. The JSON single + example closes the failed stream before `GetUnackedRecords()` and then replays. + Batch examples expect one callback per batch and wait for that callback before + exit. ### Internal Changes diff --git a/purego/examples/json/single/main.go b/purego/examples/json/single/main.go index 7083a660..48d5e924 100644 --- a/purego/examples/json/single/main.go +++ b/purego/examples/json/single/main.go @@ -64,19 +64,17 @@ func main() { } // 4. Flush once, then close. A flush timeout can leave the stream active, so - // only recover when unacked retrieval succeeds. + // close first and then retrieve unacked records for replay. if err := stream.Flush(); err != nil { log.Printf("flush failed: %v", err) + if closeErr := stream.Close(); closeErr != nil { + log.Printf("close: %v", closeErr) + } unacked, unackedErr := stream.GetUnackedRecords() if unackedErr != nil { - log.Printf("stream still active or unacked retrieval failed: %v", unackedErr) - if closeErr := stream.Close(); closeErr != nil { - log.Printf("close: %v", closeErr) - } - return + log.Fatalf("unacked retrieval failed: %v", unackedErr) } if len(unacked) == 0 { - _ = stream.Close() return } recoverUnacked(sdk, cfg, stream, unacked) diff --git a/typescript/NEXT_CHANGELOG.md b/typescript/NEXT_CHANGELOG.md index a2537538..b15ae81e 100644 --- a/typescript/NEXT_CHANGELOG.md +++ b/typescript/NEXT_CHANGELOG.md @@ -36,6 +36,7 @@ - Documented that omitted `descriptorProto` does not select JSON, that the inherited inflight default is 1,000,000, and that `close()` is still required to flush. README `main().catch` handlers now set a non-zero exit code. + The short `recreateStream()` example closes the replacement stream in `finally`. - Binding rustdoc for `maxInflightRequests` now matches that 1,000,000 default. - Clarified the high-throughput ingestion pattern across the README, API reference, JSDoc diff --git a/typescript/README.md b/typescript/README.md index 6d26df70..8bf0a410 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -791,7 +791,11 @@ try { } catch (error) { // recreateStream() rejects unless the native stream already failed closed. const newStream = await sdk.recreateStream(stream); - await newStream.flush(); + try { + await newStream.flush(); + } finally { + await newStream.close(); + } } ``` From dfbbbb91a24760caeeabcfdc2af7761843b69b28 Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Fri, 14 Aug 2026 11:13:59 +0000 Subject: [PATCH 29/36] Fix batch example wording --- cpp/examples/json/batch.cpp | 2 +- purego/examples/json/batch/main.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/examples/json/batch.cpp b/cpp/examples/json/batch.cpp index af28e4ed..510aad96 100644 --- a/cpp/examples/json/batch.cpp +++ b/cpp/examples/json/batch.cpp @@ -136,7 +136,7 @@ int main() { (void)offset; }, [](std::int64_t offset, const std::string& msg) noexcept { - std::cerr << "record at offset " << offset << " failed: " << msg + std::cerr << "batch at offset " << offset << " failed: " << msg << "\n"; }); options.callback_wait_policy = zerobus::CallbackWaitPolicy::forever(); diff --git a/purego/examples/json/batch/main.go b/purego/examples/json/batch/main.go index b60ab244..9cc1736f 100644 --- a/purego/examples/json/batch/main.go +++ b/purego/examples/json/batch/main.go @@ -1,7 +1,7 @@ // Batch JSON ingestion example. // -// Uses IngestRecordsOffset and waits on the batch offset. -// Also demonstrates async acks with WithAckCallback. +// Uses IngestRecordsOffset, flushes once, then waits for the batch's single +// ack callback. // // Set these environment variables before running: // @@ -41,7 +41,7 @@ func (o *ackObserver) OnAck(offset int64) { func (o *ackObserver) OnError(offset int64, err error) { o.failed.Store(true) - log.Printf("record at offset %d failed: %v", offset, err) + log.Printf("batch at offset %d failed: %v", offset, err) } func main() { From 2f07b450c1716025aec045a383a65ca6dc239bb4 Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Fri, 14 Aug 2026 12:21:32 +0000 Subject: [PATCH 30/36] Fix Go durability snippets --- go/NEXT_CHANGELOG.md | 4 ++-- go/README.md | 11 ++++++++++- go/zerobus.go | 14 +++++++++++++- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/go/NEXT_CHANGELOG.md b/go/NEXT_CHANGELOG.md index 521dc2f0..3c0fc07e 100644 --- a/go/NEXT_CHANGELOG.md +++ b/go/NEXT_CHANGELOG.md @@ -17,8 +17,8 @@ used from multiple goroutines. - Added `Flush()` to copyable example snippets. - Batch examples name the offset returned by `IngestRecordsOffset` `batchOffset`. - JSON and protobuf example modules declare `go 1.21` to match the SDK minimum. - The Arrow example and tests declare `go 1.22.0`, which is what `arrow-go` v18 requires. + Example and test modules keep the CI toolchain Go versions; the documented + SDK minimum remains Go 1.21+. ### Internal Changes diff --git a/go/README.md b/go/README.md index 35a3e0bb..6683a6d1 100644 --- a/go/README.md +++ b/go/README.md @@ -399,7 +399,13 @@ func example(sdk *zerobus.ZerobusSdk, tableProps zerobus.TableProperties) error } defer stream.Close() - offset, _ := stream.IngestRecordOffset(`{"data": "value"}`) + offset, err := stream.IngestRecordOffset(`{"data": "value"}`) + if err != nil { + return err + } + if err := stream.Flush(); err != nil { + return err + } log.Printf("Ingested at offset: %d", offset) return nil } @@ -1028,6 +1034,9 @@ for i := 0; i < 100; i++ { }(myRecord) } wg.Wait() +if err := stream.Flush(); err != nil { + log.Fatal(err) +} ``` ## API Reference diff --git a/go/zerobus.go b/go/zerobus.go index 9a5aeea3..37580202 100644 --- a/go/zerobus.go +++ b/go/zerobus.go @@ -533,6 +533,9 @@ func (st *ZerobusStream) IngestRecordOffset(payload interface{}) (int64, error) // if err != nil { // log.Fatal(err) // } +// if err := stream.Flush(); err != nil { +// log.Fatal(err) +// } func (st *ZerobusStream) IngestRecordNowait(payload interface{}) error { if st.ptr == nil { return &ZerobusError{Message: "Stream has been closed", IsRetryable: false} @@ -569,6 +572,12 @@ func (st *ZerobusStream) IngestRecordNowait(payload interface{}) error { // `{"field": "value1"}`, // `{"field": "value2"}`, // }) +// if err != nil { +// log.Fatal(err) +// } +// if err := stream.Flush(); err != nil { +// log.Fatal(err) +// } func (st *ZerobusStream) IngestRecordsNowait(records []interface{}) error { if st.ptr == nil { return &ZerobusError{Message: "Stream has been closed", IsRetryable: false} @@ -640,6 +649,9 @@ func (st *ZerobusStream) IngestRecordsNowait(records []interface{}) error { // if err != nil { // log.Fatal(err) // } +// if err := stream.Flush(); err != nil { +// log.Fatal(err) +// } // log.Printf("Batch ingested with offset: %d", batchOffset) func (st *ZerobusStream) IngestRecordsOffset(records []interface{}) (int64, error) { if st.ptr == nil { @@ -728,7 +740,7 @@ func (st *ZerobusStream) WaitForOffset(offset int64) error { // // Use this method to: // - Retrieve unacknowledged records after stream failure for retry logic -// - Check which records weren't durably written after Close() fails +// - Inspect payloads that were not durably written after a failed Flush() // - Implement custom retry strategies after stream errors // // Returns a slice where each element is either: From f7ba22c1c1def10295be52b9082d1cfdeb42b78e Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Fri, 14 Aug 2026 12:21:44 +0000 Subject: [PATCH 31/36] Fix Pure-Go example recovery --- purego/examples/json/batch/main.go | 7 +++++-- purego/examples/json/single/main.go | 10 ++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/purego/examples/json/batch/main.go b/purego/examples/json/batch/main.go index 9cc1736f..e6f7379e 100644 --- a/purego/examples/json/batch/main.go +++ b/purego/examples/json/batch/main.go @@ -93,8 +93,11 @@ func main() { if obs.failed.Load() { log.Fatal("batch callback reported an error") } - if obs.acked.Load() < 1 { - log.Fatal("timed out waiting for batch callback") + if got := obs.acked.Load(); got != 1 { + if got < 1 { + log.Fatal("timed out waiting for batch callback") + } + log.Fatalf("callback observed %d acknowledgements, want 1", got) } if got := obs.offset.Load(); got != batchOffset { log.Fatalf("callback offset %d != batch offset %d", got, batchOffset) diff --git a/purego/examples/json/single/main.go b/purego/examples/json/single/main.go index 48d5e924..a5a0b457 100644 --- a/purego/examples/json/single/main.go +++ b/purego/examples/json/single/main.go @@ -63,16 +63,14 @@ func main() { log.Printf("Record %d queued with offset ID: %d", i+1, offset) } - // 4. Flush once, then close. A flush timeout can leave the stream active, so - // close first and then retrieve unacked records for replay. + // 4. Flush once, then close. A flush timeout leaves the stream active, so + // GetUnackedRecords() fails and Close() would wait another FlushTimeout. + // Leave teardown to sdk.Close(), which terminates without a second flush wait. if err := stream.Flush(); err != nil { log.Printf("flush failed: %v", err) - if closeErr := stream.Close(); closeErr != nil { - log.Printf("close: %v", closeErr) - } unacked, unackedErr := stream.GetUnackedRecords() if unackedErr != nil { - log.Fatalf("unacked retrieval failed: %v", unackedErr) + log.Fatalf("unacked retrieval failed (stream may still be active): %v", unackedErr) } if len(unacked) == 0 { return From e9b96da948c3253c18589d1c5dc75366362bca1c Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Fri, 14 Aug 2026 12:22:04 +0000 Subject: [PATCH 32/36] Fix Java example docs --- java/examples/README.md | 38 ++++++++++++------- java/examples/arrow/README.md | 7 ++-- java/examples/json/README.md | 7 ++-- java/examples/legacy/README.md | 5 ++- java/examples/proto/README.md | 7 ++-- .../com/databricks/zerobus/AckCallback.java | 8 ++-- 6 files changed, 43 insertions(+), 29 deletions(-) diff --git a/java/examples/README.md b/java/examples/README.md index 73bdc886..dc08e8f4 100644 --- a/java/examples/README.md +++ b/java/examples/README.md @@ -61,6 +61,7 @@ try (ZerobusProtoStream stream = sdk.streamBuilder() stream.ingestRecordOffset(preEncodedBytes); // byte[] stream.ingestRecordsOffset(listOfMessages); // batch stream.ingestRecordsOffset(listOfByteArrays); // batch + stream.flush(); } ``` @@ -77,6 +78,7 @@ try (ZerobusJsonStream stream = sdk.streamBuilder() stream.ingestRecordOffset(jsonString); // String stream.ingestRecordsOffset(objects, gson::toJson);// batch stream.ingestRecordsOffset(jsonStrings); // batch + stream.flush(); } ``` @@ -139,39 +141,47 @@ export DATABRICKS_CLIENT_SECRET="your-client-secret" ### 4. Build the SDK -The example `java` commands below load JNI libraries from the packaged SDK. -`-Dzerobus.skipNativeLibCheck=true` compiles Java sources only and those commands -will fail at native load. Either install the published artifact from Maven Central, -or stage JNI libraries under `src/main/resources/native/` and package without the -skip flag: +The example `java` commands below load JNI from the packaged fat JAR, not from +`target/classes`. `-Dzerobus.skipNativeLibCheck=true` compiles Java sources only +and those commands will fail at native load. Either install the published +artifact from Maven Central, or stage JNI libraries under +`src/main/resources/native/` and package without the skip flag: ```bash cd .. # Go to SDK root +# Stage JNI under src/main/resources/native/, then: mvn package -DskipTests ``` +Set `SDK_JAR` to that packaged artifact before compiling or running: + +```bash +SDK_JAR=$(ls ../target/zerobus-ingest-sdk-*-jar-with-dependencies.jar | head -n 1) +``` + ## Running Examples ### Protocol Buffer Examples ```bash cd examples +SDK_JAR=$(ls ../target/zerobus-ingest-sdk-*-jar-with-dependencies.jar | head -n 1) # Generate AirQualityProto.java from the proto schema (not checked in) protoc --java_out=proto proto/air_quality.proto # Compile examples -javac -d . -cp "../target/classes:$(cd .. && mvn dependency:build-classpath -q -DincludeScope=runtime -Dmdep.outputFile=/dev/stdout)" \ +javac -d . -cp "$SDK_JAR" \ proto/com/databricks/zerobus/examples/proto/AirQualityProto.java \ proto/SingleRecordExample.java \ proto/BatchIngestionExample.java # Run single record example -java -cp ".:../target/classes:$(cd .. && mvn dependency:build-classpath -q -DincludeScope=runtime -Dmdep.outputFile=/dev/stdout)" \ +java -cp ".:$SDK_JAR" \ com.databricks.zerobus.examples.proto.SingleRecordExample # Run batch example -java -cp ".:../target/classes:$(cd .. && mvn dependency:build-classpath -q -DincludeScope=runtime -Dmdep.outputFile=/dev/stdout)" \ +java -cp ".:$SDK_JAR" \ com.databricks.zerobus.examples.proto.BatchIngestionExample ``` @@ -179,18 +189,19 @@ java -cp ".:../target/classes:$(cd .. && mvn dependency:build-classpath -q -Dinc ```bash cd examples +SDK_JAR=$(ls ../target/zerobus-ingest-sdk-*-jar-with-dependencies.jar | head -n 1) # Compile examples -javac -d . -cp "../target/classes:$(cd .. && mvn dependency:build-classpath -q -DincludeScope=runtime -Dmdep.outputFile=/dev/stdout)" \ +javac -d . -cp "$SDK_JAR" \ json/SingleRecordExample.java \ json/BatchIngestionExample.java # Run single record example -java -cp ".:../target/classes:$(cd .. && mvn dependency:build-classpath -q -DincludeScope=runtime -Dmdep.outputFile=/dev/stdout)" \ +java -cp ".:$SDK_JAR" \ com.databricks.zerobus.examples.json.SingleRecordExample # Run batch example -java -cp ".:../target/classes:$(cd .. && mvn dependency:build-classpath -q -DincludeScope=runtime -Dmdep.outputFile=/dev/stdout)" \ +java -cp ".:$SDK_JAR" \ com.databricks.zerobus.examples.json.BatchIngestionExample ``` @@ -198,17 +209,18 @@ java -cp ".:../target/classes:$(cd .. && mvn dependency:build-classpath -q -Dinc ```bash cd examples +SDK_JAR=$(ls ../target/zerobus-ingest-sdk-*-jar-with-dependencies.jar | head -n 1) # Generate AirQualityProto.java if you have not already (not checked in) protoc --java_out=proto proto/air_quality.proto # Compile -javac -d . -cp "../target/classes:$(cd .. && mvn dependency:build-classpath -q -DincludeScope=runtime -Dmdep.outputFile=/dev/stdout)" \ +javac -d . -cp "$SDK_JAR" \ proto/com/databricks/zerobus/examples/proto/AirQualityProto.java \ legacy/LegacyStreamExample.java # Run legacy example -java -cp ".:../target/classes:$(cd .. && mvn dependency:build-classpath -q -DincludeScope=runtime -Dmdep.outputFile=/dev/stdout)" \ +java -cp ".:$SDK_JAR" \ com.databricks.zerobus.examples.legacy.LegacyStreamExample ``` diff --git a/java/examples/arrow/README.md b/java/examples/arrow/README.md index 03bb1b53..d355db50 100644 --- a/java/examples/arrow/README.md +++ b/java/examples/arrow/README.md @@ -52,10 +52,9 @@ Pass both opens to your application JVM whenever you use `ZerobusArrowStream`: ```bash cd examples - -# Compile (requires Arrow JARs on classpath) +SDK_JAR=$(ls ../target/zerobus-ingest-sdk-*-jar-with-dependencies.jar | head -n 1) ARROW_CP=$(echo ../target/arrow-deps/*.jar | tr ' ' ':') -javac -d . -cp "../target/classes:$ARROW_CP" \ +javac -d . -cp "$SDK_JAR:$ARROW_CP" \ arrow/ArrowIngestionExample.java # Set environment variables @@ -68,7 +67,7 @@ export DATABRICKS_CLIENT_SECRET="your-client-secret" # Run java --add-opens=java.base/java.nio=ALL-UNNAMED \ --add-opens=java.base/java.nio=org.apache.arrow.memory.core \ - -cp ".:../target/zerobus-ingest-sdk-*-jar-with-dependencies.jar:$ARROW_CP" \ + -cp ".:$SDK_JAR:$ARROW_CP" \ com.databricks.zerobus.examples.arrow.ArrowIngestionExample ``` diff --git a/java/examples/json/README.md b/java/examples/json/README.md index 276b5a65..f900e6a7 100644 --- a/java/examples/json/README.md +++ b/java/examples/json/README.md @@ -14,9 +14,10 @@ This directory contains examples for ingesting data using `ZerobusJsonStream`. ```bash cd examples +SDK_JAR=$(ls ../target/zerobus-ingest-sdk-*-jar-with-dependencies.jar | head -n 1) # Compile -javac -d . -cp "../target/classes:$(cd .. && mvn dependency:build-classpath -q -DincludeScope=runtime -Dmdep.outputFile=/dev/stdout)" \ +javac -d . -cp "$SDK_JAR" \ json/SingleRecordExample.java \ json/BatchIngestionExample.java @@ -28,11 +29,11 @@ export DATABRICKS_CLIENT_ID="your-client-id" export DATABRICKS_CLIENT_SECRET="your-client-secret" # Run single record example -java -cp ".:../target/classes:$(cd .. && mvn dependency:build-classpath -q -DincludeScope=runtime -Dmdep.outputFile=/dev/stdout)" \ +java -cp ".:$SDK_JAR" \ com.databricks.zerobus.examples.json.SingleRecordExample # Run batch example -java -cp ".:../target/classes:$(cd .. && mvn dependency:build-classpath -q -DincludeScope=runtime -Dmdep.outputFile=/dev/stdout)" \ +java -cp ".:$SDK_JAR" \ com.databricks.zerobus.examples.json.BatchIngestionExample ``` diff --git a/java/examples/legacy/README.md b/java/examples/legacy/README.md index 2ba80642..de0e5683 100644 --- a/java/examples/legacy/README.md +++ b/java/examples/legacy/README.md @@ -15,12 +15,13 @@ This directory contains examples using the deprecated `ZerobusStream` class. ```bash cd examples +SDK_JAR=$(ls ../target/zerobus-ingest-sdk-*-jar-with-dependencies.jar | head -n 1) # Generate AirQualityProto.java from the proto schema (not checked in) protoc --java_out=proto proto/air_quality.proto # Compile -javac -d . -cp "../target/classes:$(cd .. && mvn dependency:build-classpath -q -DincludeScope=runtime -Dmdep.outputFile=/dev/stdout)" \ +javac -d . -cp "$SDK_JAR" \ proto/com/databricks/zerobus/examples/proto/AirQualityProto.java \ legacy/LegacyStreamExample.java @@ -32,7 +33,7 @@ export DATABRICKS_CLIENT_ID="your-client-id" export DATABRICKS_CLIENT_SECRET="your-client-secret" # Run -java -cp ".:../target/classes:$(cd .. && mvn dependency:build-classpath -q -DincludeScope=runtime -Dmdep.outputFile=/dev/stdout)" \ +java -cp ".:$SDK_JAR" \ com.databricks.zerobus.examples.legacy.LegacyStreamExample ``` diff --git a/java/examples/proto/README.md b/java/examples/proto/README.md index fd1fdbad..35d1f18b 100644 --- a/java/examples/proto/README.md +++ b/java/examples/proto/README.md @@ -14,12 +14,13 @@ This directory contains examples for ingesting data using `ZerobusProtoStream`. ```bash cd examples +SDK_JAR=$(ls ../target/zerobus-ingest-sdk-*-jar-with-dependencies.jar | head -n 1) # Generate AirQualityProto.java from the proto schema (not checked in) protoc --java_out=proto proto/air_quality.proto # Compile -javac -d . -cp "../target/classes:$(cd .. && mvn dependency:build-classpath -q -DincludeScope=runtime -Dmdep.outputFile=/dev/stdout)" \ +javac -d . -cp "$SDK_JAR" \ proto/com/databricks/zerobus/examples/proto/AirQualityProto.java \ proto/SingleRecordExample.java \ proto/BatchIngestionExample.java @@ -32,11 +33,11 @@ export DATABRICKS_CLIENT_ID="your-client-id" export DATABRICKS_CLIENT_SECRET="your-client-secret" # Run single record example -java -cp ".:../target/classes:$(cd .. && mvn dependency:build-classpath -q -DincludeScope=runtime -Dmdep.outputFile=/dev/stdout)" \ +java -cp ".:$SDK_JAR" \ com.databricks.zerobus.examples.proto.SingleRecordExample # Run batch example -java -cp ".:../target/classes:$(cd .. && mvn dependency:build-classpath -q -DincludeScope=runtime -Dmdep.outputFile=/dev/stdout)" \ +java -cp ".:$SDK_JAR" \ com.databricks.zerobus.examples.proto.BatchIngestionExample ``` diff --git a/java/src/main/java/com/databricks/zerobus/AckCallback.java b/java/src/main/java/com/databricks/zerobus/AckCallback.java index 75890661..02255416 100644 --- a/java/src/main/java/com/databricks/zerobus/AckCallback.java +++ b/java/src/main/java/com/databricks/zerobus/AckCallback.java @@ -43,8 +43,8 @@ public interface AckCallback { *

The offset ID represents the durability acknowledgment up to and including this offset. All * records with offset IDs less than or equal to this value have been durably stored. * - *

This method should not throw exceptions. If an exception is thrown, it will be logged but - * will not affect stream operation. + *

This method must not throw. JNI does not clear a pending Java exception from the callback, + * so a thrown exception can poison later callback operations. * * @param offsetId the offset ID that has been acknowledged */ @@ -56,8 +56,8 @@ public interface AckCallback { *

This method is called when the SDK encounters an error that affects a specific submission * offset. The error may be retryable or non-retryable depending on the nature of the failure. * - *

This method should not throw exceptions. If an exception is thrown, it will be logged but - * will not affect stream operation. + *

This method must not throw. JNI does not clear a pending Java exception from the callback, + * so a thrown exception can poison later callback operations. * * @param offsetId the offset ID that encountered an error * @param errorMessage a description of the error that occurred From 2f46a40b41c00b16655725aa7e7cc838564b46b6 Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Fri, 14 Aug 2026 12:22:18 +0000 Subject: [PATCH 33/36] Fix Rust example wording --- rust/examples/json/README.md | 11 +++++------ rust/examples/proto/README.md | 11 +++++------ rust/ffi/README.md | 6 ++++++ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/rust/examples/json/README.md b/rust/examples/json/README.md index 86b8d66f..6215f2dd 100644 --- a/rust/examples/json/README.md +++ b/rust/examples/json/README.md @@ -119,12 +119,11 @@ let stream = sdk **Expected output:** ``` -[Auto-serializing] Batch of 3 records sent with offset ID: 0 -[Auto-serializing] Batch acknowledged with offset ID: 0 -[Pre-serialized] Batch of 3 records sent with offset ID: 1 -[Pre-serialized] Batch acknowledged with offset ID: 1 -[Backward-compatible] Batch of 3 records sent with offset ID: 2 -[Backward-compatible] Batch acknowledged with offset ID: 2 +=== Offset-based API (Recommended) === +[Auto-serializing] Batch of 3 records queued with offset ID: 0 +[Pre-serialized] Batch of 3 records queued with offset ID: 1 +[Backward-compatible] Batch of 3 records queued with offset ID: 2 +All offset-API batches acknowledged Stream closed successfully ``` diff --git a/rust/examples/proto/README.md b/rust/examples/proto/README.md index d2b1e7bd..59af6bd7 100644 --- a/rust/examples/proto/README.md +++ b/rust/examples/proto/README.md @@ -143,12 +143,11 @@ let stream = sdk **Expected output:** ``` -[Auto-encoding] Batch of 3 records sent with offset ID: 0 -[Auto-encoding] Batch acknowledged with offset ID: 0 -[Pre-encoded] Batch of 3 records sent with offset ID: 1 -[Pre-encoded] Batch acknowledged with offset ID: 1 -[Backward-compatible] Batch of 3 records sent with offset ID: 2 -[Backward-compatible] Batch acknowledged with offset ID: 2 +=== Offset-based API (Recommended) === +[Auto-encoding] Batch of 3 records queued with offset ID: 0 +[Pre-encoded] Batch of 3 records queued with offset ID: 1 +[Backward-compatible] Batch of 3 records queued with offset ID: 2 +All offset-API batches acknowledged Stream closed successfully ``` diff --git a/rust/ffi/README.md b/rust/ffi/README.md index 7bf37dc5..dafac5b8 100644 --- a/rust/ffi/README.md +++ b/rust/ffi/README.md @@ -135,17 +135,23 @@ if (zerobus_stream_ingest_proto_records(stream, records, record_lens, 1, &r) < 0 } zerobus_free_proto_bytes(buf, len); +int failed = 0; r = (CResult){0}; if (!zerobus_stream_flush(stream, &r)) { zerobus_free_error_message(r.error_message); + failed = 1; } r = (CResult){0}; if (!zerobus_stream_close(stream, &r)) { zerobus_free_error_message(r.error_message); + failed = 1; } zerobus_stream_free(stream); zerobus_sdk_free(sdk); zerobus_proto_schema_free(schema); +if (failed) { + return; +} ``` Encoding contract: record object keys are matched to column names; unknown keys From 94d0898b6798b8f6061e93f31306e2fdb56c269d Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Fri, 14 Aug 2026 12:22:30 +0000 Subject: [PATCH 34/36] Fix TypeScript example docs --- typescript/NEXT_CHANGELOG.md | 6 ++++-- typescript/README.md | 5 ++++- typescript/examples/README.md | 15 ++++++++++----- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/typescript/NEXT_CHANGELOG.md b/typescript/NEXT_CHANGELOG.md index b15ae81e..ca95ed11 100644 --- a/typescript/NEXT_CHANGELOG.md +++ b/typescript/NEXT_CHANGELOG.md @@ -31,8 +31,6 @@ - Corrected README, example, and JSDoc snippets for CommonJS async entry points, generated Protobuf field names, variable declarations, stream recovery, and custom-header callbacks. -- Documented the `HeadersProvider` shape that `createStream()` actually accepts - (`getHeadersCallback` returning header tuples synchronously). - Documented that omitted `descriptorProto` does not select JSON, that the inherited inflight default is 1,000,000, and that `close()` is still required to flush. README `main().catch` handlers now set a non-zero exit code. @@ -58,3 +56,7 @@ ### Deprecations ### API Changes + +- `HeadersProvider` is the shape `createStream()` actually accepts: + `getHeadersCallback` returning header tuples synchronously. Classes with + async `getHeaders()` are not compatible. diff --git a/typescript/README.md b/typescript/README.md index 8bf0a410..9c704ab3 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -705,7 +705,10 @@ try { await replacement.flush(); } catch (recoveryError) { console.error('Stream was not terminal or recovery failed:', recoveryError); - throw error; + throw new AggregateError( + [error, recoveryError], + 'ingestion and recovery both failed', + ); } finally { if (replacement) { await replacement.close(); diff --git a/typescript/examples/README.md b/typescript/examples/README.md index cfeffb70..f1072219 100644 --- a/typescript/examples/README.md +++ b/typescript/examples/README.md @@ -125,9 +125,13 @@ const stream = await sdk.createStream( ### 5. Ingest and Acknowledge +Queue records, then flush once. Immediate `waitForOffset` after a single ingest is a low-volume confirmation pattern, not the default. + ```typescript -const offset = await stream.ingestRecordOffset(data); -await stream.waitForOffset(offset); +for (const record of records) { + await stream.ingestRecordOffset(record); +} +await stream.flush(); ``` ### 6. Close Stream @@ -150,9 +154,10 @@ Both methods return `Promise`, but the key difference is **when** the pr **Offset-based (Recommended):** ```typescript // Promise resolves immediately with offset (doesn't wait for server ack) -const offset = await stream.ingestRecordOffset(data); -// Do other work, then wait for acknowledgment when needed -await stream.waitForOffset(offset); +for (const record of records) { + await stream.ingestRecordOffset(record); +} +await stream.flush(); ``` **Future-based (Deprecated):** From e1f0e6186c6305c33bc09f21d3bbb43e061e5304 Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Fri, 14 Aug 2026 12:35:47 +0000 Subject: [PATCH 35/36] Address comments --- dotnet/README.md | 4 ++-- go/zerobus.go | 2 +- python/NEXT_CHANGELOG.md | 6 ++++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/dotnet/README.md b/dotnet/README.md index 425d0353..cd021b34 100644 --- a/dotnet/README.md +++ b/dotnet/README.md @@ -41,10 +41,10 @@ stream.Flush(); ## Installation -### NuGet +### NuGet (when published) ```bash -dotnet add package Databricks.Zerobus --version 0.5.1 +dotnet add package Databricks.Zerobus.Ingest.Sdk ``` ### From Source diff --git a/go/zerobus.go b/go/zerobus.go index 37580202..30f5a0ec 100644 --- a/go/zerobus.go +++ b/go/zerobus.go @@ -740,7 +740,7 @@ func (st *ZerobusStream) WaitForOffset(offset int64) error { // // Use this method to: // - Retrieve unacknowledged records after stream failure for retry logic -// - Inspect payloads that were not durably written after a failed Flush() +// - Inspect payloads that were queued but not acked, before Close() // - Implement custom retry strategies after stream errors // // Returns a slice where each element is either: diff --git a/python/NEXT_CHANGELOG.md b/python/NEXT_CHANGELOG.md index 3c9a095b..f1cb6f57 100644 --- a/python/NEXT_CHANGELOG.md +++ b/python/NEXT_CHANGELOG.md @@ -13,8 +13,10 @@ - Corrected README, example, and docstring snippets for record-format selection, exception handling, recovery, iterator return values, custom headers, async contexts, and durability-aware throughput measurement. -- Documented that `get_unacked_records()` and `recreate_stream()` require a closed - stream, and that enqueue failures must be closed before recovery. +- Documented that `get_unacked_records()` and `recreate_stream()` require an already + closed stream. An enqueue failure leaves the stream active and that payload was + never queued, so it is not recovered; close first only to inspect records that + were already accepted. - Removed nowait APIs from featured examples. Those calls spawn detached tasks and are not safely synchronized with `flush()`. Recommend `ingest_records_offset()` plus one `flush()` for bulk ingestion. From 786638c57d18f0edbba296d88e8eab373f13df32 Mon Sep 17 00:00:00 2001 From: teodordelibasic-db Date: Fri, 14 Aug 2026 14:18:59 +0000 Subject: [PATCH 36/36] Final pass Signed-off-by: teodordelibasic-db --- README.md | 2 +- cpp/.gitignore | 1 + cpp/examples/json/README.md | 2 +- cpp/include/zerobus/stream.hpp | 3 +- dotnet/README.md | 26 ++++++--- dotnet/examples/JsonBatch/Program.cs | 8 +-- dotnet/src/Zerobus/Native/NativeInterop.cs | 8 +-- dotnet/src/Zerobus/ZerobusStream.cs | 4 +- go/README.md | 25 ++++----- go/examples/README.md | 11 ++-- go/zerobus.go | 4 +- java/README.md | 39 ++++++++----- java/examples/json/README.md | 2 +- java/examples/legacy/README.md | 2 +- .../zerobus/ZerobusArrowStream.java | 22 ++++---- .../databricks/zerobus/ZerobusJsonStream.java | 37 +++++++------ .../zerobus/ZerobusProtoStream.java | 35 ++++++------ .../com/databricks/zerobus/ZerobusSdk.java | 54 +++++++++--------- .../com/databricks/zerobus/ZerobusStream.java | 19 ++++--- java/tools/README.md | 14 +++-- java/tools/generate_proto.sh | 2 +- purego/NEXT_CHANGELOG.md | 5 +- python/examples/async_example_json.py | 2 +- python/examples/sync_example_json.py | 2 +- python/rust/src/auth.rs | 9 ++- python/zerobus/__init__.py | 4 +- python/zerobus/sdk/aio/zerobus_sdk.py | 7 +-- python/zerobus/sdk/shared/arrow.py | 2 +- python/zerobus/sdk/sync/zerobus_sdk.py | 8 +-- rust/README.md | 2 +- rust/examples/README.md | 2 +- rust/examples/json/README.md | 13 ++--- rust/examples/proto/README.md | 17 +++--- rust/sdk/src/builder/sdk_builder.rs | 4 +- rust/sdk/src/stream/grpc/mod.rs | 18 +++--- typescript/CONTRIBUTING.md | 2 +- typescript/README.md | 2 +- typescript/examples/README.md | 2 +- typescript/examples/proto/README.md | 8 +++ typescript/src/lib.rs | 21 ++++--- typescript/test/unit.test.ts | 55 ++----------------- typescript/tsconfig.json | 2 +- 42 files changed, 251 insertions(+), 256 deletions(-) diff --git a/README.md b/README.md index 489926e2..9396a81d 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ Available in the Rust, Python, Go, TypeScript, and Java SDKs starting from their - Your workload is naturally columnar or batched — analytics pipelines, gateways aggregating short windows of rows, wide/numeric schemas where row-by-row serialization adds noticeable CPU overhead. - Your application already produces Arrow data — pyarrow, the [arrow-rs](https://github.com/apache/arrow-rs) crates, DataFusion, Polars, or other libraries built on Arrow. -For sparse, one-row-at-a-time traffic, JSON or Protocol Buffers over the standard SDK gRPC path are usually simpler. Most SDKs ship a runnable `examples/arrow/` directory; the C++ SDK covers Arrow Flight in its [README](cpp/README.md#arrow-flight-ingestion-beta) until its examples land. +For sparse, one-row-at-a-time traffic, JSON or Protocol Buffers over the standard SDK gRPC path are usually simpler. Most SDKs ship a runnable `examples/arrow/` directory (see each SDK's README for details). ### Acknowledgments and throughput diff --git a/cpp/.gitignore b/cpp/.gitignore index f2d92002..7838174c 100644 --- a/cpp/.gitignore +++ b/cpp/.gitignore @@ -1,6 +1,7 @@ # CMake build trees (see Makefile BUILD_DIR; sanitizer builds use build-*). /build/ /build-*/ +/Testing/ # clangd index / editor caches. /.cache/ diff --git a/cpp/examples/json/README.md b/cpp/examples/json/README.md index 45550815..e4703725 100644 --- a/cpp/examples/json/README.md +++ b/cpp/examples/json/README.md @@ -138,7 +138,7 @@ Each `UnackedRecord` exposes `is_json()`, the raw `data()` bytes, and ``` Batch of 3 records queued; batch offset ID: 0 Batch acknowledged at offset ID: 0 -Stream closed successfully. Callback observed 1 logical submission acknowledgement. +Stream closed successfully. Callback observed 1 logical submission acknowledgement(s). ``` ### Code Highlights diff --git a/cpp/include/zerobus/stream.hpp b/cpp/include/zerobus/stream.hpp index d5d565a9..fd52e9fb 100644 --- a/cpp/include/zerobus/stream.hpp +++ b/cpp/include/zerobus/stream.hpp @@ -105,7 +105,8 @@ class Stream { /// Return all unacknowledged records from a closed or failed stream, for the /// caller to re-ingest on a fresh stream. Remains callable after a failed /// `close()` (which keeps the handle alive precisely so recovery is - /// possible). + /// possible). Calling on an active stream (e.g. after a flush timeout before + /// the stream has closed) throws ZerobusException. /// /// @return The records that were ingested but not acknowledged. /// @throws ZerobusException if the records cannot be retrieved. diff --git a/dotnet/README.md b/dotnet/README.md index cd021b34..5ceaec21 100644 --- a/dotnet/README.md +++ b/dotnet/README.md @@ -219,8 +219,8 @@ string[] records = [ """{"device": "sensor-001", "temp": 20}""", """{"device": "sensor-002", "temp": 21}""", ]; -long lastOffset = stream.IngestRecords(records); -stream.WaitForOffset(lastOffset); +long batchOffset = stream.IngestRecords(records); +stream.WaitForOffset(batchOffset); ``` #### `WaitForOffset` (sync) @@ -241,14 +241,26 @@ stream.Flush(); #### `GetUnackedRecords` -Retrieves unacknowledged records after stream failure (call after close/failure only). +Retrieves unacknowledged records after stream failure (call after close/failure only). A flush timeout can leave the stream active, in which case `GetUnackedRecords()` throws until the stream closes. ```csharp -ReadOnlyMemory[] unacked = stream.GetUnackedRecords(); -foreach (var payload in unacked) +try +{ + stream.Flush(); +} +catch (ZerobusException) { - // Decode as UTF-8 if you know this stream ingests JSON. - Console.WriteLine($"record bytes: {payload.Length}"); + // A flush timeout can leave the stream active. GetUnackedRecords + // requires a closed or failed stream. + try + { + var unacked = stream.GetUnackedRecords(); + Console.WriteLine($"Failed to acknowledge {unacked.Length} records"); + } + catch (ZerobusException retrieval) + { + Console.WriteLine($"Could not inspect unacked records (stream may still be active): {retrieval.Message}"); + } } ``` diff --git a/dotnet/examples/JsonBatch/Program.cs b/dotnet/examples/JsonBatch/Program.cs index 8627f531..2347c003 100644 --- a/dotnet/examples/JsonBatch/Program.cs +++ b/dotnet/examples/JsonBatch/Program.cs @@ -42,11 +42,11 @@ """{"device_name": "sensor-005", "temp": 24, "humidity": 64}""", ]; -long lastOffset = stream.IngestRecords(batchRecords); -Console.WriteLine($"Batch of {batchRecords.Length} records ingested, last offset: {lastOffset}"); +long batchOffset = stream.IngestRecords(batchRecords); +Console.WriteLine($"Batch of {batchRecords.Length} records ingested, batch offset: {batchOffset}"); -// Wait for the last offset to ensure the entire batch is acknowledged. -stream.WaitForOffset(lastOffset); +// Wait for the batch offset to ensure the entire batch is acknowledged. +stream.WaitForOffset(batchOffset); Console.WriteLine("Batch acknowledged!"); Console.WriteLine("All operations completed successfully!"); diff --git a/dotnet/src/Zerobus/Native/NativeInterop.cs b/dotnet/src/Zerobus/Native/NativeInterop.cs index a65982e1..cd60ccf1 100644 --- a/dotnet/src/Zerobus/Native/NativeInterop.cs +++ b/dotnet/src/Zerobus/Native/NativeInterop.cs @@ -484,7 +484,7 @@ public static Task StreamIngestJsonRecordAsync( } ///

- /// Ingests a batch of protobuf records and returns the last offset. + /// Ingests a batch of protobuf records and returns the batch offset (or -1 if empty). /// public static unsafe long StreamIngestProtoRecords(IntPtr streamPtr, byte[][] records) { @@ -544,7 +544,7 @@ public static unsafe long StreamIngestProtoRecords(IntPtr streamPtr, byte[][] re } /// - /// Ingests a batch of protobuf records asynchronously and returns the last offset. + /// Ingests a batch of protobuf records asynchronously and returns the batch offset (or -1 if empty). /// public static Task StreamIngestProtoRecordsAsync( IntPtr streamPtr, @@ -622,7 +622,7 @@ public static Task StreamIngestProtoRecordsAsync( } /// - /// Ingests a batch of JSON records and returns the last offset. + /// Ingests a batch of JSON records and returns the batch offset (or -1 if empty). /// public static unsafe long StreamIngestJsonRecords(IntPtr streamPtr, string[] records) { @@ -680,7 +680,7 @@ public static unsafe long StreamIngestJsonRecords(IntPtr streamPtr, string[] rec } /// - /// Ingests a batch of JSON records asynchronously and returns the last offset. + /// Ingests a batch of JSON records asynchronously and returns the batch offset (or -1 if empty). /// public static Task StreamIngestJsonRecordsAsync( IntPtr streamPtr, diff --git a/dotnet/src/Zerobus/ZerobusStream.cs b/dotnet/src/Zerobus/ZerobusStream.cs index adf4bfec..244501de 100644 --- a/dotnet/src/Zerobus/ZerobusStream.cs +++ b/dotnet/src/Zerobus/ZerobusStream.cs @@ -62,9 +62,7 @@ public bool IsClosed() /// The offset of the ingested record. /// Thrown if ingestion fails. /// Thrown if the stream has been disposed. - /// - /// Thrown if the payload type is not string or byte[]. - /// + /// Thrown if is null. /// /// /// // JSON stream diff --git a/go/README.md b/go/README.md index 6683a6d1..3d8e2493 100644 --- a/go/README.md +++ b/go/README.md @@ -224,11 +224,6 @@ if err != nil { log.Fatal(err) } -descriptor := &descriptorpb.DescriptorProto{} -if err := proto.Unmarshal(descriptorBytes, descriptor); err != nil { - log.Fatal(err) -} - // 2. Create stream for Proto records options := zerobus.DefaultStreamConfigurationOptions() options.RecordType = zerobus.RecordTypeProto @@ -309,7 +304,7 @@ go/ ### Key Components - **Root directory** - The main Go SDK library -- **`zerobus-ffi/`** - Rust FFI wrapper for high-performance ingestion +- **`lib/`** - Pre-built static FFI libraries for supported platforms - **`examples/`** - Complete working examples demonstrating SDK usage - **`Makefile`** - Standard make targets for building, testing, and linting @@ -626,8 +621,7 @@ for partition := 0; partition < 4; partition++ { for i := p * 25000; i < (p+1)*25000; i++ { data := fmt.Sprintf(`{"id": %d}`, i) - offset, err := stream.IngestRecordOffset(data) - if err != nil { + if _, err := stream.IngestRecordOffset(data); err != nil { log.Printf("Failed to ingest: %v", err) continue } @@ -1219,9 +1213,12 @@ Unlike `Flush()` which waits for all pending records, this waits only for a spec **Example:** ```go // Send multiple records -offset1, _ := stream.IngestRecordOffset(`{"id": 1}`) -offset2, _ := stream.IngestRecordOffset(`{"id": 2}`) -offset3, _ := stream.IngestRecordOffset(`{"id": 3}`) +_, _ = stream.IngestRecordOffset(`{"id": 1}`) +_, _ = stream.IngestRecordOffset(`{"id": 2}`) +offset3, err := stream.IngestRecordOffset(`{"id": 3}`) +if err != nil { + log.Fatal(err) +} // Only wait for confirmation that record 3 is durable // (records 1 and 2 will also be durable since offsets are sequential) @@ -1297,7 +1294,9 @@ Waits for the server to acknowledge all records that have been sent. This ensure ```go // Send many records for i := 0; i < 1000; i++ { - stream.IngestRecordOffset(data) + if _, err := stream.IngestRecordOffset(data); err != nil { + log.Fatalf("Ingest failed: %v", err) + } } // Wait for all of them to be confirmed @@ -1449,7 +1448,7 @@ Flushes and closes the stream. func (s *ZerobusArrowStream) GetUnackedBatches() ([][]byte, error) ``` -Returns unacknowledged batches as Arrow IPC bytes. Call only after stream failure. +Returns unacknowledged batches as Arrow IPC bytes. Call on a failed stream before `Close()`. ### `ArrowStreamConfigurationOptions` (Beta) diff --git a/go/examples/README.md b/go/examples/README.md index 3113a908..658936e4 100644 --- a/go/examples/README.md +++ b/go/examples/README.md @@ -110,7 +110,7 @@ go run main.go ```go jsonRecord := `{"device_name": "sensor-001", "temp": 20, "humidity": 60}` -offset, err := stream.IngestRecordOffset(jsonRecord) +_, err := stream.IngestRecordOffset(jsonRecord) if err != nil { log.Fatal(err) } @@ -128,7 +128,7 @@ message := &pb.AirQuality{ Humidity: proto.Int64(60), } data, _ := proto.Marshal(message) -offset, err := stream.IngestRecordOffset(data) +_, err := stream.IngestRecordOffset(data) if err != nil { log.Fatal(err) } @@ -146,7 +146,7 @@ records := []interface{}{ `{"device_name": "sensor-001", "temp": 20, "humidity": 60}`, `{"device_name": "sensor-002", "temp": 21, "humidity": 61}`, } -batchOffset, err := stream.IngestRecordsOffset(records) +_, err := stream.IngestRecordsOffset(records) if err != nil { log.Fatal(err) } @@ -164,7 +164,7 @@ for i := 0; i < 5; i++ { data, _ := proto.Marshal(message) records = append(records, data) } -batchOffset, err := stream.IngestRecordsOffset(records) +_, err := stream.IngestRecordsOffset(records) if err != nil { log.Fatal(err) } @@ -191,6 +191,9 @@ err = stream.IngestRecordsNowait([]interface{}{ `{"device_name": "sensor-001", "temp": 20, "humidity": 60}`, `{"device_name": "sensor-002", "temp": 21, "humidity": 61}`, }) +if err != nil { + log.Fatal(err) +} // Call stream.Flush() or stream.Close() before exiting to ensure durability. ``` diff --git a/go/zerobus.go b/go/zerobus.go index 30f5a0ec..7aef61e0 100644 --- a/go/zerobus.go +++ b/go/zerobus.go @@ -783,7 +783,9 @@ func (st *ZerobusStream) GetUnackedRecords() ([]interface{}, error) { // Example: // // for _, r := range records { -// stream.IngestRecordOffset(r) +// if _, err := stream.IngestRecordOffset(r); err != nil { +// log.Printf("Ingest failed: %v", err) +// } // } // if err := stream.Flush(); err != nil { // log.Printf("Flush failed: %v", err) diff --git a/java/README.md b/java/README.md index 1670f033..3076ba8f 100644 --- a/java/README.md +++ b/java/README.md @@ -590,14 +590,22 @@ Best for production systems with type safety and schema validation: ```bash # Single record ingestion -cd examples/proto -protoc --java_out=. air_quality.proto -javac -d . -cp "../../target/zerobus-ingest-sdk-*-jar-with-dependencies.jar:." *.java -java -cp "../../target/zerobus-ingest-sdk-*-jar-with-dependencies.jar:." \ +cd examples +SDK_JAR=$(ls ../target/zerobus-ingest-sdk-*-jar-with-dependencies.jar | head -n 1) + +# Generate proto class and compile examples +protoc --java_out=proto proto/air_quality.proto +javac -d . -cp "$SDK_JAR" \ + proto/com/databricks/zerobus/examples/proto/AirQualityProto.java \ + proto/SingleRecordExample.java \ + proto/BatchIngestionExample.java + +# Run single record example +java -cp ".:$SDK_JAR" \ com.databricks.zerobus.examples.proto.SingleRecordExample -# Batch ingestion -java -cp "../../target/zerobus-ingest-sdk-*-jar-with-dependencies.jar:." \ +# Run batch example +java -cp ".:$SDK_JAR" \ com.databricks.zerobus.examples.proto.BatchIngestionExample ``` @@ -606,9 +614,12 @@ java -cp "../../target/zerobus-ingest-sdk-*-jar-with-dependencies.jar:." \ Best for rapid prototyping and flexible schemas. No Protocol Buffer types required: ```bash -cd examples/json -javac -d . -cp "../../target/zerobus-ingest-sdk-*-jar-with-dependencies.jar:." *.java -java -cp "../../target/zerobus-ingest-sdk-*-jar-with-dependencies.jar:." \ +cd examples +SDK_JAR=$(ls ../target/zerobus-ingest-sdk-*-jar-with-dependencies.jar | head -n 1) +javac -d . -cp "$SDK_JAR" \ + json/SingleRecordExample.java \ + json/BatchIngestionExample.java +java -cp ".:$SDK_JAR" \ com.databricks.zerobus.examples.json.SingleRecordExample ``` @@ -622,6 +633,7 @@ try (ZerobusJsonStream stream = sdk.streamBuilder() .build() .join()) { stream.ingestRecordOffset("{\"field\": \"value\"}"); + stream.flush(); } ``` @@ -656,10 +668,7 @@ try (ZerobusArrowStream stream = sdk.streamBuilder() .join(); VectorSchemaRoot batch = VectorSchemaRoot.create(schema, allocator)) { // populate batch... - Optional offset = stream.ingestBatch(batch); - if (offset.isPresent()) { - stream.waitForOffset(offset.get()); - } + stream.ingestBatch(batch); stream.flush(); } ``` @@ -777,7 +786,7 @@ try { **Migration:** ```java // Before (deprecated ZerobusStream): wait once after the loop -CompletableFuture last = null; +CompletableFuture last = null; for (AirQuality record : records) { last = stream.ingestRecord(record); } @@ -1000,7 +1009,7 @@ The SDK throws two types of exceptions: ```java try { - stream.ingestRecord(record); + stream.ingestRecordOffset(record); } catch (NonRetriableException e) { // Fatal error - do not retry logger.error("Non-retriable error: " + e.getMessage()); diff --git a/java/examples/json/README.md b/java/examples/json/README.md index f900e6a7..1233dc4b 100644 --- a/java/examples/json/README.md +++ b/java/examples/json/README.md @@ -173,7 +173,7 @@ stream.ingestRecordOffset(myObject, obj -> { | Use Case | Recommended | |----------|-------------| -| Rapid prototyping | SON | +| Rapid prototyping | JSON | | Data already in JSON format | JSON | | Schema changes frequently | JSON | | Production with stable schema | Proto | diff --git a/java/examples/legacy/README.md b/java/examples/legacy/README.md index de0e5683..783d9dad 100644 --- a/java/examples/legacy/README.md +++ b/java/examples/legacy/README.md @@ -108,7 +108,7 @@ try (ZerobusProtoStream stream = sdk.streamBuilder() Demonstrates the Future-based API plus recreateStream: - Creates stream with `TableProperties` -- Ingests 11 records using `ingestRecord().join()` (1 + 10) +- Ingests 11 records using future-based API: 1 single record joined, then 10 records in a loop joining only the last future - Demonstrates `getUnackedRecords()` (returns empty due to type erasure) - Demonstrates `recreateStream()` with 3 additional records diff --git a/java/src/main/java/com/databricks/zerobus/ZerobusArrowStream.java b/java/src/main/java/com/databricks/zerobus/ZerobusArrowStream.java index cc6cc90e..c1a52fab 100644 --- a/java/src/main/java/com/databricks/zerobus/ZerobusArrowStream.java +++ b/java/src/main/java/com/databricks/zerobus/ZerobusArrowStream.java @@ -32,19 +32,17 @@ * Field.nullable("age", new ArrowType.Int(32, true)) * )); * - * ZerobusArrowStream stream = sdk.streamBuilder() - * .table("catalog.schema.table") - * .oauth(clientId, clientSecret) - * .arrow(schema) - * .build() - * .join(); - * - * // Create and populate a VectorSchemaRoot, then ingest - * Optional offset = stream.ingestBatch(batch); - * if (offset.isPresent()) { - * stream.waitForOffset(offset.get()); + * try (ZerobusArrowStream stream = sdk.streamBuilder() + * .table("catalog.schema.table") + * .oauth(clientId, clientSecret) + * .arrow(schema) + * .build() + * .join(); + * VectorSchemaRoot batch = VectorSchemaRoot.create(schema, allocator)) { + * // Create and populate a VectorSchemaRoot, then ingest + * stream.ingestBatch(batch); + * stream.flush(); * } - * stream.close(); * } * *

Resource Management

diff --git a/java/src/main/java/com/databricks/zerobus/ZerobusJsonStream.java b/java/src/main/java/com/databricks/zerobus/ZerobusJsonStream.java index 21354350..0c71ef81 100644 --- a/java/src/main/java/com/databricks/zerobus/ZerobusJsonStream.java +++ b/java/src/main/java/com/databricks/zerobus/ZerobusJsonStream.java @@ -13,26 +13,27 @@ *

Create instances using {@link ZerobusSdk#streamBuilder()}: * * 

{@code
- * ZerobusJsonStream stream = sdk.streamBuilder()
- *     .table("catalog.schema.table")
- *     .oauth(clientId, clientSecret)
- *     .json()
- *     .build()
- *     .join();
+ * try (ZerobusJsonStream stream = sdk.streamBuilder()
+ *         .table("catalog.schema.table")
+ *         .oauth(clientId, clientSecret)
+ *         .json()
+ *         .build()
+ *         .join()) {
+ *     // Main: Ingest objects with a serializer
+ *     Gson gson = new Gson();
+ *     for (MyData record : records) {
+ *         stream.ingestRecordOffset(record, gson::toJson);
+ *     }
+ *     stream.flush();
  *
- * // Main: Ingest objects with a serializer
- * Gson gson = new Gson();
- * long offset = stream.ingestRecordOffset(myObject, gson::toJson);
- * stream.waitForOffset(offset);
+ *     // Alt: Ingest raw JSON strings
+ *     stream.ingestRecordOffset("{\"field\": \"value\"}");
+ *     stream.flush();
  *
- * // Alt: Ingest raw JSON strings
- * stream.ingestRecordOffset("{\"field\": \"value\"}");
- *
- * // Batch ingestion
- * List records = ...;
- * Optional batchOffset = stream.ingestRecordsOffset(records, gson::toJson);
- *
- * stream.close();
+ *     // Batch ingestion
+ *     stream.ingestRecordsOffset(records, gson::toJson);
+ *     stream.flush();
+ * }
  * }
* * @see ZerobusSdk#streamBuilder() diff --git a/java/src/main/java/com/databricks/zerobus/ZerobusProtoStream.java b/java/src/main/java/com/databricks/zerobus/ZerobusProtoStream.java index 1f4bae8c..972989ae 100644 --- a/java/src/main/java/com/databricks/zerobus/ZerobusProtoStream.java +++ b/java/src/main/java/com/databricks/zerobus/ZerobusProtoStream.java @@ -16,25 +16,26 @@ *

Create instances using {@link ZerobusSdk#streamBuilder()}: * * 

{@code
- * ZerobusProtoStream stream = sdk.streamBuilder()
- *     .table("catalog.schema.table")
- *     .oauth(clientId, clientSecret)
- *     .compiledProto(MyProto.getDescriptor().toProto())
- *     .build()
- *     .join();
+ * try (ZerobusProtoStream stream = sdk.streamBuilder()
+ *         .table("catalog.schema.table")
+ *         .oauth(clientId, clientSecret)
+ *         .compiledProto(MyProto.getDescriptor().toProto())
+ *         .build()
+ *         .join()) {
+ *     // Ingest proto messages
+ *     for (MyProto record : records) {
+ *         stream.ingestRecordOffset(record);
+ *     }
+ *     stream.flush();
  *
- * // Ingest proto messages
- * long offset = stream.ingestRecordOffset(myProtoMessage);
- * stream.waitForOffset(offset);
+ *     // Or ingest pre-encoded bytes
+ *     stream.ingestRecordOffset(protoBytes);
+ *     stream.flush();
  *
- * // Or ingest pre-encoded bytes
- * stream.ingestRecordOffset(protoBytes);
- *
- * // Batch ingestion
- * List records = ...;
- * Optional batchOffset = stream.ingestRecordsOffset(records);
- *
- * stream.close();
+ *     // Batch ingestion
+ *     stream.ingestRecordsOffset(records);
+ *     stream.flush();
+ * }
  * }
* * @see ZerobusSdk#streamBuilder() diff --git a/java/src/main/java/com/databricks/zerobus/ZerobusSdk.java b/java/src/main/java/com/databricks/zerobus/ZerobusSdk.java index 6cdf2cdc..c803047f 100644 --- a/java/src/main/java/com/databricks/zerobus/ZerobusSdk.java +++ b/java/src/main/java/com/databricks/zerobus/ZerobusSdk.java @@ -147,16 +147,17 @@ public StreamBuilder streamBuilder() { *

Example usage: * * 

{@code
-   * ZerobusProtoStream stream = sdk.streamBuilder()
-   *     .table("catalog.schema.table")
-   *     .oauth(clientId, clientSecret)
-   *     .compiledProto(MyProto.getDescriptor().toProto())
-   *     .build()
-   *     .join();
-   *
-   * long offset = stream.ingestRecordOffset(myProtoMessage);
-   * stream.waitForOffset(offset);
-   * stream.close();
+   * try (ZerobusProtoStream stream = sdk.streamBuilder()
+   *         .table("catalog.schema.table")
+   *         .oauth(clientId, clientSecret)
+   *         .compiledProto(MyProto.getDescriptor().toProto())
+   *         .build()
+   *         .join()) {
+   *     for (MyProto record : records) {
+   *         stream.ingestRecordOffset(record);
+   *     }
+   *     stream.flush();
+   * }
    * }
* * @param tableName The fully qualified table name (catalog.schema.table). @@ -264,22 +265,23 @@ CompletableFuture createProtoStreamInternal( *

Example usage: * * 

{@code
-   * ZerobusJsonStream stream = sdk.streamBuilder()
-   *     .table("catalog.schema.table")
-   *     .oauth(clientId, clientSecret)
-   *     .json()
-   *     .build()
-   *     .join();
-   *
-   * // Main: Ingest objects with a serializer
-   * Gson gson = new Gson();
-   * long offset = stream.ingestRecordOffset(myObject, gson::toJson);
-   * stream.waitForOffset(offset);
-   *
-   * // Or: Ingest raw JSON strings
-   * stream.ingestRecordOffset("{\"field\": \"value\"}");
-   *
-   * stream.close();
+   * try (ZerobusJsonStream stream = sdk.streamBuilder()
+   *         .table("catalog.schema.table")
+   *         .oauth(clientId, clientSecret)
+   *         .json()
+   *         .build()
+   *         .join()) {
+   *     // Main: Ingest objects with a serializer
+   *     Gson gson = new Gson();
+   *     for (MyData record : records) {
+   *         stream.ingestRecordOffset(record, gson::toJson);
+   *     }
+   *     stream.flush();
+   *
+   *     // Or: Ingest raw JSON strings
+   *     stream.ingestRecordOffset("{\"field\": \"value\"}");
+   *     stream.flush();
+   * }
    * }
* * @param tableName The fully qualified table name (catalog.schema.table). diff --git a/java/src/main/java/com/databricks/zerobus/ZerobusStream.java b/java/src/main/java/com/databricks/zerobus/ZerobusStream.java index ca871189..0d89106c 100644 --- a/java/src/main/java/com/databricks/zerobus/ZerobusStream.java +++ b/java/src/main/java/com/databricks/zerobus/ZerobusStream.java @@ -16,16 +16,19 @@ *

Streams should be created using {@link ZerobusSdk#createStream} and closed when no longer * needed. * - *

Example usage: + *

Example usage (deprecated): * * 

{@code
- * ZerobusStream stream = sdk.createStream(tableProperties, clientId, clientSecret).join();
- *
- * // Ingest a record and wait for acknowledgment
- * stream.ingestRecord(myRecord).join();
- *
- * // Close when done
- * stream.close();
+ * try (ZerobusStream stream =
+ *         sdk.createStream(tableProperties, clientId, clientSecret).join()) {
+ *     CompletableFuture last = null;
+ *     for (MyRecord record : records) {
+ *         last = stream.ingestRecord(record);
+ *     }
+ *     if (last != null) {
+ *         last.join();
+ *     }
+ * }
  * }
* * @param the Protocol Buffer message type for this stream diff --git a/java/tools/README.md b/java/tools/README.md index 01b8f460..d78d5567 100644 --- a/java/tools/README.md +++ b/java/tools/README.md @@ -207,11 +207,15 @@ After generating the `.proto` file: ``` 3. Use the generated Java classes with the Zerobus SDK: ```java - TableProperties tableProperties = - new TableProperties<>("catalog.schema.table", YourMessage.getDefaultInstance()); - - ZerobusStream stream = sdk.createStream( - tableProperties, clientId, clientSecret).join(); + try (ZerobusProtoStream stream = sdk.streamBuilder() + .table("catalog.schema.table") + .oauth(clientId, clientSecret) + .compiledProto(YourMessage.getDescriptor().toProto()) + .build() + .join()) { + stream.ingestRecordOffset(record); + stream.flush(); + } ``` ## Troubleshooting diff --git a/java/tools/generate_proto.sh b/java/tools/generate_proto.sh index 33c7f3aa..b0357f3d 100755 --- a/java/tools/generate_proto.sh +++ b/java/tools/generate_proto.sh @@ -15,7 +15,7 @@ PROJECT_ROOT="${SCRIPT_DIR}/.." TARGET_DIR="${PROJECT_ROOT}/target" # Find the shaded JAR (with dependencies) -SHADED_JAR=$(find "${TARGET_DIR}" -name "databricks-zerobus-ingest-sdk-*-jar-with-dependencies.jar" 2>/dev/null | head -n 1) +SHADED_JAR=$(find "${TARGET_DIR}" -name "*zerobus-ingest-sdk-*-jar-with-dependencies.jar" 2>/dev/null | head -n 1) if [ -z "${SHADED_JAR}" ] || [ ! -f "${SHADED_JAR}" ]; then echo "Error: Zerobus SDK JAR not found in ${TARGET_DIR}" diff --git a/purego/NEXT_CHANGELOG.md b/purego/NEXT_CHANGELOG.md index 3494231f..dab60eba 100644 --- a/purego/NEXT_CHANGELOG.md +++ b/purego/NEXT_CHANGELOG.md @@ -9,8 +9,9 @@ ### Documentation - Flush recovery no longer treats every flush error as terminal. The JSON single - example closes the failed stream before `GetUnackedRecords()` and then replays. - Batch examples expect one callback per batch and wait for that callback before + example retrieves unacknowledged records on flush failure before teardown and + replays them on a fresh stream. The JSON batch example demonstrates that a + batch produces a single ack callback event and waits for that callback before exit. ### Internal Changes diff --git a/python/examples/async_example_json.py b/python/examples/async_example_json.py index f8b11aaf..c96a9648 100644 --- a/python/examples/async_example_json.py +++ b/python/examples/async_example_json.py @@ -123,7 +123,7 @@ def on_ack(self, offset): async def main(): - print("Starting asynchronous ingestion example (Explicit JSON Mode)...") + print("Starting asynchronous ingestion example (JSON)...") print("=" * 60) # Check if credentials are configured diff --git a/python/examples/sync_example_json.py b/python/examples/sync_example_json.py index 2e95de51..d852e430 100644 --- a/python/examples/sync_example_json.py +++ b/python/examples/sync_example_json.py @@ -95,7 +95,7 @@ def get_headers(self): def main(): - print("Starting synchronous ingestion example (Explicit JSON Mode)...") + print("Starting synchronous ingestion example (JSON)...") print("=" * 60) # Check if credentials are configured diff --git a/python/rust/src/auth.rs b/python/rust/src/auth.rs index f616702c..e6bed4a4 100644 --- a/python/rust/src/auth.rs +++ b/python/rust/src/auth.rs @@ -28,9 +28,12 @@ pub struct HeadersProvider {} #[pymethods] impl HeadersProvider { #[new] - #[pyo3(signature = (**_kwargs))] - fn new(_kwargs: Option<&Bound<'_, pyo3::types::PyDict>>) -> Self { - // Accept and ignore kwargs to allow Python subclasses to pass their own arguments + #[pyo3(signature = (*_args, **_kwargs))] + fn new( + _args: &Bound<'_, pyo3::types::PyTuple>, + _kwargs: Option<&Bound<'_, pyo3::types::PyDict>>, + ) -> Self { + // Accept and ignore any arguments to allow Python subclasses to pass their own arguments Self {} } diff --git a/python/zerobus/__init__.py b/python/zerobus/__init__.py index fff1823f..24eff233 100644 --- a/python/zerobus/__init__.py +++ b/python/zerobus/__init__.py @@ -20,9 +20,9 @@ >>> >>> props = TableProperties("catalog.schema.table") >>> stream = sdk.create_stream( - ... table_properties=props, ... client_id="your-client-id", - ... client_secret="your-client-secret" + ... client_secret="your-client-secret", + ... table_properties=props ... ) >>> >>> # New optimized API diff --git a/python/zerobus/sdk/aio/zerobus_sdk.py b/python/zerobus/sdk/aio/zerobus_sdk.py index 077f7307..5ee3ac9b 100644 --- a/python/zerobus/sdk/aio/zerobus_sdk.py +++ b/python/zerobus/sdk/aio/zerobus_sdk.py @@ -18,9 +18,9 @@ ... ... props = TableProperties("catalog.schema.table") ... stream = await sdk.create_stream( - ... table_properties=props, ... client_id="your-client-id", - ... client_secret="your-client-secret" + ... client_secret="your-client-secret", + ... table_properties=props ... ) ... ... # Optimized async API - returns offset directly @@ -64,8 +64,7 @@ def __init__(self, rust_stream: _RustZerobusStream): self._inner = rust_stream async def ingest_record(self, payload: Any): - """ - Ingest a single record and return a future for acknowledgment. + """Ingest a single record (deprecated - use ingest_record_offset). This method uses a two-stage await pattern for optimal performance: - First await (this method): Submits the record and returns quickly with a future diff --git a/python/zerobus/sdk/shared/arrow.py b/python/zerobus/sdk/shared/arrow.py index b13576b5..11ca6a2f 100644 --- a/python/zerobus/sdk/shared/arrow.py +++ b/python/zerobus/sdk/shared/arrow.py @@ -19,7 +19,7 @@ ... ) >>> batch = pa.record_batch({"device_name": ["s1"], "temp": [22]}, schema=schema) >>> offset = stream.ingest_batch(batch) - >>> stream.wait_for_offset(offset) + >>> stream.flush() >>> stream.close() """ diff --git a/python/zerobus/sdk/sync/zerobus_sdk.py b/python/zerobus/sdk/sync/zerobus_sdk.py index 9f142258..1343a1b5 100644 --- a/python/zerobus/sdk/sync/zerobus_sdk.py +++ b/python/zerobus/sdk/sync/zerobus_sdk.py @@ -16,9 +16,9 @@ >>> >>> props = TableProperties("catalog.schema.table") >>> stream = sdk.create_stream( - ... table_properties=props, ... client_id="your-client-id", - ... client_secret="your-client-secret" + ... client_secret="your-client-secret", + ... table_properties=props ... ) >>> >>> # Optimized API - returns offset directly @@ -121,7 +121,7 @@ def get_unacked_records(self) -> Iterator[bytes]: records = self._inner.get_unacked_records() return iter(records) - def get_unacked_batches(self) -> Iterator[list]: + def get_unacked_batches(self) -> Iterator[list[bytes]]: """ Get iterator of unacknowledged batches. @@ -154,7 +154,7 @@ class ZerobusArrowStream: >>> stream = sdk.create_arrow_stream("catalog.schema.table", schema, client_id, client_secret) >>> batch = pa.record_batch({"temp": [22, 23]}, schema=schema) >>> offset = stream.ingest_batch(batch) - >>> stream.wait_for_offset(offset) + >>> stream.flush() >>> stream.close() """ diff --git a/rust/README.md b/rust/README.md index 3b4c4172..bf291984 100644 --- a/rust/README.md +++ b/rust/README.md @@ -998,7 +998,7 @@ match stream.ingest_record_offset(payload).await { ### Complete Working Examples -The `examples/` directory contains four working examples covering different serialization formats and ingestion patterns: +The `examples/` directory contains working examples covering different serialization formats and ingestion patterns: | Example | Serialization | Ingestion | Run with | |---------|--------------|-----------|----------| diff --git a/rust/examples/README.md b/rust/examples/README.md index 5c796f57..e2ea92a5 100644 --- a/rust/examples/README.md +++ b/rust/examples/README.md @@ -205,7 +205,7 @@ stream.flush().await?; **Single-record:** ```rust for record in records { - let offset = stream.ingest_record_offset(record).await?; + stream.ingest_record_offset(record).await?; } stream.flush().await?; ``` diff --git a/rust/examples/json/README.md b/rust/examples/json/README.md index 6215f2dd..0aaec5c1 100644 --- a/rust/examples/json/README.md +++ b/rust/examples/json/README.md @@ -55,12 +55,11 @@ The SDK supports three approaches for passing JSON data: **Expected output:** ``` -[Auto-serializing] Record sent with offset ID: 0 -[Auto-serializing] Record acknowledged with offset ID: 0 -[Pre-serialized] Record sent with offset ID: 1 -[Pre-serialized] Record acknowledged with offset ID: 1 -[Backward-compatible] Record sent with offset ID: 2 -[Backward-compatible] Record acknowledged with offset ID: 2 +=== Offset-based API (Recommended) === +[Auto-serializing] Record queued with offset ID: 0 +[Pre-serialized] Record queued with offset ID: 1 +[Backward-compatible] Record queued with offset ID: 2 +All records acknowledged Stream closed successfully ``` @@ -186,4 +185,4 @@ let json = r#"{ }"#.to_string(); ``` -**3. Update table name and credentials** in the constants at the top of `main.rs`. +**3. Update table name and credentials** in the constants at the top of `single.rs` or `batch.rs`. diff --git a/rust/examples/proto/README.md b/rust/examples/proto/README.md index 59af6bd7..5d75ca81 100644 --- a/rust/examples/proto/README.md +++ b/rust/examples/proto/README.md @@ -19,7 +19,7 @@ This directory contains examples demonstrating Protocol Buffers-based data inges - [Dynamic Batch](#dynamic-batch) - [Adapting for Your Custom Table](#adapting-for-your-custom-table) - [Generate Schema Files](#generate-schema-files) - - [Update main.rs](#update-mainrs) + - [Update Example Files](#update-example-files) ## Overview @@ -72,12 +72,11 @@ The SDK supports three approaches for passing Protocol Buffers data: **Expected output:** ``` -[Auto-encoding] Record sent with offset ID: 0 -[Auto-encoding] Record acknowledged with offset ID: 0 -[Pre-encoded] Record sent with offset ID: 1 -[Pre-encoded] Record acknowledged with offset ID: 1 -[Backward-compatible] Record sent with offset ID: 2 -[Backward-compatible] Record acknowledged with offset ID: 2 +=== Offset-based API (Recommended) === +[Auto-encoding] Record queued with offset ID: 0 +[Pre-encoded] Record queued with offset ID: 1 +[Backward-compatible] Record queued with offset ID: 2 +All records acknowledged Stream closed successfully ``` @@ -298,7 +297,7 @@ This generates: - `output/.rs` - Rust structs with serialization code - `output/.descriptor` - Binary descriptor for runtime validation -### Update main.rs +### Update Example Files **1. Update the module and use statements:** @@ -354,4 +353,4 @@ ProtoMessage(TableInventory { }) ``` -**4. Update table name and credentials** in the constants at the top of `main.rs`. +**4. Update table name and credentials** in the constants at the top of `single.rs` or `batch.rs`. diff --git a/rust/sdk/src/builder/sdk_builder.rs b/rust/sdk/src/builder/sdk_builder.rs index 955d5824..9391894e 100644 --- a/rust/sdk/src/builder/sdk_builder.rs +++ b/rust/sdk/src/builder/sdk_builder.rs @@ -66,8 +66,8 @@ impl ZerobusSdkBuilder { /// Sets the Unity Catalog endpoint URL. /// - /// This is only required when using OAuth authentication via `create_stream()`. - /// When using `create_stream_with_headers_provider()` with a custom headers + /// This is only required when using OAuth authentication via `StreamBuilder::oauth()`. + /// When using `StreamBuilder::headers_provider()` with a custom headers /// provider, this can be omitted. /// /// # Arguments diff --git a/rust/sdk/src/stream/grpc/mod.rs b/rust/sdk/src/stream/grpc/mod.rs index 9c3cf5cb..8caac1bb 100644 --- a/rust/sdk/src/stream/grpc/mod.rs +++ b/rust/sdk/src/stream/grpc/mod.rs @@ -63,22 +63,22 @@ pub(super) const STREAM_TEARDOWN_DRAIN_TIMEOUT_MS: u64 = 500; /// # Lifecycle /// /// 1. Create a stream via `ZerobusSdk::stream_builder()` -/// 2. Ingest records with `ingest_record_offset()` and `wait_for_offset()` for acknowledgments -/// 3. Optionally call `flush()` to ensure all records are persisted +/// 2. Ingest records in a loop with `ingest_record_offset()` +/// 3. Call `flush()` to confirm all queued records are acknowledged /// 4. Close the stream with `close()` to release resources /// /// # Examples /// /// ```no_run /// # use databricks_zerobus_ingest_sdk::*; -/// # async fn example(mut stream: ZerobusStream, data: Vec) -> Result<(), ZerobusError> { -/// // Ingest a single record -/// let offset = stream.ingest_record_offset(data).await?; -/// println!("Record sent with offset: {}", offset); +/// # async fn example(mut stream: ZerobusStream, records: Vec>) -> Result<(), ZerobusError> { +/// // Ingest records in a loop (queue only) +/// for data in records { +/// stream.ingest_record_offset(data).await?; +/// } /// -/// // Wait for acknowledgment -/// stream.wait_for_offset(offset).await?; -/// println!("Record acknowledged at offset: {}", offset); +/// // Confirm all queued records at once +/// stream.flush().await?; /// /// // Close the stream gracefully /// stream.close().await?; diff --git a/typescript/CONTRIBUTING.md b/typescript/CONTRIBUTING.md index 31fb7548..794a0c46 100644 --- a/typescript/CONTRIBUTING.md +++ b/typescript/CONTRIBUTING.md @@ -17,7 +17,7 @@ This document covers TypeScript-specific development setup and workflow. 1. **Clone the repository:** ```bash git clone https://github.com/databricks/zerobus-sdk.git - cd zerobus-sdk/ts + cd zerobus-sdk/typescript ``` 2. **Install dependencies:** diff --git a/typescript/README.md b/typescript/README.md index 9c704ab3..d7c0a18e 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -114,7 +114,7 @@ async function main(): Promise { const zerobusEndpoint = 'https://.zerobus..cloud.databricks.com'; const workspaceUrl = 'https://.cloud.databricks.com'; // For Azure: -// const zerobusEndpoint = '.zerobus..azuredatabricks.net'; +// const zerobusEndpoint = 'https://.zerobus..azuredatabricks.net'; // const workspaceUrl = 'https://.azuredatabricks.net'; const tableName = 'main.default.air_quality'; diff --git a/typescript/examples/README.md b/typescript/examples/README.md index f1072219..daedbd12 100644 --- a/typescript/examples/README.md +++ b/typescript/examples/README.md @@ -88,7 +88,7 @@ const sdk = new ZerobusSdk(SERVER_ENDPOINT, DATABRICKS_WORKSPACE_URL); **JSON:** ```typescript const tableProperties: TableProperties = { - tableName: TABLE_NAME + tableName: TABLE_NAME, // No descriptor needed for JSON }; ``` diff --git a/typescript/examples/proto/README.md b/typescript/examples/proto/README.md index 6bb039db..86511fcf 100644 --- a/typescript/examples/proto/README.md +++ b/typescript/examples/proto/README.md @@ -144,7 +144,15 @@ const messageBatch = [ AirQuality.create({ deviceName: 'sensor-003', temp: 24, humidity: 69 }) ]; const messageBatchOffset = await stream.ingestRecordsOffset(messageBatch); + +// 2. Pre-encoded: array of Buffers +const bufferBatch = [ + AirQuality.create({ deviceName: 'sensor-004', temp: 25, humidity: 71 }), + AirQuality.create({ deviceName: 'sensor-005', temp: 26, humidity: 73 }), + AirQuality.create({ deviceName: 'sensor-006', temp: 27, humidity: 75 }) +].map(record => Buffer.from(AirQuality.encode(record).finish())); const bufferBatchOffset = await stream.ingestRecordsOffset(bufferBatch); + await stream.flush(); ``` diff --git a/typescript/src/lib.rs b/typescript/src/lib.rs index ddbe2e7f..9f4b8009 100644 --- a/typescript/src/lib.rs +++ b/typescript/src/lib.rs @@ -221,7 +221,8 @@ fn convert_js_to_record_payload(env: &Env, payload: Unknown) -> Result Promise> + /// JavaScript function: () => Array<[string, string]> pub get_headers_callback: JsFunction, } @@ -1739,7 +1737,8 @@ impl ZerobusSdk { /// /// # Arguments /// - /// * `stream` - The failed or closed Arrow stream to recreate + /// * `stream` - The terminally failed Arrow stream to recreate. The TypeScript wrapper + /// must not have been closed because `close()` releases its native handle. /// /// # Returns /// diff --git a/typescript/test/unit.test.ts b/typescript/test/unit.test.ts index af557494..4053ec7c 100644 --- a/typescript/test/unit.test.ts +++ b/typescript/test/unit.test.ts @@ -11,7 +11,7 @@ import { execFileSync } from 'node:child_process'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; -import { ZerobusSdk, RecordType, TableProperties, StreamConfigurationOptions, JsAckCallback } from '../index'; +import { ZerobusSdk, RecordType, TableProperties, StreamConfigurationOptions } from '../index'; import { HeadersProvider } from '../src/headers_provider'; import { loadDescriptorProto } from '../utils/descriptor.js'; @@ -75,23 +75,19 @@ describe('ZerobusSdk', () => { assert.strictEqual(options.recordType, RecordType.Proto); }); - it('should accept new v0.4.0 configuration options', () => { + it('should accept streamPausedMaxWaitTimeMs configuration option', () => { const options: StreamConfigurationOptions = { recordType: RecordType.Json, maxInflightRequests: 100, - callbackMaxWaitTimeMs: 5000, // New in v0.4.0 - streamPausedMaxWaitTimeMs: 3000, // New in v0.4.0 + streamPausedMaxWaitTimeMs: 3000, }; - assert.strictEqual(options.callbackMaxWaitTimeMs, 5000); assert.strictEqual(options.streamPausedMaxWaitTimeMs, 3000); }); - it('should allow undefined for new callback timeout options', () => { + it('should allow undefined for streamPausedMaxWaitTimeMs', () => { const options: StreamConfigurationOptions = { recordType: RecordType.Json, - // callbackMaxWaitTimeMs and streamPausedMaxWaitTimeMs are optional }; - assert.strictEqual(options.callbackMaxWaitTimeMs, undefined); assert.strictEqual(options.streamPausedMaxWaitTimeMs, undefined); }); }); @@ -432,49 +428,6 @@ describe('Batch operations', () => { }); }); -describe('AckCallback (v0.4.0)', () => { - it('should accept ack callback with onAck function', () => { - let ackCount = 0; - const callback: JsAckCallback = { - onAck: (offsetId: string) => { - ackCount++; - } - }; - assert.ok(callback.onAck); - assert.strictEqual(typeof callback.onAck, 'function'); - }); - - it('should accept ack callback with onError function', () => { - let errorCount = 0; - const callback: JsAckCallback = { - onError: (offsetId: string, errorMsg: string) => { - errorCount++; - } - }; - assert.ok(callback.onError); - assert.strictEqual(typeof callback.onError, 'function'); - }); - - it('should accept ack callback with both onAck and onError', () => { - const callback: JsAckCallback = { - onAck: (offsetId: string) => { - console.log(`Ack: ${offsetId}`); - }, - onError: (offsetId: string, errorMsg: string) => { - console.error(`Error: ${offsetId} - ${errorMsg}`); - } - }; - assert.ok(callback.onAck); - assert.ok(callback.onError); - }); - - it('should accept empty ack callback', () => { - const callback: JsAckCallback = {}; - assert.strictEqual(callback.onAck, undefined); - assert.strictEqual(callback.onError, undefined); - }); -}); - describe('New v0.4.0 API types', () => { it('should define ingestRecordOffset method on ZerobusStream type', () => { // Type check - these are defined in index.d.ts diff --git a/typescript/tsconfig.json b/typescript/tsconfig.json index 475ad4d0..cd5e9286 100644 --- a/typescript/tsconfig.json +++ b/typescript/tsconfig.json @@ -14,6 +14,6 @@ "resolveJsonModule": true, "allowSyntheticDefaultImports": true }, - "include": ["examples/**/*"], + "include": ["examples/**/*", "src/**/*", "test/**/*", "utils/**/*"], "exclude": ["node_modules", "target", "dist"] }