test: exercise real database worker artifacts

Author: rustyconover

CLAUDE.md

@@ -92,6 +92,18 @@ the matrix had the machinery but nothing ran it, so the failures accumulated
 silently. It is not wired into CI yet (four toolchains on one runner); until it
 is, running it locally is the guardrail.
 
+### Database-packaged worker artifacts (`make test_database_workers`)
+
+The ordinary `database_worker/package.test` uses a tiny executable wrapper so
+the default regression stays fast. The explicit heavyweight target builds and
+packages three real workers, then restores and executes each one through
+`database://`: a PyInstaller-frozen Python/PyArrow worker, Bun's standalone
+build of `~/Development/vgi-open-meteo`, and a `.tar.gz` containing the native
+Rust example worker plus a metadata file. It expects the sibling vgi-python,
+vgi-open-meteo, and vgi-rust repositories and the `uv`, `bun`, `cargo`, and
+`tar` tools. Repository paths and cross-build targets are overrideable; see
+`docs/database-worker-transport.md`.
+
 ### HTTP Transport
 
 ```bash

Makefile

@@ -446,7 +446,33 @@ VGI_JAVA_DIR   ?= $(HOME)/vgi-java
 VGI_RUST_DIR   ?= $(HOME)/Development/vgi-rust
 VGI_CSHARP_DIR ?= $(HOME)/Development/vgi-csharp
 
-.PHONY: test_python test_python_crash test_go test_typescript test_java test_rust test_csharp test_languages
+.PHONY: test_python test_python_crash test_go test_typescript test_java test_rust test_csharp test_languages \
+	build_database_workers test_database_workers
+
+# Real database-package demonstrations. These are intentionally separate from
+# the fast default suite: PyInstaller and Bun each produce a ~100 MiB native
+# executable, and Rust compiles the full example worker. The resulting tests do
+# not delegate to VGI_TEST_WORKER; the BLOB restored from DuckDB is the worker.
+VGI_DATABASE_WORKER_FIXTURE_DIR ?= $(PROJ_DIR)build/database-worker-fixtures
+VGI_DATABASE_PYTHON_DIR ?= $(HOME)/Development/vgi-python
+VGI_DATABASE_RUST_DIR ?= $(HOME)/Development/vgi-rust
+VGI_DATABASE_OPEN_METEO_DIR ?= $(HOME)/Development/vgi-open-meteo
+
+build_database_workers:
+	VGI_DATABASE_WORKER_FIXTURE_DIR="$(VGI_DATABASE_WORKER_FIXTURE_DIR)" \
+	VGI_DATABASE_PYTHON_DIR="$(VGI_DATABASE_PYTHON_DIR)" \
+	VGI_DATABASE_RUST_DIR="$(VGI_DATABASE_RUST_DIR)" \
+	VGI_DATABASE_OPEN_METEO_DIR="$(VGI_DATABASE_OPEN_METEO_DIR)" \
+	./scripts/build_database_worker_fixtures.sh
+
+test_database_workers: build_database_workers
+	database_worker_cache="$$(mktemp -d)"; \
+	trap 'rm -rf "$$database_worker_cache"' 0; \
+	XDG_CACHE_HOME="$$database_worker_cache" \
+	VGI_DATABASE_PYTHON_WORKER="$(VGI_DATABASE_WORKER_FIXTURE_DIR)/vgi-python-worker" \
+	VGI_DATABASE_BUN_WORKER="$(VGI_DATABASE_WORKER_FIXTURE_DIR)/vgi-open-meteo" \
+	VGI_DATABASE_RUST_WORKER="$(VGI_DATABASE_WORKER_FIXTURE_DIR)/vgi-rust-worker.tar.gz" \
+	./build/release/test/unittest "test/sql/integration/database_worker/real_*"
 
 # Python uses this repo's default worker set. It does NOT just chain to
 # test_launcher, which invokes `unittest` bare: a lane that stops running tests

docs/database-worker-transport.md

@@ -127,3 +127,41 @@ before retrying when immediate deletion is required.
 This feature executes native code stored in the selected database. Only attach
 package registries whose writers you trust, and use the digest pin when the
 expected build is known out of band.
+
+## Real-worker packaging tests
+
+The fast default regression packages a small executable wrapper so cache and
+lease behavior stays cheap. The explicit heavyweight lane builds and executes
+three complete artifacts instead:
+
+```bash
+make test_database_workers
+```
+
+- Python: `uv` runs PyInstaller against
+  `test/support/database_workers/python_worker.py`, producing one executable
+  containing CPython, PyArrow, VGI, and its dependencies.
+- JavaScript/TypeScript: Bun compiles the actual
+  `~/Development/vgi-open-meteo/src/bin/worker.ts` graph and the Bun runtime
+  into one executable. The test restores it from the BLOB and queries the
+  real catalog's static `weather_codes` view.
+- Rust: Cargo builds `vgi-example-worker`; the lane places it and a metadata
+  file in a `.tar.gz`, then proves VGI extracts `bin/vgi-rust-worker` and runs
+  `sequence(7)`.
+
+Override `VGI_DATABASE_PYTHON_DIR`, `VGI_DATABASE_OPEN_METEO_DIR`, or
+`VGI_DATABASE_RUST_DIR` when the sibling repositories live elsewhere. Bun can
+cross-compile by setting `VGI_BUN_TARGET` (for example,
+`bun-linux-arm64`). Rust cross-compilation uses `VGI_RUST_TARGET` and
+`cargo-zigbuild`, for example:
+
+```bash
+CARGO_ZIGBUILD_ZIG_COMMAND="$(command -v python-zig)" \
+  VGI_RUST_TARGET=x86_64-unknown-linux-gnu make build_database_workers
+```
+
+Use `build_database_workers`, rather than `test_database_workers`, when the
+target architecture differs from the build host. The builder itself is a
+POSIX-host development tool. PyInstaller artifacts always target the build OS
+and architecture; `VGI_BUN_TARGET` affects only the Bun artifact and
+`VGI_RUST_TARGET` affects only the Rust artifact.

scripts/build_database_worker_fixtures.sh

@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+# Build real, self-contained database:// worker artifacts for the host platform.
+
+set -euo pipefail
+
+project_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+output_dir=${VGI_DATABASE_WORKER_FIXTURE_DIR:-"${project_dir}/build/database-worker-fixtures"}
+python_dir=${VGI_DATABASE_PYTHON_DIR:-"${HOME}/Development/vgi-python"}
+rust_dir=${VGI_DATABASE_RUST_DIR:-"${HOME}/Development/vgi-rust"}
+open_meteo_dir=${VGI_DATABASE_OPEN_METEO_DIR:-"${HOME}/Development/vgi-open-meteo"}
+
+for command_name in uv bun cargo tar; do
+	if ! command -v "${command_name}" >/dev/null 2>&1; then
+		echo "database-worker fixtures require ${command_name} on PATH" >&2
+		exit 1
+	fi
+done
+for source_dir in "${python_dir}" "${rust_dir}" "${open_meteo_dir}"; do
+	if [[ ! -d "${source_dir}" ]]; then
+		echo "database-worker fixture source directory does not exist: ${source_dir}" >&2
+		exit 1
+	fi
+done
+
+mkdir -p "${output_dir}"
+scratch_dir=$(mktemp -d "${TMPDIR:-/tmp}/vgi-database-workers.XXXXXX")
+cleanup() {
+	rm -rf "${scratch_dir}"
+}
+trap cleanup EXIT
+
+echo "Building frozen Python worker"
+uv run --project "${python_dir}" --locked --with pyinstaller==6.22.2 \
+	pyinstaller --clean --onefile --name vgi-python-worker \
+	--distpath "${output_dir}" \
+	--workpath "${scratch_dir}/pyinstaller-build" \
+	--specpath "${scratch_dir}/pyinstaller-spec" \
+	"${project_dir}/test/support/database_workers/python_worker.py"
+
+if [[ -n "${VGI_BUN_TARGET:-}" ]]; then
+	bun_target=${VGI_BUN_TARGET}
+else
+	host_os=$(uname -s)
+	host_arch=$(uname -m)
+	case "${host_os}/${host_arch}" in
+		Darwin/arm64) bun_target=bun-darwin-arm64 ;;
+		Darwin/x86_64) bun_target=bun-darwin-x64 ;;
+		Linux/aarch64|Linux/arm64) bun_target=bun-linux-arm64 ;;
+		Linux/x86_64) bun_target=bun-linux-x64-baseline ;;
+		*)
+			echo "cannot infer a Bun executable target for ${host_os}/${host_arch}" >&2
+			exit 1
+			;;
+	esac
+fi
+
+echo "Building Open Meteo worker for ${bun_target}"
+(
+	cd "${open_meteo_dir}"
+	bun install --frozen-lockfile
+	bun build src/bin/worker.ts --compile --target="${bun_target}" \
+		--outfile "${output_dir}/vgi-open-meteo"
+)
+
+echo "Building Rust worker"
+if [[ -n "${VGI_RUST_TARGET:-}" ]]; then
+	if ! command -v cargo-zigbuild >/dev/null 2>&1; then
+		echo "VGI_RUST_TARGET requires cargo-zigbuild on PATH" >&2
+		exit 1
+	fi
+	(
+		cd "${rust_dir}"
+		cargo zigbuild --locked --release --target "${VGI_RUST_TARGET}" -p vgi-example-worker
+	)
+	rust_binary="${rust_dir}/target/${VGI_RUST_TARGET}/release/vgi-example-worker"
+else
+	(
+		cd "${rust_dir}"
+		cargo build --locked --release -p vgi-example-worker
+	)
+	rust_binary="${rust_dir}/target/release/vgi-example-worker"
+fi
+
+# Use a multi-file archive for Rust so this lane proves safe tar.gz extraction,
+# relative entrypoint selection, and execution rather than only raw BLOB copies.
+mkdir -p "${scratch_dir}/rust-package/bin"
+install -m 0755 "${rust_binary}" "${scratch_dir}/rust-package/bin/vgi-rust-worker"
+printf '%s\n' "VGI Rust database-worker fixture" > "${scratch_dir}/rust-package/BUILD.txt"
+tar -czf "${output_dir}/vgi-rust-worker.tar.gz" \
+	-C "${scratch_dir}/rust-package" bin BUILD.txt
+
+echo "Built database-worker artifacts in ${output_dir}"
+ls -lh "${output_dir}/vgi-python-worker" \
+	"${output_dir}/vgi-open-meteo" \
+	"${output_dir}/vgi-rust-worker.tar.gz"

test/sql/integration/database_worker/errors.test

@@ -1,5 +1,5 @@
 # name: test/sql/integration/database_worker/errors.test
-# description: database:// URI, row uniqueness, digest, size, and archive-path validation
+# description: database:// URI, row uniqueness, digest, size, and entrypoint validation
 # group: [vgi_integration_database_worker]
 
 require vgi

test/sql/integration/database_worker/package.test

@@ -1,18 +1,31 @@
 # name: test/sql/integration/database_worker/package.test
-# description: package macro + database:// resolve/cache/lease/cleanup lifecycle
+# description: fast wrapper smoke + database:// resolve/cache/lease/cleanup lifecycle
 # group: [vgi_integration_database_worker]
 
 require-env VGI_TEST_WORKER
 
 require vgi
 
+# Keep cache lifecycle tests out of the developer's persistent cache. Pointing
+# both transports at the same test-local root preserves the namespace-safety
+# assertion below.
+statement ok
+SET vgi_worker_cache_dir = '__TEST_DIR__/vgi_database_worker_package_cache';
+
+statement ok
+SET vgi_github_cache_dir = '__TEST_DIR__/vgi_database_worker_package_cache';
+
 statement ok
 CREATE TABLE worker_packages AS
 SELECT * FROM vgi_worker_package(
     'test/support/database_worker_fixture.sh',
     'fixture',
     'test-1');
 
+# This fast default-suite fixture is an executable wrapper around
+# VGI_TEST_WORKER. The heavyweight test_database_workers target separately
+# packages and runs self-contained Python, Bun, and Rust worker artifacts.
+
 query TITT
 SELECT worker_name, platform = (SELECT platform FROM pragma_platform()), package_version, package_format
 FROM worker_packages;
@@ -61,7 +74,9 @@ SELECT count(*) FROM db_worker_2.main.sequence(3);
 3
 
 query II
-SELECT count(*), bool_and(in_use) FROM vgi_worker_cache();
+SELECT count(*), bool_and(in_use)
+FROM vgi_worker_cache()
+WHERE source = 'database://memory/main/worker_packages/fixture?package_version=test-1';
 ----
 1	true
 
@@ -89,6 +104,8 @@ SELECT removed, skipped_in_use FROM vgi_worker_cache_flush();
 1	0
 
 query I
-SELECT count(*) FROM vgi_worker_cache();
+SELECT count(*)
+FROM vgi_worker_cache()
+WHERE source = 'database://memory/main/worker_packages/fixture?package_version=test-1';
 ----
 0

test/sql/integration/database_worker/real_bun_open_meteo.test

@@ -0,0 +1,48 @@
+# name: test/sql/integration/database_worker/real_bun_open_meteo.test
+# description: compile the real Bun Open Meteo worker, store it as a BLOB, and execute it
+# group: [vgi_integration_database_worker]
+
+require-env VGI_DATABASE_BUN_WORKER
+
+require vgi
+
+statement ok
+SET vgi_worker_cache_dir = '__TEST_DIR__/vgi_database_worker_bun_cache';
+
+statement ok
+CREATE TABLE bun_worker_packages AS
+SELECT * FROM vgi_worker_package(
+    '${VGI_DATABASE_BUN_WORKER}',
+    'open-meteo',
+    'bun-standalone-1');
+
+query IT
+SELECT octet_length(contents) > 1000000, package_format
+FROM bun_worker_packages;
+----
+true	executable
+
+statement ok
+ATTACH 'open_meteo' AS meteo (
+    TYPE vgi,
+    LOCATION 'database://memory/main/bun_worker_packages/open-meteo?package_version=bun-standalone-1');
+
+# Static catalog data makes this deterministic and proves the real worker runs
+# without making the test depend on the live Open-Meteo API.
+query I
+SELECT count(*) > 0 FROM meteo.main.weather_codes;
+----
+true
+
+statement ok
+DETACH meteo;
+
+query I
+SELECT flushed >= 1 FROM vgi_worker_pool_flush();
+----
+true
+
+query II
+SELECT removed, skipped_in_use FROM vgi_worker_cache_flush();
+----
+1	0

test/sql/integration/database_worker/real_python.test

@@ -0,0 +1,49 @@
+# name: test/sql/integration/database_worker/real_python.test
+# description: freeze a real Python VGI worker, store it as a BLOB, and execute it
+# group: [vgi_integration_database_worker]
+
+require-env VGI_DATABASE_PYTHON_WORKER
+
+require vgi
+
+statement ok
+SET vgi_worker_cache_dir = '__TEST_DIR__/vgi_database_worker_python_cache';
+
+statement ok
+CREATE TABLE python_worker_packages AS
+SELECT * FROM vgi_worker_package(
+    '${VGI_DATABASE_PYTHON_WORKER}',
+    'python-fixture',
+    'pyinstaller-1');
+
+query IITT
+SELECT octet_length(contents) > 1000000,
+       length(sha256),
+       package_format,
+       platform = (SELECT platform FROM pragma_platform())
+FROM python_worker_packages;
+----
+true	64	executable	true
+
+statement ok
+ATTACH 'python_package' AS python_worker (
+    TYPE vgi,
+    LOCATION 'database://memory/main/python_worker_packages/python-fixture?package_version=pyinstaller-1');
+
+query II
+SELECT count(*), sum(n) FROM python_worker.main.series(6);
+----
+6	15
+
+statement ok
+DETACH python_worker;
+
+query I
+SELECT flushed >= 1 FROM vgi_worker_pool_flush();
+----
+true
+
+query II
+SELECT removed, skipped_in_use FROM vgi_worker_cache_flush();
+----
+1	0