From 83cb3198bb7fef6b9d370545883a1dd933664005 Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Mon, 31 Aug 2026 11:21:47 +0700 Subject: [PATCH] ci(embeddings): expand cross-platform checks - Build C++ libraries and check embeddings across three platforms - Run Rust formatting, Clippy, unit, API, and matrix build checks - Cache third-party builds, Cargo data, targets, and ML models - Download ONNX Runtime and protoc; clean artifacts before caching - Avoid Windows linker conflicts and make invalid-path tests portable - Update embedding dependencies, lockfile, CI checks, and cache tests - Use typed pointer casting in the text model wrapper --- .github/workflows/ci.yml | 308 +++++++++++++++++++++ embeddings/Cargo.lock | 123 +------- embeddings/Cargo.toml | 8 +- embeddings/src/model/local_test.rs | 8 +- embeddings/src/model/text_model_wrapper.rs | 2 +- 5 files changed, 325 insertions(+), 124 deletions(-) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..3e658920 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,308 @@ +name: 🧪 CI + +# Isolated, daemon-free checks owned by the columnar repository itself. +# The manticoresearch repository keeps the integration flow: it builds MCL +# runtime libs and packages from the pinned submodule SHA (with exact-SHA +# caching) and runs daemon test suites. This workflow covers what can be +# validated without any manticore daemon checkout: +# - C++ libraries compile on all supported platforms; Windows is +# cross-compiled from Linux in the same external_toolchain container +# the daemon's mcl_runtime workflow uses (the shipped configuration) +# - embeddings crate: formatting, clippy, unit and API integration tests +# across the shipped platform matrix (linux x86_64/aarch64, macos +# x86_64/arm64, windows x86_64) + +on: + push: + branches: + - master + - columnar-* + pull_request: + workflow_dispatch: + +permissions: + contents: read + +# cancels the previous workflow run when a new one appears in the same branch +concurrency: + group: ci_${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + name: MCL ${{ matrix.os }} ${{ matrix.arch }} + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + # arm64/aarch64 entries rely on the SIMDe path: on arm, libfastpfor + # exports the FastPFOR::SIMDe interface target, columnar_root links it + # into every target, and USE_SIMDE redirects the x86 intrinsics + # (including the unguarded ones in util/delta_impl.h and + # columnar/accessor/accessortraits.h) to SIMDe implementations. + # macos-15 because GitHub fully deprecates macos-14 on 2026-11-02. + # No windows-2022/MSVC entry: nothing ships MSVC-built columnar — Windows + # DLLs are cross-compiled from Linux by the manticoresearch mcl_runtime + # workflow, and win_test_template runs the test suites on windows-2022 + # runners against those DLLs. The repo pins C++17 while secondary.cpp + # uses designated initializers, which MSVC rejects under C++17 (C7555). + include: + - os: ubuntu-24.04 + arch: x86_64 + - os: ubuntu-24.04-arm + arch: aarch64 + - os: macos-15 + arch: arm64 + env: + # persistent dir for third-party builds (FastPFOR, streamvbyte, hnswlib, PGM); + # relative on purpose: resolved against the build dir into /cache + # (cmake/init_cache_settings.cmake), and absolute Windows paths break the + # APPEND_PREFIX macro in cmake/helpers.cmake (backslash escapes re-parsed) + CACHEB: ../cache + steps: + - uses: actions/checkout@v5 + + - name: Cache third-party builds + uses: actions/cache@v5 + with: + path: ${{ github.workspace }}/cache + key: deps-${{ runner.os }}-${{ matrix.arch }}-${{ hashFiles('cmake/Get*.cmake', 'libfastpfor/CMakeLists.txt', 'pgm/CMakeLists.txt', 'streamvbyte/CMakeLists.txt') }} + restore-keys: | + deps-${{ runner.os }}-${{ matrix.arch }}- + + - name: Configure + # BUILD_TESTING=OFF: standalone tests need a daemon checkout (MANTICORE_LOCATOR) + run: cmake -B build -DBUILD_TESTING=OFF + + - name: Build libraries + # building explicit targets skips the AVX2/AVX-512 external projects; + # those are covered by the manticoresearch packaging workflows + run: cmake --build build --config RelWithDebInfo --target columnar_lib secondary_index knn_lib --parallel + + build-windows: + # The shipped Windows configuration: cross-compile from Linux in the same + # external_toolchain container the manticoresearch mcl_runtime workflow + # uses. Validates the _WIN32 code paths (mmap vs file IO, LoadLibraryEx + # shim in knn) on every push, not just when the daemon bumps the submodule. + name: MCL windows x64 (cross) + runs-on: ubuntu-22.04 + timeout-minutes: 40 + container: + image: manticoresearch/external_toolchain:vcpkg331_20260310 + env: + CACHEB: "../cache" + DIAGNOSTIC: 1 + DISTR: windows + boost: boost_80 + sysroot: roots_mysql83_jan17 + arch: x64 + SYSROOT_URL: https://repo.manticoresearch.com/repository/sysroots + steps: + - uses: actions/checkout@v5 + with: + set-safe-directory: true + + - name: Initialization + # entry_point.sh installs the cross toolchain (clang + windows sysroot) + run: | + bash /sysroot/root/entry_point.sh + git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Cache third-party builds + uses: actions/cache@v5 + with: + path: cache + key: deps-windows-x64-${{ hashFiles('cmake/Get*.cmake', 'libfastpfor/CMakeLists.txt', 'pgm/CMakeLists.txt', 'streamvbyte/CMakeLists.txt') }} + restore-keys: | + deps-windows-x64- + + - name: Configure + run: cmake -B build -DBUILD_TESTING=OFF -G "Ninja Multi-Config" + + - name: Build libraries + run: cmake --build build --config RelWithDebInfo --target columnar_lib secondary_index knn_lib --parallel + + embeddings: + # Quality gate for every shipped embeddings config — mirrors the + # embeddings_builds matrix in the daemon's mcl_pack_publish.yml (linux + # x86_64/aarch64, macos x86_64/arm64, windows x86_64). The daemon's own + # quality-check job runs on linux x64 only, so this repo owns + # fmt/clippy/test coverage for the whole shipped matrix. + name: Embeddings quality (${{ matrix.name }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + defaults: + run: + shell: bash + strategy: + fail-fast: false + matrix: + include: + - name: linux x86_64 + slug: linux-x64 + os: ubuntu-24.04 + protoc: linux-x86_64 + ort_platform: linux + ort_arch: x64 + rust_target: "" + run_tests: true + fmt: true + api_tests: true + - name: linux aarch64 + slug: linux-arm64 + os: ubuntu-24.04-arm + protoc: linux-aarch_64 + ort_platform: linux + ort_arch: aarch64 + rust_target: "" + run_tests: true + fmt: false + api_tests: false + - name: macos arm64 + slug: macos-arm64 + os: macos-15 + protoc: osx-aarch_64 + ort_platform: macos + ort_arch: arm64 + rust_target: "" + run_tests: true + fmt: false + api_tests: false + - name: macos x86_64 + # cross-compiled from the arm64 runner, exactly like the daemon's + # mcl_embeddings_core.yml build job; clippy-only because the + # Rosetta guard in embeddings/src/model/mod.rs refuses local + # inference under x86 emulation, so unit tests would exercise a + # configuration that never ships + slug: macos-x64-cross + os: macos-15 + protoc: osx-aarch_64 + ort_platform: macos + ort_arch: x86_64 + rust_target: x86_64-apple-darwin + run_tests: false + fmt: false + api_tests: false + - name: windows x86_64 + slug: windows-x64 + os: windows-2022 + protoc: win64 + ort_platform: windows + ort_arch: x64 + rust_target: "" + run_tests: true + fmt: false + api_tests: false + env: + # empty for native entries; --target for the cross-compiled one + CARGO_TARGET_FLAG: ${{ matrix.rust_target && format('--target {0}', matrix.rust_target) || '' }} + steps: + - uses: actions/checkout@v5 + + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + components: clippy, rustfmt + target: ${{ matrix.rust_target }} + + - name: Install protoc + run: | + set -euo pipefail + version="29.3" + url="https://github.com/protocolbuffers/protobuf/releases/download/v${version}/protoc-${version}-${{ matrix.protoc }}.zip" + dest="${RUNNER_TEMP}/protoc" + rm -rf "${dest}" + mkdir -p "${dest}" + curl --fail --show-error --location --retry 5 --retry-all-errors --retry-delay 2 "${url}" -o "${RUNNER_TEMP}/protoc.zip" + unzip -q "${RUNNER_TEMP}/protoc.zip" -d "${dest}" + echo "${dest}/bin" >> "${GITHUB_PATH}" + + - name: Cache cargo registry + uses: actions/cache@v5 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: cargo-registry-${{ runner.os }}-${{ hashFiles('embeddings/Cargo.lock') }} + restore-keys: cargo-registry-${{ runner.os }}- + + - name: Cache cargo build target + uses: actions/cache@v5 + with: + path: ./embeddings/target + key: cargo-target-${{ matrix.slug }}-${{ hashFiles('embeddings/Cargo.lock') }}-${{ hashFiles('embeddings/src/**/*.rs') }} + restore-keys: | + cargo-target-${{ matrix.slug }}-${{ hashFiles('embeddings/Cargo.lock') }}- + cargo-target-${{ matrix.slug }}- + + - name: Cache ML test models + if: matrix.run_tests + uses: actions/cache@v5 + with: + path: ./embeddings/.cache/manticore + key: ml-models-${{ matrix.slug }}-${{ hashFiles('embeddings/src/model/local_test.rs') }} + restore-keys: ml-models-${{ matrix.slug }}- + + - name: Download ONNX Runtime + # ort build.rs needs the static lib even for clippy; same source of + # truth as local builds: cmake/GetONNXRuntime.cmake + run: | + set -euo pipefail + cmake -DORT_PLATFORM=${{ matrix.ort_platform }} -DORT_ARCH=${{ matrix.ort_arch }} -DORT_OUT_DIR="${RUNNER_TEMP}/ort" -P cmake/GetONNXRuntime.cmake + echo "ORT_LIB_LOCATION=$(cat "${RUNNER_TEMP}/ort/lib_path.txt")" >> "$GITHUB_ENV" + + - name: Check code formatting + # target-independent; one matrix leg is enough + if: matrix.fmt + run: cargo fmt --check --manifest-path ./embeddings/Cargo.toml + + - name: Run Clippy (strict mode) + run: cargo clippy --lib $CARGO_TARGET_FLAG --manifest-path ./embeddings/Cargo.toml -- -D warnings + + - name: Run unit tests + if: matrix.run_tests + run: cargo test --lib $CARGO_TARGET_FLAG --verbose --manifest-path ./embeddings/Cargo.toml + env: + RUST_BACKTRACE: 1 + + - name: Run API integration tests (if API keys available) + # secrets-driven and platform-independent; one matrix leg avoids + # burning API quota on every platform + if: matrix.api_tests + run: | + cd ./embeddings + if [ -n "${{ secrets.OPENAI_API_KEY }}" ]; then + echo "✅ Running OpenAI integration tests..." + cargo test --lib openai_test::tests --verbose + else + echo "⏭️ Skipping OpenAI tests - OPENAI_API_KEY not configured" + fi + + if [ -n "${{ secrets.VOYAGE_API_KEY }}" ]; then + echo "✅ Running Voyager integration tests..." + cargo test --lib voyager_test::tests --verbose + else + echo "⏭️ Skipping Voyager tests - VOYAGE_API_KEY not configured" + fi + + if [ -n "${{ secrets.JINA_API_KEY }}" ]; then + echo "✅ Running Jina integration tests..." + cargo test --lib jina_test::tests --verbose + else + echo "⏭️ Skipping Jina tests - JINA_API_KEY not configured" + fi + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + VOYAGE_API_KEY: ${{ secrets.VOYAGE_API_KEY }} + JINA_API_KEY: ${{ secrets.JINA_API_KEY }} + RUST_BACKTRACE: 1 + + - name: Clean build target bloat + # keeps ./embeddings/target cache under GitHub's 10GB repo limit; + # mirrors the daemon-side mcl_embeddings_core.yml quality-check job + if: always() + run: | + rm -rf ./embeddings/target/debug/incremental ./embeddings/target/debug/.fingerprint + rm -rf ./embeddings/target/*/debug/incremental ./embeddings/target/*/debug/.fingerprint + find ./embeddings/target -name "*.d" -delete 2>/dev/null || true \ No newline at end of file diff --git a/embeddings/Cargo.lock b/embeddings/Cargo.lock index 452ff0f8..e94a79b6 100644 --- a/embeddings/Cargo.lock +++ b/embeddings/Cargo.lock @@ -38,56 +38,6 @@ dependencies = [ "libc", ] -[[package]] -name = "anstream" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" - -[[package]] -name = "anstyle-parse" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" -dependencies = [ - "anstyle", - "once_cell_polyfill", - "windows-sys 0.61.2", -] - [[package]] name = "anyhow" version = "1.0.102" @@ -352,7 +302,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da6bc11b07529f16944307272d5bd9b22530bc7d05751717c9d416586cedab49" dependencies = [ - "clap 3.2.25", + "clap", "heck 0.4.1", "indexmap 1.9.3", "log", @@ -408,47 +358,13 @@ checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" dependencies = [ "atty", "bitflags 1.3.2", - "clap_lex 0.2.4", + "clap_lex", "indexmap 1.9.3", "strsim 0.10.0", "termcolor", "textwrap", ] -[[package]] -name = "clap" -version = "4.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" -dependencies = [ - "clap_builder", - "clap_derive", -] - -[[package]] -name = "clap_builder" -version = "4.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" -dependencies = [ - "anstream", - "anstyle", - "clap_lex 1.1.0", - "strsim 0.11.1", -] - -[[package]] -name = "clap_derive" -version = "4.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" -dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "clap_lex" version = "0.2.4" @@ -458,18 +374,6 @@ dependencies = [ "os_str_bytes", ] -[[package]] -name = "clap_lex" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" - -[[package]] -name = "colorchoice" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" - [[package]] name = "console" version = "0.15.11" @@ -844,9 +748,6 @@ name = "esaxx-rs" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" -dependencies = [ - "cc", -] [[package]] name = "fancy-regex" @@ -1720,12 +1621,6 @@ version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" -[[package]] -name = "is_terminal_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" - [[package]] name = "itertools" version = "0.11.0" @@ -2222,12 +2117,6 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" -[[package]] -name = "once_cell_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" - [[package]] name = "onig" version = "6.5.3" @@ -3172,11 +3061,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dd47962b0ba36e7fd33518fbf1754d136fd1474000162bbf2a8b5fcb2d3654d" dependencies = [ "aho-corasick", - "clap 4.6.1", "derive_builder 0.12.0", "esaxx-rs", "getrandom 0.2.17", - "indicatif", "itertools 0.12.1", "lazy_static", "log", @@ -3531,12 +3418,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - [[package]] name = "uuid" version = "1.23.1" diff --git a/embeddings/Cargo.toml b/embeddings/Cargo.toml index b1c0b3cf..8758130d 100644 --- a/embeddings/Cargo.toml +++ b/embeddings/Cargo.toml @@ -6,7 +6,13 @@ edition = "2021" # Candle: git dep so CI works without a local candle clone. # For local dev with ../../candle, add a [patch] section to use path deps. [dependencies] -tokenizers = "0.15.2" +# Inference only: no trainer, no CLI. Default features are avoided because +# esaxx_fast enables esaxx-rs's "cpp" feature, whose build.rs hardcodes +# cc::Build::static_crt(true): esaxx.o is compiled /MT while Rust std and +# the ORT static lib are /MD, so linking any binary on Windows fails with +# LNK2038. esaxx is used exclusively by the unigram trainer, which we never +# run. onig stays: tokenizers' SysRegex is onig-backed on non-wasm targets. +tokenizers = { version = "0.15.2", default-features = false, features = ["onig"] } hf-hub = { git = "https://github.com/huggingface/hf-hub.git", rev = "ac22200ea0b5af4d8c362f699be0340647b19060", default-features = false,features = ["ureq"] } anyhow = "1.0.81" serde_json = "1.0.114" diff --git a/embeddings/src/model/local_test.rs b/embeddings/src/model/local_test.rs index 6bd9dad1..62ce4979 100644 --- a/embeddings/src/model/local_test.rs +++ b/embeddings/src/model/local_test.rs @@ -26,7 +26,13 @@ mod tests { #[test] fn test_local_model_creation_invalid_path() { let model_id = "sentence-transformers/all-MiniLM-L6-v2"; - let cache_path = PathBuf::from("/nonexistent/path"); + // A cache path under a regular file fails on every platform: hf-hub + // create_dir_all()s the download dir under it. A bare "/nonexistent" + // path is creatable on Windows (resolves to :\nonexistent), + // which made this test download the model and return Ok there. + let blocker = std::env::temp_dir().join("manticore_knn_not_a_dir"); + std::fs::write(&blocker, b"").unwrap(); + let cache_path = blocker.join("cache"); let result = LocalModel::new(model_id, cache_path, false, None); diff --git a/embeddings/src/model/text_model_wrapper.rs b/embeddings/src/model/text_model_wrapper.rs index 396de4a6..638c8769 100644 --- a/embeddings/src/model/text_model_wrapper.rs +++ b/embeddings/src/model/text_model_wrapper.rs @@ -228,7 +228,7 @@ impl TextModelWrapper { let string_refs: Vec<&str> = string_slice .iter() .map(|item| unsafe { - let bytes = std::slice::from_raw_parts(item.ptr as *const u8, item.len); + let bytes = std::slice::from_raw_parts(item.ptr.cast::(), item.len); std::str::from_utf8_unchecked(bytes) }) .collect();