Skip to content

Commit 83cb319

Browse files
donhardmandonhardman
committed
ci(embeddings): expand cross-platform checks
- Build C++ libraries and check embeddings across three platforms - Run Rust formatting, Clippy, unit, API, and matrix build checks - Cache third-party builds, Cargo data, targets, and ML models - Download ONNX Runtime and protoc; clean artifacts before caching - Avoid Windows linker conflicts and make invalid-path tests portable - Update embedding dependencies, lockfile, CI checks, and cache tests - Use typed pointer casting in the text model wrapper
1 parent a34ffcc commit 83cb319

5 files changed

Lines changed: 325 additions & 124 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 308 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,308 @@
1+
name: 🧪 CI
2+
3+
# Isolated, daemon-free checks owned by the columnar repository itself.
4+
# The manticoresearch repository keeps the integration flow: it builds MCL
5+
# runtime libs and packages from the pinned submodule SHA (with exact-SHA
6+
# caching) and runs daemon test suites. This workflow covers what can be
7+
# validated without any manticore daemon checkout:
8+
# - C++ libraries compile on all supported platforms; Windows is
9+
# cross-compiled from Linux in the same external_toolchain container
10+
# the daemon's mcl_runtime workflow uses (the shipped configuration)
11+
# - embeddings crate: formatting, clippy, unit and API integration tests
12+
# across the shipped platform matrix (linux x86_64/aarch64, macos
13+
# x86_64/arm64, windows x86_64)
14+
15+
on:
16+
push:
17+
branches:
18+
- master
19+
- columnar-*
20+
pull_request:
21+
workflow_dispatch:
22+
23+
permissions:
24+
contents: read
25+
26+
# cancels the previous workflow run when a new one appears in the same branch
27+
concurrency:
28+
group: ci_${{ github.ref }}
29+
cancel-in-progress: true
30+
31+
jobs:
32+
build:
33+
name: MCL ${{ matrix.os }} ${{ matrix.arch }}
34+
runs-on: ${{ matrix.os }}
35+
timeout-minutes: 40
36+
strategy:
37+
fail-fast: false
38+
matrix:
39+
# arm64/aarch64 entries rely on the SIMDe path: on arm, libfastpfor
40+
# exports the FastPFOR::SIMDe interface target, columnar_root links it
41+
# into every target, and USE_SIMDE redirects the x86 intrinsics
42+
# (including the unguarded ones in util/delta_impl.h and
43+
# columnar/accessor/accessortraits.h) to SIMDe implementations.
44+
# macos-15 because GitHub fully deprecates macos-14 on 2026-11-02.
45+
# No windows-2022/MSVC entry: nothing ships MSVC-built columnar — Windows
46+
# DLLs are cross-compiled from Linux by the manticoresearch mcl_runtime
47+
# workflow, and win_test_template runs the test suites on windows-2022
48+
# runners against those DLLs. The repo pins C++17 while secondary.cpp
49+
# uses designated initializers, which MSVC rejects under C++17 (C7555).
50+
include:
51+
- os: ubuntu-24.04
52+
arch: x86_64
53+
- os: ubuntu-24.04-arm
54+
arch: aarch64
55+
- os: macos-15
56+
arch: arm64
57+
env:
58+
# persistent dir for third-party builds (FastPFOR, streamvbyte, hnswlib, PGM);
59+
# relative on purpose: resolved against the build dir into <workspace>/cache
60+
# (cmake/init_cache_settings.cmake), and absolute Windows paths break the
61+
# APPEND_PREFIX macro in cmake/helpers.cmake (backslash escapes re-parsed)
62+
CACHEB: ../cache
63+
steps:
64+
- uses: actions/checkout@v5
65+
66+
- name: Cache third-party builds
67+
uses: actions/cache@v5
68+
with:
69+
path: ${{ github.workspace }}/cache
70+
key: deps-${{ runner.os }}-${{ matrix.arch }}-${{ hashFiles('cmake/Get*.cmake', 'libfastpfor/CMakeLists.txt', 'pgm/CMakeLists.txt', 'streamvbyte/CMakeLists.txt') }}
71+
restore-keys: |
72+
deps-${{ runner.os }}-${{ matrix.arch }}-
73+
74+
- name: Configure
75+
# BUILD_TESTING=OFF: standalone tests need a daemon checkout (MANTICORE_LOCATOR)
76+
run: cmake -B build -DBUILD_TESTING=OFF
77+
78+
- name: Build libraries
79+
# building explicit targets skips the AVX2/AVX-512 external projects;
80+
# those are covered by the manticoresearch packaging workflows
81+
run: cmake --build build --config RelWithDebInfo --target columnar_lib secondary_index knn_lib --parallel
82+
83+
build-windows:
84+
# The shipped Windows configuration: cross-compile from Linux in the same
85+
# external_toolchain container the manticoresearch mcl_runtime workflow
86+
# uses. Validates the _WIN32 code paths (mmap vs file IO, LoadLibraryEx
87+
# shim in knn) on every push, not just when the daemon bumps the submodule.
88+
name: MCL windows x64 (cross)
89+
runs-on: ubuntu-22.04
90+
timeout-minutes: 40
91+
container:
92+
image: manticoresearch/external_toolchain:vcpkg331_20260310
93+
env:
94+
CACHEB: "../cache"
95+
DIAGNOSTIC: 1
96+
DISTR: windows
97+
boost: boost_80
98+
sysroot: roots_mysql83_jan17
99+
arch: x64
100+
SYSROOT_URL: https://repo.manticoresearch.com/repository/sysroots
101+
steps:
102+
- uses: actions/checkout@v5
103+
with:
104+
set-safe-directory: true
105+
106+
- name: Initialization
107+
# entry_point.sh installs the cross toolchain (clang + windows sysroot)
108+
run: |
109+
bash /sysroot/root/entry_point.sh
110+
git config --global --add safe.directory "$GITHUB_WORKSPACE"
111+
112+
- name: Cache third-party builds
113+
uses: actions/cache@v5
114+
with:
115+
path: cache
116+
key: deps-windows-x64-${{ hashFiles('cmake/Get*.cmake', 'libfastpfor/CMakeLists.txt', 'pgm/CMakeLists.txt', 'streamvbyte/CMakeLists.txt') }}
117+
restore-keys: |
118+
deps-windows-x64-
119+
120+
- name: Configure
121+
run: cmake -B build -DBUILD_TESTING=OFF -G "Ninja Multi-Config"
122+
123+
- name: Build libraries
124+
run: cmake --build build --config RelWithDebInfo --target columnar_lib secondary_index knn_lib --parallel
125+
126+
embeddings:
127+
# Quality gate for every shipped embeddings config — mirrors the
128+
# embeddings_builds matrix in the daemon's mcl_pack_publish.yml (linux
129+
# x86_64/aarch64, macos x86_64/arm64, windows x86_64). The daemon's own
130+
# quality-check job runs on linux x64 only, so this repo owns
131+
# fmt/clippy/test coverage for the whole shipped matrix.
132+
name: Embeddings quality (${{ matrix.name }})
133+
runs-on: ${{ matrix.os }}
134+
timeout-minutes: 40
135+
defaults:
136+
run:
137+
shell: bash
138+
strategy:
139+
fail-fast: false
140+
matrix:
141+
include:
142+
- name: linux x86_64
143+
slug: linux-x64
144+
os: ubuntu-24.04
145+
protoc: linux-x86_64
146+
ort_platform: linux
147+
ort_arch: x64
148+
rust_target: ""
149+
run_tests: true
150+
fmt: true
151+
api_tests: true
152+
- name: linux aarch64
153+
slug: linux-arm64
154+
os: ubuntu-24.04-arm
155+
protoc: linux-aarch_64
156+
ort_platform: linux
157+
ort_arch: aarch64
158+
rust_target: ""
159+
run_tests: true
160+
fmt: false
161+
api_tests: false
162+
- name: macos arm64
163+
slug: macos-arm64
164+
os: macos-15
165+
protoc: osx-aarch_64
166+
ort_platform: macos
167+
ort_arch: arm64
168+
rust_target: ""
169+
run_tests: true
170+
fmt: false
171+
api_tests: false
172+
- name: macos x86_64
173+
# cross-compiled from the arm64 runner, exactly like the daemon's
174+
# mcl_embeddings_core.yml build job; clippy-only because the
175+
# Rosetta guard in embeddings/src/model/mod.rs refuses local
176+
# inference under x86 emulation, so unit tests would exercise a
177+
# configuration that never ships
178+
slug: macos-x64-cross
179+
os: macos-15
180+
protoc: osx-aarch_64
181+
ort_platform: macos
182+
ort_arch: x86_64
183+
rust_target: x86_64-apple-darwin
184+
run_tests: false
185+
fmt: false
186+
api_tests: false
187+
- name: windows x86_64
188+
slug: windows-x64
189+
os: windows-2022
190+
protoc: win64
191+
ort_platform: windows
192+
ort_arch: x64
193+
rust_target: ""
194+
run_tests: true
195+
fmt: false
196+
api_tests: false
197+
env:
198+
# empty for native entries; --target <triple> for the cross-compiled one
199+
CARGO_TARGET_FLAG: ${{ matrix.rust_target && format('--target {0}', matrix.rust_target) || '' }}
200+
steps:
201+
- uses: actions/checkout@v5
202+
203+
- name: Setup Rust toolchain
204+
uses: dtolnay/rust-toolchain@stable
205+
with:
206+
components: clippy, rustfmt
207+
target: ${{ matrix.rust_target }}
208+
209+
- name: Install protoc
210+
run: |
211+
set -euo pipefail
212+
version="29.3"
213+
url="https://github.com/protocolbuffers/protobuf/releases/download/v${version}/protoc-${version}-${{ matrix.protoc }}.zip"
214+
dest="${RUNNER_TEMP}/protoc"
215+
rm -rf "${dest}"
216+
mkdir -p "${dest}"
217+
curl --fail --show-error --location --retry 5 --retry-all-errors --retry-delay 2 "${url}" -o "${RUNNER_TEMP}/protoc.zip"
218+
unzip -q "${RUNNER_TEMP}/protoc.zip" -d "${dest}"
219+
echo "${dest}/bin" >> "${GITHUB_PATH}"
220+
221+
- name: Cache cargo registry
222+
uses: actions/cache@v5
223+
with:
224+
path: |
225+
~/.cargo/registry
226+
~/.cargo/git
227+
key: cargo-registry-${{ runner.os }}-${{ hashFiles('embeddings/Cargo.lock') }}
228+
restore-keys: cargo-registry-${{ runner.os }}-
229+
230+
- name: Cache cargo build target
231+
uses: actions/cache@v5
232+
with:
233+
path: ./embeddings/target
234+
key: cargo-target-${{ matrix.slug }}-${{ hashFiles('embeddings/Cargo.lock') }}-${{ hashFiles('embeddings/src/**/*.rs') }}
235+
restore-keys: |
236+
cargo-target-${{ matrix.slug }}-${{ hashFiles('embeddings/Cargo.lock') }}-
237+
cargo-target-${{ matrix.slug }}-
238+
239+
- name: Cache ML test models
240+
if: matrix.run_tests
241+
uses: actions/cache@v5
242+
with:
243+
path: ./embeddings/.cache/manticore
244+
key: ml-models-${{ matrix.slug }}-${{ hashFiles('embeddings/src/model/local_test.rs') }}
245+
restore-keys: ml-models-${{ matrix.slug }}-
246+
247+
- name: Download ONNX Runtime
248+
# ort build.rs needs the static lib even for clippy; same source of
249+
# truth as local builds: cmake/GetONNXRuntime.cmake
250+
run: |
251+
set -euo pipefail
252+
cmake -DORT_PLATFORM=${{ matrix.ort_platform }} -DORT_ARCH=${{ matrix.ort_arch }} -DORT_OUT_DIR="${RUNNER_TEMP}/ort" -P cmake/GetONNXRuntime.cmake
253+
echo "ORT_LIB_LOCATION=$(cat "${RUNNER_TEMP}/ort/lib_path.txt")" >> "$GITHUB_ENV"
254+
255+
- name: Check code formatting
256+
# target-independent; one matrix leg is enough
257+
if: matrix.fmt
258+
run: cargo fmt --check --manifest-path ./embeddings/Cargo.toml
259+
260+
- name: Run Clippy (strict mode)
261+
run: cargo clippy --lib $CARGO_TARGET_FLAG --manifest-path ./embeddings/Cargo.toml -- -D warnings
262+
263+
- name: Run unit tests
264+
if: matrix.run_tests
265+
run: cargo test --lib $CARGO_TARGET_FLAG --verbose --manifest-path ./embeddings/Cargo.toml
266+
env:
267+
RUST_BACKTRACE: 1
268+
269+
- name: Run API integration tests (if API keys available)
270+
# secrets-driven and platform-independent; one matrix leg avoids
271+
# burning API quota on every platform
272+
if: matrix.api_tests
273+
run: |
274+
cd ./embeddings
275+
if [ -n "${{ secrets.OPENAI_API_KEY }}" ]; then
276+
echo "✅ Running OpenAI integration tests..."
277+
cargo test --lib openai_test::tests --verbose
278+
else
279+
echo "⏭️ Skipping OpenAI tests - OPENAI_API_KEY not configured"
280+
fi
281+
282+
if [ -n "${{ secrets.VOYAGE_API_KEY }}" ]; then
283+
echo "✅ Running Voyager integration tests..."
284+
cargo test --lib voyager_test::tests --verbose
285+
else
286+
echo "⏭️ Skipping Voyager tests - VOYAGE_API_KEY not configured"
287+
fi
288+
289+
if [ -n "${{ secrets.JINA_API_KEY }}" ]; then
290+
echo "✅ Running Jina integration tests..."
291+
cargo test --lib jina_test::tests --verbose
292+
else
293+
echo "⏭️ Skipping Jina tests - JINA_API_KEY not configured"
294+
fi
295+
env:
296+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
297+
VOYAGE_API_KEY: ${{ secrets.VOYAGE_API_KEY }}
298+
JINA_API_KEY: ${{ secrets.JINA_API_KEY }}
299+
RUST_BACKTRACE: 1
300+
301+
- name: Clean build target bloat
302+
# keeps ./embeddings/target cache under GitHub's 10GB repo limit;
303+
# mirrors the daemon-side mcl_embeddings_core.yml quality-check job
304+
if: always()
305+
run: |
306+
rm -rf ./embeddings/target/debug/incremental ./embeddings/target/debug/.fingerprint
307+
rm -rf ./embeddings/target/*/debug/incremental ./embeddings/target/*/debug/.fingerprint
308+
find ./embeddings/target -name "*.d" -delete 2>/dev/null || true

0 commit comments

Comments
 (0)