From e3c9239fc1ead0fab110e82a9c3cf4ffb5add88d Mon Sep 17 00:00:00 2001 From: Martin Johannesson Date: Sat, 19 Sep 2026 22:35:29 +0200 Subject: [PATCH 1/4] Build the NIF on Windows with Zig MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The compile hook ran only on linux and darwin, so on Windows rebar3 built nothing and then failed on the missing priv/ezstd_nif.so artifact. A Windows hook — matched as "(win32|windows)", since OTP 25+ reports the arch as x86_64-pc-windows — now runs build_win32.ps1, which fetches zstd at the commit build_deps.sh pins, compiles it and the NIF with zig cc/c++ for x86_64-windows-gnu and links priv/ezstd_nif.dll. Zig carries its own compiler and Windows headers, so it is the only tool the build needs; the NIF API is reached through the callback table erl_nif.h uses on Windows, so no import library is involved. rebar.config.script renames the artifact to .dll there. A CI workflow runs the suite on Linux, macOS and Windows. --- .github/workflows/ci.yml | 29 +++++++++ README.md | 8 +++ build_win32.ps1 | 125 +++++++++++++++++++++++++++++++++++++++ rebar.config | 11 +++- rebar.config.script | 8 +++ src/ezstd.app.src | 2 + 6 files changed, 181 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 build_win32.ps1 create mode 100644 rebar.config.script diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..eaada1e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,29 @@ +name: CI + +on: + push: + pull_request: + +jobs: + test: + name: ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-14, windows-2022] + steps: + - uses: actions/checkout@v4 + - uses: erlef/setup-beam@v1 + with: + otp-version: "28" + rebar3-version: "3.25" + - uses: mlugg/setup-zig@v2 + if: runner.os == 'Windows' + with: + version: 0.16.0 + - run: rebar3 ct + - run: rebar3 clean + - name: The clean hook removed the NIF + shell: bash + run: test ! -e _build/default/lib/ezstd/priv/ezstd_nif.so && test ! -e _build/default/lib/ezstd/priv/ezstd_nif.dll && test ! -e priv/ezstd_nif.so && test ! -e priv/ezstd_nif.dll diff --git a/README.md b/README.md index 1381434..e16a669 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,14 @@ ContentCompressed = ezstd:compress_using_cdict(Plaintext, CDict), Plaintext = ezstd:decompress_using_ddict(ContentCompressed, DDict). ``` +## Building on Windows + +Linux and macOS build the NIF with `make`. Windows builds it with [Zig](https://ziglang.org/download/), +which brings its own C/C++ compiler and Windows headers, so `zig` on the `PATH` is the only +requirement (no Visual Studio, no MSYS2). `rebar.config` runs `build_win32.ps1` as the +Windows compile hook; it fetches zstd at the same pinned commit and produces +`priv/ezstd_nif.dll`. + ## Running tests ```sh diff --git a/build_win32.ps1 b/build_win32.ps1 new file mode 100644 index 0000000..06282a3 --- /dev/null +++ b/build_win32.ps1 @@ -0,0 +1,125 @@ +# Builds priv/ezstd_nif.dll on Windows with Zig (https://ziglang.org), which carries its +# own C/C++ toolchain and the mingw-w64 headers, so nothing else has to be installed: +# no Visual Studio, no MSYS2. rebar.config runs this as the `win32` compile hook, the +# way `make compile_nif` runs on Linux and macOS. +# +# zstd itself is fetched at the commit `ZSTD_SHA` in build_deps.sh names (one pin for +# every platform) and compiled from source into the DLL; there is no separate libzstd. +# +# Runs under Windows PowerShell 5.1 and PowerShell 7. `-Clean` removes what it built. + +[CmdletBinding()] +param([switch]$Clean) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2 + +$root = $PSScriptRoot +$build = Join-Path $root '_build\win32' +$zstdDir = Join-Path $root '_build\deps\zstd' +$zstdLib = Join-Path $zstdDir 'lib' +$objDir = Join-Path $build 'obj' + +# Where rebar3 (or Mix, which sets REBAR_BARE_COMPILER_OUTPUT_DIR) expects the NIF. +$privDir = if ($env:REBAR_BARE_COMPILER_OUTPUT_DIR) { + Join-Path $env:REBAR_BARE_COMPILER_OUTPUT_DIR 'priv' +} else { + Join-Path $root 'priv' +} +$output = Join-Path $privDir 'ezstd_nif.dll' + +if ($Clean) { + Remove-Item -Recurse -Force -ErrorAction SilentlyContinue $build + Remove-Item -Force -ErrorAction SilentlyContinue $output + exit 0 +} + +function Invoke-Checked { + param([string]$Exe, [string[]]$Arguments) + & $Exe @Arguments + if ($LASTEXITCODE -ne 0) { + throw "ezstd: '$Exe $($Arguments -join ' ')' failed with exit code $LASTEXITCODE" + } +} + +# --- toolchain ---------------------------------------------------------------------------- + +if (-not (Get-Command zig -ErrorAction SilentlyContinue)) { + throw "ezstd: building the NIF on Windows needs 'zig' on the PATH (https://ziglang.org/download/)" +} + +$target = switch ($env:PROCESSOR_ARCHITECTURE) { + 'ARM64' { 'aarch64-windows-gnu' } + default { 'x86_64-windows-gnu' } +} + +# erl_nif.h: Mix hands ERTS_INCLUDE_DIR to rebar3, rebar3 hands ERLANG_ROOT_DIR and +# ERLANG_ERTS_VER to its hooks, and a script run by hand asks erl. (The format string is +# an atom because PowerShell 5.1 strips double quotes out of a native command line.) +$ertsInclude = $env:ERTS_INCLUDE_DIR +if (-not $ertsInclude -and $env:ERLANG_ROOT_DIR -and $env:ERLANG_ERTS_VER) { + $ertsInclude = Join-Path $env:ERLANG_ROOT_DIR "erts-$env:ERLANG_ERTS_VER\include" +} +if (-not $ertsInclude) { + if (-not (Get-Command erl -ErrorAction SilentlyContinue)) { + throw "ezstd: neither ERTS_INCLUDE_DIR nor 'erl' on the PATH; cannot find erl_nif.h" + } + $ertsInclude = & erl -noshell -eval 'io:format(''~s/erts-~s/include'', [code:root_dir(), erlang:system_info(version)]), halt().' +} +if (-not (Test-Path (Join-Path $ertsInclude 'erl_nif.h'))) { + throw "ezstd: no erl_nif.h under '$ertsInclude'" +} + +# --- zstd source --------------------------------------------------------------------------- + +$pin = Select-String -Path (Join-Path $root 'build_deps.sh') -Pattern '^ZSTD_SHA="([0-9a-f]+)"' +if (-not $pin) { throw 'ezstd: no ZSTD_SHA in build_deps.sh' } +$zstdSha = $pin.Matches[0].Groups[1].Value + +if (-not (Test-Path (Join-Path $zstdLib 'zstd.h'))) { + Write-Host "ezstd: fetching zstd $zstdSha" + New-Item -ItemType Directory -Force $zstdDir | Out-Null + Push-Location $zstdDir + try { + Invoke-Checked git @('init', '-q') + Invoke-Checked git @('remote', 'add', 'origin', 'https://github.com/facebook/zstd.git') + Invoke-Checked git @('fetch', '-q', '--depth', '1', 'origin', $zstdSha) + Invoke-Checked git @('checkout', '-q', 'FETCH_HEAD') + } finally { + Pop-Location + } +} + +# --- compile ------------------------------------------------------------------------------- + +New-Item -ItemType Directory -Force $objDir, $privDir | Out-Null + +# The same library the Makefile's `lib-release` produces: common, compress, decompress and +# dictBuilder, no legacy formats, no multithreading. The assembly Huffman decoder is left +# out (`ZSTD_DISABLE_ASM`) so the whole thing is plain C for one compiler. +$cflags = @( + '-target', $target, '-O3', '-DNDEBUG', + '-DZSTD_DISABLE_ASM=1', '-DZSTD_LEGACY_SUPPORT=0', '-DXXH_NAMESPACE=ZSTD_', + "-I$zstdLib" +) +$objects = @() +foreach ($sub in 'common', 'compress', 'decompress', 'dictBuilder') { + foreach ($src in Get-ChildItem (Join-Path $zstdLib $sub) -Filter '*.c') { + $obj = Join-Path $objDir ($src.BaseName + '.o') + $objects += $obj + if ((Test-Path $obj) -and (Get-Item $obj).LastWriteTime -ge $src.LastWriteTime) { continue } + Invoke-Checked zig (@('cc') + $cflags + @('-c', $src.FullName, '-o', $obj)) + } +} + +Write-Host "ezstd: linking $output ($target)" +$cxxflags = @( + '-target', $target, '-O3', '-DNDEBUG', '-std=c++11', '-fno-exceptions', '-fno-rtti', + '-Wall', '-Wextra', '-Wno-missing-field-initializers', '-Wno-nullability-completeness', + "-I$ertsInclude", "-I$zstdLib", '-shared', '-s', '-o', $output, + (Join-Path $root 'c_src\ezstd_nif.cc'), (Join-Path $root 'c_src\nif_utils.cc') +) +Invoke-Checked zig (@('c++') + $cxxflags + $objects) + +# The linker leaves an import library and a debug file beside the DLL; neither is loaded. +Remove-Item -Force -ErrorAction SilentlyContinue (Join-Path $privDir 'ezstd_nif.lib'), (Join-Path $privDir 'ezstd_nif.pdb') diff --git a/rebar.config b/rebar.config index 72be085..6f73ef4 100644 --- a/rebar.config +++ b/rebar.config @@ -1,6 +1,13 @@ -{pre_hooks, [{"(linux|darwin)", compile, "make compile_nif"}]}. -{post_hooks, [{"(linux|darwin)", clean, "make clean_nif"}]}. +{pre_hooks, [ + {"(linux|darwin)", compile, "make compile_nif"}, + {"(win32|windows)", compile, "powershell -NoProfile -ExecutionPolicy Bypass -File build_win32.ps1"} +]}. +{post_hooks, [ + {"(linux|darwin)", clean, "make clean_nif"}, + {"(win32|windows)", clean, "powershell -NoProfile -ExecutionPolicy Bypass -File build_win32.ps1 -Clean"} +]}. +%% rebar.config.script makes this priv/ezstd_nif.dll on Windows. {artifacts, ["priv/ezstd_nif.so"]}. {project_plugins, [rebar3_hex, rebar3_ex_doc]}. diff --git a/rebar.config.script b/rebar.config.script new file mode 100644 index 0000000..8505df8 --- /dev/null +++ b/rebar.config.script @@ -0,0 +1,8 @@ +%% The NIF is a .so on Linux and macOS and a .dll on Windows; `artifacts` in rebar.config +%% is a plain list, so the Windows name is substituted here. +case os:type() of + {win32, _} -> + lists:keyreplace(artifacts, 1, CONFIG, {artifacts, ["priv/ezstd_nif.dll"]}); + _ -> + CONFIG +end. diff --git a/src/ezstd.app.src b/src/ezstd.app.src index e42e629..ad27a8b 100644 --- a/src/ezstd.app.src +++ b/src/ezstd.app.src @@ -15,8 +15,10 @@ "*.MD", "Makefile", "build_deps.sh", + "build_win32.ps1", "bench.sh", "rebar.config", + "rebar.config.script", "rebar.lock", "scripts/*.erl", "test/*.erl", From 0d12697de0e05f5ea0fd1a3d342cffb977fb2885 Mon Sep 17 00:00:00 2001 From: Martin Johannesson Date: Sun, 20 Sep 2026 00:02:32 +0200 Subject: [PATCH 2/4] Prove the Windows build on Travis, where this project's CI runs The GitHub Actions workflow was the wrong place for it: this repository builds on Travis. Travis has no Erlang language support on Windows, so the job is a shell job that installs Erlang and Zig from Chocolatey and runs rebar3 compile and ct through the escript. The Linux jobs are unchanged. --- .github/workflows/ci.yml | 29 ----------------------------- .travis.yml | 16 +++++++++++++++- 2 files changed, 15 insertions(+), 30 deletions(-) delete mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index eaada1e..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: CI - -on: - push: - pull_request: - -jobs: - test: - name: ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-24.04, macos-14, windows-2022] - steps: - - uses: actions/checkout@v4 - - uses: erlef/setup-beam@v1 - with: - otp-version: "28" - rebar3-version: "3.25" - - uses: mlugg/setup-zig@v2 - if: runner.os == 'Windows' - with: - version: 0.16.0 - - run: rebar3 ct - - run: rebar3 clean - - name: The clean hook removed the NIF - shell: bash - run: test ! -e _build/default/lib/ezstd/priv/ezstd_nif.so && test ! -e _build/default/lib/ezstd/priv/ezstd_nif.dll && test ! -e priv/ezstd_nif.so && test ! -e priv/ezstd_nif.dll diff --git a/.travis.yml b/.travis.yml index 9796127..1650fa2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,3 @@ - language: erlang matrix: @@ -16,6 +15,21 @@ matrix: dist: noble otp_release: 26.1.1 + # Windows has no Erlang language support on Travis, so the job is a shell job that + # installs Erlang and Zig from Chocolatey. Zig is what build_win32.ps1 compiles the + # NIF with; nothing else is needed. + - os: windows + language: shell + before_install: + - choco install -y --no-progress erlang zig + - export PATH="$PATH:$(ls -d '/c/Program Files/Erlang OTP'* '/c/Program Files/erl'* 2>/dev/null | head -1)/bin" + - erl -noshell -eval 'io:format("~s~n",[erlang:system_info(system_architecture)]),halt().' + - zig version + script: + - ./rebar3 compile + - ./rebar3 ct + after_success: true + before_script: - if [[ $TRAVIS_OS_NAME == linux ]]; then sudo apt-get -y update || true ; fi From efce7b7e70860e91b4d8b0a9c04fce4cd8a74838 Mon Sep 17 00:00:00 2001 From: Martin Johannesson Date: Sun, 20 Sep 2026 00:19:44 +0200 Subject: [PATCH 3/4] Build libc++ quietly before the link, so the log stays under Travis's limit The link needs libc++, which Zig compiles from its bundled sources the first time it links it for a target, echoing some forty thousand warning lines from those sources while it does; Travis kills a job whose log passes 4 MB. The script now builds it once on a two-line file with the same flags, output kept unless it fails, and the real link then finds it in the cache and prints only its own diagnostics. --- build_win32.ps1 | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/build_win32.ps1 b/build_win32.ps1 index 06282a3..fffa8a3 100644 --- a/build_win32.ps1 +++ b/build_win32.ps1 @@ -1,6 +1,6 @@ # Builds priv/ezstd_nif.dll on Windows with Zig (https://ziglang.org), which carries its # own C/C++ toolchain and the mingw-w64 headers, so nothing else has to be installed: -# no Visual Studio, no MSYS2. rebar.config runs this as the `win32` compile hook, the +# no Visual Studio, no MSYS2. rebar.config runs this as the Windows compile hook, the # way `make compile_nif` runs on Linux and macOS. # # zstd itself is fetched at the commit `ZSTD_SHA` in build_deps.sh names (one pin for @@ -42,6 +42,20 @@ function Invoke-Checked { } } +# Runs a command with its output captured, and shows that output only if it fails. +function Invoke-Quiet { + param([string]$Exe, [string[]]$Arguments) + $out = Join-Path $build 'quiet.out' + $err = Join-Path $build 'quiet.err' + $quoted = $Arguments | ForEach-Object { if ($_ -match '\s') { '"' + $_ + '"' } else { $_ } } + $p = Start-Process -FilePath $Exe -ArgumentList $quoted -NoNewWindow -Wait -PassThru ` + -RedirectStandardOutput $out -RedirectStandardError $err + if ($p.ExitCode -ne 0) { + Get-Content $out, $err -ErrorAction SilentlyContinue | Write-Host + throw "ezstd: '$Exe $($Arguments -join ' ')' failed with exit code $($p.ExitCode)" + } +} + # --- toolchain ---------------------------------------------------------------------------- if (-not (Get-Command zig -ErrorAction SilentlyContinue)) { @@ -112,14 +126,28 @@ foreach ($sub in 'common', 'compress', 'decompress', 'dictBuilder') { } } +# The NIF is C++ (std::unique_ptr, new[]), so the link needs libc++, which Zig compiles +# from its bundled sources the first time it links it for a target and a set of flags. +# While it does, it echoes some forty thousand warning lines from those sources, none of +# them ours, and Travis kills a job whose log passes 4 MB. So it is built once here, on a +# one-line file with the same flags, with the output kept unless it fails; the real link +# below then finds it in the cache and prints only its own diagnostics. +$cxxflags = @('-target', $target, '-O3', '-DNDEBUG', '-std=c++11', '-fno-exceptions', '-fno-rtti') +$warm = Join-Path $build 'libcxx_warmup.cc' +Set-Content -Path $warm -Encoding ascii -Value @( + '#include ', + 'extern "C" int ezstd_warmup() { std::unique_ptr p(new int[1]); return p ? 0 : 1; }' +) +Write-Host "ezstd: preparing libc++ for $target" +Invoke-Quiet zig (@('c++') + $cxxflags + @('-shared', '-s', '-o', (Join-Path $build 'libcxx_warmup.dll'), $warm)) + Write-Host "ezstd: linking $output ($target)" -$cxxflags = @( - '-target', $target, '-O3', '-DNDEBUG', '-std=c++11', '-fno-exceptions', '-fno-rtti', +$linkflags = $cxxflags + @( '-Wall', '-Wextra', '-Wno-missing-field-initializers', '-Wno-nullability-completeness', "-I$ertsInclude", "-I$zstdLib", '-shared', '-s', '-o', $output, (Join-Path $root 'c_src\ezstd_nif.cc'), (Join-Path $root 'c_src\nif_utils.cc') ) -Invoke-Checked zig (@('c++') + $cxxflags + $objects) +Invoke-Checked zig (@('c++') + $linkflags + $objects) # The linker leaves an import library and a debug file beside the DLL; neither is loaded. Remove-Item -Force -ErrorAction SilentlyContinue (Join-Path $privDir 'ezstd_nif.lib'), (Join-Path $privDir 'ezstd_nif.pdb') From 19eaefd72fa6c80100d0e10fc134fd4ce08ae080 Mon Sep 17 00:00:00 2001 From: Martin Johannesson Date: Sun, 20 Sep 2026 00:37:14 +0200 Subject: [PATCH 4/4] Say something every half minute while libc++ builds Travis stops a job that has printed nothing for ten minutes, and on its Windows VM the quiet libc++ build takes longer than that. The quiet runner now reports every thirty seconds that it is still going, and the zstd object compiles announce each directory. --- build_win32.ps1 | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/build_win32.ps1 b/build_win32.ps1 index fffa8a3..f8d9049 100644 --- a/build_win32.ps1 +++ b/build_win32.ps1 @@ -42,14 +42,21 @@ function Invoke-Checked { } } -# Runs a command with its output captured, and shows that output only if it fails. +# Runs a command with its output captured, and shows that output only if it fails. A +# line every half minute says it is still running, for CI that stops a job that has been +# silent too long. function Invoke-Quiet { - param([string]$Exe, [string[]]$Arguments) + param([string]$Exe, [string[]]$Arguments, [string]$What, [int]$HeartbeatSeconds = 30) $out = Join-Path $build 'quiet.out' $err = Join-Path $build 'quiet.err' $quoted = $Arguments | ForEach-Object { if ($_ -match '\s') { '"' + $_ + '"' } else { $_ } } - $p = Start-Process -FilePath $Exe -ArgumentList $quoted -NoNewWindow -Wait -PassThru ` + $p = Start-Process -FilePath $Exe -ArgumentList $quoted -NoNewWindow -PassThru ` -RedirectStandardOutput $out -RedirectStandardError $err + $null = $p.Handle # without this, Windows PowerShell 5.1 reports no ExitCode + $started = Get-Date + while (-not $p.WaitForExit($HeartbeatSeconds * 1000)) { + Write-Host ("ezstd: {0}, {1:n0}s so far" -f $What, ((Get-Date) - $started).TotalSeconds) + } if ($p.ExitCode -ne 0) { Get-Content $out, $err -ErrorAction SilentlyContinue | Write-Host throw "ezstd: '$Exe $($Arguments -join ' ')' failed with exit code $($p.ExitCode)" @@ -118,7 +125,9 @@ $cflags = @( ) $objects = @() foreach ($sub in 'common', 'compress', 'decompress', 'dictBuilder') { - foreach ($src in Get-ChildItem (Join-Path $zstdLib $sub) -Filter '*.c') { + $sources = Get-ChildItem (Join-Path $zstdLib $sub) -Filter '*.c' + Write-Host "ezstd: compiling zstd $sub ($($sources.Count) files)" + foreach ($src in $sources) { $obj = Join-Path $objDir ($src.BaseName + '.o') $objects += $obj if ((Test-Path $obj) -and (Get-Item $obj).LastWriteTime -ge $src.LastWriteTime) { continue } @@ -139,7 +148,7 @@ Set-Content -Path $warm -Encoding ascii -Value @( 'extern "C" int ezstd_warmup() { std::unique_ptr p(new int[1]); return p ? 0 : 1; }' ) Write-Host "ezstd: preparing libc++ for $target" -Invoke-Quiet zig (@('c++') + $cxxflags + @('-shared', '-s', '-o', (Join-Path $build 'libcxx_warmup.dll'), $warm)) +Invoke-Quiet zig (@('c++') + $cxxflags + @('-shared', '-s', '-o', (Join-Path $build 'libcxx_warmup.dll'), $warm)) -What 'still building libc++' Write-Host "ezstd: linking $output ($target)" $linkflags = $cxxflags + @(