diff --git a/.travis.yml b/.travis.yml index 9796127..1650fa2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,3 @@ - language: erlang matrix: @@ -16,6 +15,21 @@ matrix: dist: noble otp_release: 26.1.1 + # Windows has no Erlang language support on Travis, so the job is a shell job that + # installs Erlang and Zig from Chocolatey. Zig is what build_win32.ps1 compiles the + # NIF with; nothing else is needed. + - os: windows + language: shell + before_install: + - choco install -y --no-progress erlang zig + - export PATH="$PATH:$(ls -d '/c/Program Files/Erlang OTP'* '/c/Program Files/erl'* 2>/dev/null | head -1)/bin" + - erl -noshell -eval 'io:format("~s~n",[erlang:system_info(system_architecture)]),halt().' + - zig version + script: + - ./rebar3 compile + - ./rebar3 ct + after_success: true + before_script: - if [[ $TRAVIS_OS_NAME == linux ]]; then sudo apt-get -y update || true ; fi diff --git a/README.md b/README.md index 1381434..e16a669 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,14 @@ ContentCompressed = ezstd:compress_using_cdict(Plaintext, CDict), Plaintext = ezstd:decompress_using_ddict(ContentCompressed, DDict). ``` +## Building on Windows + +Linux and macOS build the NIF with `make`. Windows builds it with [Zig](https://ziglang.org/download/), +which brings its own C/C++ compiler and Windows headers, so `zig` on the `PATH` is the only +requirement (no Visual Studio, no MSYS2). `rebar.config` runs `build_win32.ps1` as the +Windows compile hook; it fetches zstd at the same pinned commit and produces +`priv/ezstd_nif.dll`. + ## Running tests ```sh diff --git a/build_win32.ps1 b/build_win32.ps1 new file mode 100644 index 0000000..f8d9049 --- /dev/null +++ b/build_win32.ps1 @@ -0,0 +1,162 @@ +# Builds priv/ezstd_nif.dll on Windows with Zig (https://ziglang.org), which carries its +# own C/C++ toolchain and the mingw-w64 headers, so nothing else has to be installed: +# no Visual Studio, no MSYS2. rebar.config runs this as the Windows compile hook, the +# way `make compile_nif` runs on Linux and macOS. +# +# zstd itself is fetched at the commit `ZSTD_SHA` in build_deps.sh names (one pin for +# every platform) and compiled from source into the DLL; there is no separate libzstd. +# +# Runs under Windows PowerShell 5.1 and PowerShell 7. `-Clean` removes what it built. + +[CmdletBinding()] +param([switch]$Clean) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2 + +$root = $PSScriptRoot +$build = Join-Path $root '_build\win32' +$zstdDir = Join-Path $root '_build\deps\zstd' +$zstdLib = Join-Path $zstdDir 'lib' +$objDir = Join-Path $build 'obj' + +# Where rebar3 (or Mix, which sets REBAR_BARE_COMPILER_OUTPUT_DIR) expects the NIF. +$privDir = if ($env:REBAR_BARE_COMPILER_OUTPUT_DIR) { + Join-Path $env:REBAR_BARE_COMPILER_OUTPUT_DIR 'priv' +} else { + Join-Path $root 'priv' +} +$output = Join-Path $privDir 'ezstd_nif.dll' + +if ($Clean) { + Remove-Item -Recurse -Force -ErrorAction SilentlyContinue $build + Remove-Item -Force -ErrorAction SilentlyContinue $output + exit 0 +} + +function Invoke-Checked { + param([string]$Exe, [string[]]$Arguments) + & $Exe @Arguments + if ($LASTEXITCODE -ne 0) { + throw "ezstd: '$Exe $($Arguments -join ' ')' failed with exit code $LASTEXITCODE" + } +} + +# Runs a command with its output captured, and shows that output only if it fails. A +# line every half minute says it is still running, for CI that stops a job that has been +# silent too long. +function Invoke-Quiet { + param([string]$Exe, [string[]]$Arguments, [string]$What, [int]$HeartbeatSeconds = 30) + $out = Join-Path $build 'quiet.out' + $err = Join-Path $build 'quiet.err' + $quoted = $Arguments | ForEach-Object { if ($_ -match '\s') { '"' + $_ + '"' } else { $_ } } + $p = Start-Process -FilePath $Exe -ArgumentList $quoted -NoNewWindow -PassThru ` + -RedirectStandardOutput $out -RedirectStandardError $err + $null = $p.Handle # without this, Windows PowerShell 5.1 reports no ExitCode + $started = Get-Date + while (-not $p.WaitForExit($HeartbeatSeconds * 1000)) { + Write-Host ("ezstd: {0}, {1:n0}s so far" -f $What, ((Get-Date) - $started).TotalSeconds) + } + if ($p.ExitCode -ne 0) { + Get-Content $out, $err -ErrorAction SilentlyContinue | Write-Host + throw "ezstd: '$Exe $($Arguments -join ' ')' failed with exit code $($p.ExitCode)" + } +} + +# --- toolchain ---------------------------------------------------------------------------- + +if (-not (Get-Command zig -ErrorAction SilentlyContinue)) { + throw "ezstd: building the NIF on Windows needs 'zig' on the PATH (https://ziglang.org/download/)" +} + +$target = switch ($env:PROCESSOR_ARCHITECTURE) { + 'ARM64' { 'aarch64-windows-gnu' } + default { 'x86_64-windows-gnu' } +} + +# erl_nif.h: Mix hands ERTS_INCLUDE_DIR to rebar3, rebar3 hands ERLANG_ROOT_DIR and +# ERLANG_ERTS_VER to its hooks, and a script run by hand asks erl. (The format string is +# an atom because PowerShell 5.1 strips double quotes out of a native command line.) +$ertsInclude = $env:ERTS_INCLUDE_DIR +if (-not $ertsInclude -and $env:ERLANG_ROOT_DIR -and $env:ERLANG_ERTS_VER) { + $ertsInclude = Join-Path $env:ERLANG_ROOT_DIR "erts-$env:ERLANG_ERTS_VER\include" +} +if (-not $ertsInclude) { + if (-not (Get-Command erl -ErrorAction SilentlyContinue)) { + throw "ezstd: neither ERTS_INCLUDE_DIR nor 'erl' on the PATH; cannot find erl_nif.h" + } + $ertsInclude = & erl -noshell -eval 'io:format(''~s/erts-~s/include'', [code:root_dir(), erlang:system_info(version)]), halt().' +} +if (-not (Test-Path (Join-Path $ertsInclude 'erl_nif.h'))) { + throw "ezstd: no erl_nif.h under '$ertsInclude'" +} + +# --- zstd source --------------------------------------------------------------------------- + +$pin = Select-String -Path (Join-Path $root 'build_deps.sh') -Pattern '^ZSTD_SHA="([0-9a-f]+)"' +if (-not $pin) { throw 'ezstd: no ZSTD_SHA in build_deps.sh' } +$zstdSha = $pin.Matches[0].Groups[1].Value + +if (-not (Test-Path (Join-Path $zstdLib 'zstd.h'))) { + Write-Host "ezstd: fetching zstd $zstdSha" + New-Item -ItemType Directory -Force $zstdDir | Out-Null + Push-Location $zstdDir + try { + Invoke-Checked git @('init', '-q') + Invoke-Checked git @('remote', 'add', 'origin', 'https://github.com/facebook/zstd.git') + Invoke-Checked git @('fetch', '-q', '--depth', '1', 'origin', $zstdSha) + Invoke-Checked git @('checkout', '-q', 'FETCH_HEAD') + } finally { + Pop-Location + } +} + +# --- compile ------------------------------------------------------------------------------- + +New-Item -ItemType Directory -Force $objDir, $privDir | Out-Null + +# The same library the Makefile's `lib-release` produces: common, compress, decompress and +# dictBuilder, no legacy formats, no multithreading. The assembly Huffman decoder is left +# out (`ZSTD_DISABLE_ASM`) so the whole thing is plain C for one compiler. +$cflags = @( + '-target', $target, '-O3', '-DNDEBUG', + '-DZSTD_DISABLE_ASM=1', '-DZSTD_LEGACY_SUPPORT=0', '-DXXH_NAMESPACE=ZSTD_', + "-I$zstdLib" +) +$objects = @() +foreach ($sub in 'common', 'compress', 'decompress', 'dictBuilder') { + $sources = Get-ChildItem (Join-Path $zstdLib $sub) -Filter '*.c' + Write-Host "ezstd: compiling zstd $sub ($($sources.Count) files)" + foreach ($src in $sources) { + $obj = Join-Path $objDir ($src.BaseName + '.o') + $objects += $obj + if ((Test-Path $obj) -and (Get-Item $obj).LastWriteTime -ge $src.LastWriteTime) { continue } + Invoke-Checked zig (@('cc') + $cflags + @('-c', $src.FullName, '-o', $obj)) + } +} + +# The NIF is C++ (std::unique_ptr, new[]), so the link needs libc++, which Zig compiles +# from its bundled sources the first time it links it for a target and a set of flags. +# While it does, it echoes some forty thousand warning lines from those sources, none of +# them ours, and Travis kills a job whose log passes 4 MB. So it is built once here, on a +# one-line file with the same flags, with the output kept unless it fails; the real link +# below then finds it in the cache and prints only its own diagnostics. +$cxxflags = @('-target', $target, '-O3', '-DNDEBUG', '-std=c++11', '-fno-exceptions', '-fno-rtti') +$warm = Join-Path $build 'libcxx_warmup.cc' +Set-Content -Path $warm -Encoding ascii -Value @( + '#include ', + 'extern "C" int ezstd_warmup() { std::unique_ptr p(new int[1]); return p ? 0 : 1; }' +) +Write-Host "ezstd: preparing libc++ for $target" +Invoke-Quiet zig (@('c++') + $cxxflags + @('-shared', '-s', '-o', (Join-Path $build 'libcxx_warmup.dll'), $warm)) -What 'still building libc++' + +Write-Host "ezstd: linking $output ($target)" +$linkflags = $cxxflags + @( + '-Wall', '-Wextra', '-Wno-missing-field-initializers', '-Wno-nullability-completeness', + "-I$ertsInclude", "-I$zstdLib", '-shared', '-s', '-o', $output, + (Join-Path $root 'c_src\ezstd_nif.cc'), (Join-Path $root 'c_src\nif_utils.cc') +) +Invoke-Checked zig (@('c++') + $linkflags + $objects) + +# The linker leaves an import library and a debug file beside the DLL; neither is loaded. +Remove-Item -Force -ErrorAction SilentlyContinue (Join-Path $privDir 'ezstd_nif.lib'), (Join-Path $privDir 'ezstd_nif.pdb') diff --git a/rebar.config b/rebar.config index 72be085..6f73ef4 100644 --- a/rebar.config +++ b/rebar.config @@ -1,6 +1,13 @@ -{pre_hooks, [{"(linux|darwin)", compile, "make compile_nif"}]}. -{post_hooks, [{"(linux|darwin)", clean, "make clean_nif"}]}. +{pre_hooks, [ + {"(linux|darwin)", compile, "make compile_nif"}, + {"(win32|windows)", compile, "powershell -NoProfile -ExecutionPolicy Bypass -File build_win32.ps1"} +]}. +{post_hooks, [ + {"(linux|darwin)", clean, "make clean_nif"}, + {"(win32|windows)", clean, "powershell -NoProfile -ExecutionPolicy Bypass -File build_win32.ps1 -Clean"} +]}. +%% rebar.config.script makes this priv/ezstd_nif.dll on Windows. {artifacts, ["priv/ezstd_nif.so"]}. {project_plugins, [rebar3_hex, rebar3_ex_doc]}. diff --git a/rebar.config.script b/rebar.config.script new file mode 100644 index 0000000..8505df8 --- /dev/null +++ b/rebar.config.script @@ -0,0 +1,8 @@ +%% The NIF is a .so on Linux and macOS and a .dll on Windows; `artifacts` in rebar.config +%% is a plain list, so the Windows name is substituted here. +case os:type() of + {win32, _} -> + lists:keyreplace(artifacts, 1, CONFIG, {artifacts, ["priv/ezstd_nif.dll"]}); + _ -> + CONFIG +end. diff --git a/src/ezstd.app.src b/src/ezstd.app.src index e42e629..ad27a8b 100644 --- a/src/ezstd.app.src +++ b/src/ezstd.app.src @@ -15,8 +15,10 @@ "*.MD", "Makefile", "build_deps.sh", + "build_win32.ps1", "bench.sh", "rebar.config", + "rebar.config.script", "rebar.lock", "scripts/*.erl", "test/*.erl",