diff --git a/Cargo.lock b/Cargo.lock index bd0e44db2..16797e9a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -246,6 +246,15 @@ dependencies = [ "serde", ] +[[package]] +name = "built" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c0e531d93d39c34eef561e929e8a7f86d77a5af08aac4f6d6e39976c51858e9" +dependencies = [ + "git2 0.21.0", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -1026,6 +1035,18 @@ dependencies = [ "url", ] +[[package]] +name = "git2" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e" +dependencies = [ + "bitflags", + "libc", + "libgit2-sys", + "log", +] + [[package]] name = "globset" version = "0.4.18" @@ -1125,12 +1146,13 @@ name = "hf-kernel-builder" version = "0.17.0-dev0" dependencies = [ "base32", + "built", "clap", "clap-markdown", "clap_complete", "dirs", "eyre", - "git2", + "git2 0.20.4", "hf-hub", "indicatif", "itertools 0.13.0", @@ -1693,9 +1715,9 @@ checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libgit2-sys" -version = "0.18.3+1.9.2" +version = "0.18.5+1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9b3acc4b91781bb0b3386669d325163746af5f6e4f73e6d2d630e09a35f3487" +checksum = "005d6ae6eac1912906073e069f7db60b1fa98e052a68227824afe3e3a1c59ca2" dependencies = [ "cc", "libc", diff --git a/docs/source/builder-cli.md b/docs/source/builder-cli.md index 6c876b709..961358a45 100644 --- a/docs/source/builder-cli.md +++ b/docs/source/builder-cli.md @@ -244,6 +244,8 @@ Generate CMake files for a kernel extension build * `-f`, `--force` — Force-overwrite existing files * `--unique-id ` — This is an optional unique identifier that is suffixed to the kernel name to avoid name collisions. (e.g. Git SHA) +* `--kernel-sha ` — Full commit SHA of the kernel source, recorded in the build metadata. When absent, it is detected from the kernel's git repository (used by Nix builds where the source has no `.git`) +* `--kernel-dirty` — Mark the kernel source as having uncommitted changes in the build metadata. Only meaningful together with `--kernel-sha` diff --git a/docs/source/kernel-requirements.md b/docs/source/kernel-requirements.md index c32347713..cc54f7160 100644 --- a/docs/source/kernel-requirements.md +++ b/docs/source/kernel-requirements.md @@ -81,6 +81,31 @@ metadata. Currently the following top-level keys are supported: - `digest` (`Digest`, required): hash digest of the kernel files. - `python-depends` (`list[str]`, optional): list of Python dependencies from a curated set of Python dependencies. +- `provenance` (`dict`, optional): provenance of the build, used to flag + non-reproducible (dirty) builds. It contains two optional sub-objects: + - `kernel-builder`: the `kernel-builder` that produced the build, with its + `version` (`str`), the `sha` (`str`) of the `kernel-builder` source it was + built from (when known), and a `dirty` (`bool`) flag that is `true` when + `kernel-builder` was built from a source tree with uncommitted changes. + - `kernel`: the kernel source that was built, with its commit `sha` (`str`) + and a `dirty` (`bool`) flag that is `true` when the kernel source had + uncommitted changes. + + When either `dirty` flag is set, the kernel was built from uncommitted + sources and cannot be reliably reproduced. + + > **Note:** For Nix builds, dirtiness follows Nix's flake tree status, which + > also counts **untracked** files (including an uncommitted `flake.lock`). + > Commit your `flake.lock` (and avoid stray untracked files) so that clean + > builds are not flagged as dirty. Local `create-pyproject` runs only + > consider changes to tracked files. + + > **Note:** The kernel `sha`/`dirty` are captured at the moment + > `create-pyproject` runs, so they describe the source tree as it was *then*. + > Running `create-pyproject` and committing afterwards is bad practice: the + > recorded provenance keeps pointing at the pre-commit state (a stale `sha`, + > and `dirty: true` if the tree was dirty) even though the committed source + > differs. Generate the metadata from the final, committed source instead. Example `metadata.json`: diff --git a/flake.nix b/flake.nix index aa4b29c57..cc3993a75 100644 --- a/flake.nix +++ b/flake.nix @@ -27,9 +27,16 @@ rust-overlay, }: let + # Git provenance (`{ sha, dirty }` or `null`) of the `kernel-builder` + # flake itself, so it can be burned into the binary at build time and + # recorded in the build metadata of the kernels it builds. + builderProvenance = import ./nix-builder/lib/flake-provenance.nix { + inherit (nixpkgs) lib; + } self; + inherit (import ./nix-builder/lib/build-sets.nix { - inherit nixpkgs rust-overlay; + inherit nixpkgs rust-overlay builderProvenance; }) mkBuildSets partitionBuildSetsBySystem diff --git a/kernel-builder/Cargo.toml b/kernel-builder/Cargo.toml index 1763a6eb7..59a92fdfe 100644 --- a/kernel-builder/Cargo.toml +++ b/kernel-builder/Cargo.toml @@ -37,6 +37,7 @@ reqwest = { version = "0.13", features = ["blocking"] } walkdir = "2" [build-dependencies] +built = { version = "0.8", features = ["git2"] } minijinja-embed = "2.5" [dev-dependencies] diff --git a/kernel-builder/build.rs b/kernel-builder/build.rs index 3acdfe018..6347861a2 100644 --- a/kernel-builder/build.rs +++ b/kernel-builder/build.rs @@ -1,3 +1,4 @@ fn main() { minijinja_embed::embed_templates!("src/pyproject/templates"); + built::write_built_file().expect("Failed to acquire build-time information"); } diff --git a/kernel-builder/src/main.rs b/kernel-builder/src/main.rs index 5e4783a07..9f544214e 100644 --- a/kernel-builder/src/main.rs +++ b/kernel-builder/src/main.rs @@ -50,6 +50,13 @@ use validate_builds::check_builds; use crate::util::parse_and_validate_compat; +/// Build-time metadata gathered by the [`built`](https://crates.io/crates/built) +/// crate (see `build.rs`), including the `kernel-builder` git provenance that is +/// recorded in the build metadata of the kernels it builds. +mod built_info { + include!(concat!(env!("OUT_DIR"), "/built.rs")); +} + #[derive(Args, Debug)] struct NixArgs { /// Maximum number of parallel Nix build jobs. @@ -174,6 +181,17 @@ enum Commands { /// kernel name to avoid name collisions. (e.g. Git SHA) #[arg(long)] unique_id: Option, + + /// Full commit SHA of the kernel source, recorded in the build + /// metadata. When absent, it is detected from the kernel's git + /// repository (used by Nix builds where the source has no `.git`). + #[arg(long)] + kernel_sha: Option, + + /// Mark the kernel source as having uncommitted changes in the build + /// metadata. Only meaningful together with `--kernel-sha`. + #[arg(long)] + kernel_dirty: bool, }, /// Spawn a kernel development shell. @@ -365,7 +383,16 @@ fn main() -> Result<()> { force, target_dir, unique_id, - } => create_pyproject(kernel_dir, target_dir, force, unique_id), + kernel_sha, + kernel_dirty, + } => create_pyproject( + kernel_dir, + target_dir, + force, + unique_id, + kernel_sha, + kernel_dirty, + ), Commands::Devshell { kernel_dir, variant, diff --git a/kernel-builder/src/pyproject/common.rs b/kernel-builder/src/pyproject/common.rs index 5443f0b5e..87ea2fdf5 100644 --- a/kernel-builder/src/pyproject/common.rs +++ b/kernel-builder/src/pyproject/common.rs @@ -4,7 +4,7 @@ use eyre::Result; use itertools::Itertools; use kernels_data::config::{Backend, Build}; -use kernels_data::metadata::Metadata; +use kernels_data::metadata::{GitHash, KernelBuilderVersion, Metadata, Provenance}; use crate::pyproject::ops_identifier::KernelIdentifier; use crate::pyproject::FileSet; @@ -20,16 +20,34 @@ pub fn write_compat_py(file_set: &mut FileSet) -> Result<()> { Ok(()) } +/// `kernel-builder` provenance baked in at compile time. +/// +/// The git provenance is gathered by the `built` crate (see `build.rs`). In +/// build sandboxes without a `.git` (e.g. Nix), the derivation supplies it via +/// `built`'s `BUILT_OVERRIDE_hf_kernel_builder_GIT_*` environment variables. +pub(crate) fn kernel_builder_version() -> KernelBuilderVersion { + let git = crate::built_info::GIT_COMMIT_HASH.map(|sha| GitHash { + sha: sha.to_owned(), + dirty: crate::built_info::GIT_DIRTY.unwrap_or(false), + }); + KernelBuilderVersion { + version: env!("CARGO_PKG_VERSION").to_owned(), + git, + } +} + pub fn write_metadata( build: &Build, kernel_id: &KernelIdentifier, + provenance: Option<&Provenance>, file_set: &mut FileSet, ) -> Result<()> { for backend in &Backend::all() { let writer = file_set.entry(format!("metadata-{backend}.json")); - let metadata = + let mut metadata = Metadata::for_backend(build, kernel_id.to_string_for_backend(*backend), *backend)?; + metadata.provenance = provenance.cloned(); serde_json::to_writer_pretty(writer, &metadata)?; } diff --git a/kernel-builder/src/pyproject/mod.rs b/kernel-builder/src/pyproject/mod.rs index 5582d5b6a..1c7d5376c 100644 --- a/kernel-builder/src/pyproject/mod.rs +++ b/kernel-builder/src/pyproject/mod.rs @@ -6,6 +6,7 @@ use std::{ use eyre::{bail, Result}; use kernels_data::config::{Build, Framework}; +use kernels_data::metadata::{GitHash, Provenance}; use minijinja::Environment; use crate::{ @@ -23,17 +24,21 @@ mod tvm_ffi; pub use fileset::FileSet; -pub fn create_pyproject_file_set(build: Build, kernel_id: &KernelIdentifier) -> Result { +pub fn create_pyproject_file_set( + build: Build, + kernel_id: &KernelIdentifier, + provenance: Option<&Provenance>, +) -> Result { let mut env = Environment::new(); env.set_trim_blocks(true); minijinja_embed::load_templates!(&mut env); let file_set = if matches!(build.framework, Framework::TvmFfi(_)) { - tvm_ffi::write_tvm_ffi_ext(&env, &build, kernel_id)? + tvm_ffi::write_tvm_ffi_ext(&env, &build, kernel_id, provenance)? } else if build.is_noarch() { - torch::write_torch_ext_noarch(&env, &build, kernel_id)? + torch::write_torch_ext_noarch(&env, &build, kernel_id, provenance)? } else { - torch::write_torch_ext(&env, &build, kernel_id)? + torch::write_torch_ext(&env, &build, kernel_id, provenance)? }; Ok(file_set) @@ -44,12 +49,31 @@ pub fn create_pyproject( target_dir: Option, force: bool, unique_id: Option, + kernel_sha: Option, + kernel_dirty: bool, ) -> Result<()> { let kernel_dir = check_or_infer_kernel_dir(kernel_dir)?; let target_dir = check_or_infer_target_dir(&kernel_dir, target_dir)?; let build = parse_build(&kernel_dir)?; + + // Assemble build provenance. Prefer an explicitly provided kernel git + // provenance (e.g. passed by Nix builds, where the source tree has no + // `.git`); fall back to detecting it from the kernel's git repository. The + // `kernel-builder` provenance is always the one baked into this binary at + // compile time. + let kernel = kernel_sha + .map(|sha| GitHash { + sha, + dirty: kernel_dirty, + }) + .or_else(|| ops_identifier::git_hash(&kernel_dir)); + let provenance = Provenance { + kernel_builder: common::kernel_builder_version(), + kernel, + }; + let kernel_id = KernelIdentifier::new(&kernel_dir, build.general.name.python_name(), unique_id); - let file_set = create_pyproject_file_set(build, &kernel_id)?; + let file_set = create_pyproject_file_set(build, &kernel_id, Some(&provenance))?; file_set.write(&target_dir, force)?; Ok(()) @@ -65,9 +89,10 @@ pub fn clean_pyproject( let kernel_dir = check_or_infer_kernel_dir(kernel_dir)?; let target_dir = check_or_infer_target_dir(&kernel_dir, target_dir)?; let build = parse_build(&kernel_dir)?; + // Provenance is irrelevant when computing the set of files to clean. let kernel_id = KernelIdentifier::new(&kernel_dir, build.general.name.python_name(), unique_id); - let generated_files = create_pyproject_file_set(build, &kernel_id)?.into_names(); + let generated_files = create_pyproject_file_set(build, &kernel_id, None)?.into_names(); if generated_files.is_empty() { eprintln!("No generated artifacts found to clean."); diff --git a/kernel-builder/src/pyproject/ops_identifier.rs b/kernel-builder/src/pyproject/ops_identifier.rs index 46157260a..dae82b68a 100644 --- a/kernel-builder/src/pyproject/ops_identifier.rs +++ b/kernel-builder/src/pyproject/ops_identifier.rs @@ -3,6 +3,7 @@ use std::path::Path; use eyre::{Result, WrapErr}; use git2::Repository; use kernels_data::config::Backend; +use kernels_data::metadata::GitHash; use rand::Rng; pub fn random_identifier() -> String { @@ -28,6 +29,25 @@ pub fn git_identifier(target_dir: impl AsRef) -> Result { Ok(if dirty { format!("{rev}_dirty") } else { rev }) } +pub fn git_hash(target_dir: impl AsRef) -> Option { + let repo = Repository::discover(target_dir.as_ref()).ok()?; + let head = repo.head().ok()?; + let commit = head.peel_to_commit().ok()?; + let sha = commit.id().to_string(); + + let mut status_options = git2::StatusOptions::new(); + status_options.include_untracked(false); // Ignore untracked files (like generated CMake files) + status_options.exclude_submodules(true); + let dirty = !repo.statuses(Some(&mut status_options)).ok()?.is_empty(); + + Some(GitHash { sha, dirty }) +} + +/// Uniquely identifies a kernel for the purpose of ops-name generation. +/// +/// This type is intentionally small and atomic: it carries only what is +/// needed to name a kernel's ops. Build provenance (git SHA, dirty flags, +/// `kernel-builder` version) is handled separately when writing metadata. pub struct KernelIdentifier { name: String, unique_id: String, @@ -63,3 +83,23 @@ impl KernelIdentifier { &self.unique_id } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn kernel_identifier_uses_provided_unique_id() { + let tmp = tempfile::tempdir().unwrap(); + let id = KernelIdentifier::new(tmp.path(), "relu".to_owned(), Some("rev123".to_owned())); + + assert_eq!(id.unique_id(), "rev123"); + assert_eq!(id.to_string_for_backend(Backend::Cuda), "_relu_cuda_rev123"); + } + + #[test] + fn git_hash_is_none_in_non_git_dir() { + let tmp = tempfile::tempdir().unwrap(); + assert!(git_hash(tmp.path()).is_none()); + } +} diff --git a/kernel-builder/src/pyproject/torch/mod.rs b/kernel-builder/src/pyproject/torch/mod.rs index 11dcb1ee4..08f5d7ca4 100644 --- a/kernel-builder/src/pyproject/torch/mod.rs +++ b/kernel-builder/src/pyproject/torch/mod.rs @@ -4,6 +4,7 @@ use std::path::PathBuf; use eyre::{bail, Context, Result}; use itertools::Itertools; use kernels_data::config::{Backend, Build, General, Torch}; +use kernels_data::metadata::Provenance; use minijinja::context; use crate::pyproject::common::{ @@ -271,6 +272,7 @@ pub fn write_torch_ext( env: &minijinja::Environment, build: &Build, kernel_id: &KernelIdentifier, + provenance: Option<&Provenance>, ) -> Result { let torch_ext = match build.framework.torch() { Some(torch_ext) => torch_ext, @@ -296,7 +298,7 @@ pub fn write_torch_ext( write_torch_registration_macros(&mut file_set)?; - write_metadata(build, kernel_id, &mut file_set)?; + write_metadata(build, kernel_id, provenance, &mut file_set)?; Ok(file_set) } diff --git a/kernel-builder/src/pyproject/torch/noarch.rs b/kernel-builder/src/pyproject/torch/noarch.rs index 1fbe8d523..5cbe86f60 100644 --- a/kernel-builder/src/pyproject/torch/noarch.rs +++ b/kernel-builder/src/pyproject/torch/noarch.rs @@ -4,6 +4,7 @@ use eyre::{Context, Result}; use itertools::Itertools; use kernels_data::config::TorchNoarch; use kernels_data::config::{Backend, Build, General}; +use kernels_data::metadata::Provenance; use minijinja::{context, Environment}; use crate::pyproject::common::write_compat_py; @@ -17,6 +18,7 @@ pub fn write_torch_ext_noarch( env: &Environment, build: &Build, kernel_id: &KernelIdentifier, + provenance: Option<&Provenance>, ) -> Result { let mut file_set = FileSet::default(); @@ -34,7 +36,7 @@ pub fn write_torch_ext_noarch( &mut file_set, )?; write_setup_py(&mut file_set)?; - write_metadata(build, kernel_id, &mut file_set)?; + write_metadata(build, kernel_id, provenance, &mut file_set)?; Ok(file_set) } diff --git a/kernel-builder/src/pyproject/tvm_ffi/mod.rs b/kernel-builder/src/pyproject/tvm_ffi/mod.rs index 410ceb531..e08565a27 100644 --- a/kernel-builder/src/pyproject/tvm_ffi/mod.rs +++ b/kernel-builder/src/pyproject/tvm_ffi/mod.rs @@ -3,6 +3,7 @@ use std::io::Write; use eyre::{bail, Context, Result}; use itertools::Itertools; use kernels_data::config::{Backend, Build, General, TvmFfi}; +use kernels_data::metadata::Provenance; use minijinja::{context, Environment}; use crate::pyproject::common::{ @@ -42,6 +43,7 @@ pub fn write_tvm_ffi_ext( env: &Environment, build: &Build, kernel_id: &KernelIdentifier, + provenance: Option<&Provenance>, ) -> Result { let tvm_ffi_ext = match build.framework.tvm_ffi() { Some(torch_ext) => torch_ext, @@ -65,7 +67,7 @@ pub fn write_tvm_ffi_ext( write_pyproject_toml(env, &build.general, &mut file_set)?; - write_metadata(build, kernel_id, &mut file_set)?; + write_metadata(build, kernel_id, provenance, &mut file_set)?; Ok(file_set) } diff --git a/kernels-data/bindings/python/kernels_data.pyi b/kernels-data/bindings/python/kernels_data.pyi index f0397c8e4..20e29619f 100644 --- a/kernels-data/bindings/python/kernels_data.pyi +++ b/kernels-data/bindings/python/kernels_data.pyi @@ -7,7 +7,10 @@ from typing import Optional, final __all__ = [ "Backend", "BackendInfo", + "Provenance", "DigestAlgorithm", + "GitHash", + "KernelBuilderVersion", "KernelName", "Metadata", "Digest", @@ -63,6 +66,59 @@ class BackendInfo: def __repr__(self) -> str: ... +@final +class GitHash: + """Git provenance (commit SHA and dirty state) of a source tree.""" + + @property + def sha(self) -> str: + """Full 40-character commit SHA.""" + ... + + @property + def dirty(self) -> bool: + """Whether the working tree had uncommitted changes to tracked files.""" + ... + + def __repr__(self) -> str: ... + +@final +class KernelBuilderVersion: + """Provenance of the `kernel-builder` that produced a build.""" + + @property + def version(self) -> str: + """`kernel-builder` package version.""" + ... + + @property + def git(self) -> Optional[GitHash]: + """Commit SHA and dirty state of the `kernel-builder` source, when known.""" + ... + + def __repr__(self) -> str: ... + +@final +class Provenance: + """Build provenance: git state of the `kernel-builder` and kernel source.""" + + @property + def kernel_builder(self) -> KernelBuilderVersion: + """The `kernel-builder` that produced the build (always known).""" + ... + + @property + def kernel(self) -> Optional[GitHash]: + """Git provenance of the kernel source that was built.""" + ... + + @property + def dirty(self) -> bool: + """Whether either the `kernel-builder` or the kernel source was dirty.""" + ... + + def __repr__(self) -> str: ... + @final class Version: """A dotted numeric version (e.g. `12.8.0`). @@ -263,4 +319,6 @@ class Metadata: def backend(self) -> BackendInfo: ... @property def digest(self) -> Optional[Digest]: ... + @property + def provenance(self) -> Optional[Provenance]: ... def __repr__(self) -> str: ... diff --git a/kernels-data/bindings/python/src/lib.rs b/kernels-data/bindings/python/src/lib.rs index 25d234caa..9aa29f06d 100644 --- a/kernels-data/bindings/python/src/lib.rs +++ b/kernels-data/bindings/python/src/lib.rs @@ -6,7 +6,7 @@ use std::str::FromStr; use kernels_data::config::{Backend, KernelName}; use kernels_data::digest::{Digest, DigestAlgorithm, DigestViolation}; -use kernels_data::metadata::{BackendInfo, Metadata}; +use kernels_data::metadata::{BackendInfo, GitHash, KernelBuilderVersion, Metadata, Provenance}; use kernels_data::version::Version; use pyo3::Bound as PyBound; use pyo3::exceptions::{PyException, PyOSError, PyRuntimeError, PyValueError}; @@ -188,6 +188,125 @@ impl PyBackendInfo { } } +#[pyclass(name = "GitHash", frozen)] +#[derive(Clone, Debug)] +struct PyGitHash { + sha: String, + dirty: bool, +} + +impl From for PyGitHash { + fn from(g: GitHash) -> Self { + Self { + sha: g.sha, + dirty: g.dirty, + } + } +} + +#[pymethods] +impl PyGitHash { + #[getter] + fn sha(&self) -> &str { + &self.sha + } + + #[getter] + fn dirty(&self) -> bool { + self.dirty + } + + fn __repr__(&self) -> String { + format!("GitHash(sha={:?}, dirty={})", self.sha, self.dirty) + } +} + +#[pyclass(name = "KernelBuilderVersion", frozen)] +#[derive(Clone, Debug)] +struct PyKernelBuilderVersion { + version: String, + git: Option, +} + +impl From for PyKernelBuilderVersion { + fn from(kb: KernelBuilderVersion) -> Self { + Self { + version: kb.version, + git: kb.git.map(Into::into), + } + } +} + +#[pymethods] +impl PyKernelBuilderVersion { + #[getter] + fn version(&self) -> &str { + &self.version + } + + /// Commit SHA + dirty state of the `kernel-builder` source, when known. + #[getter] + fn git(&self) -> Option { + self.git.clone() + } + + fn __repr__(&self) -> String { + format!( + "KernelBuilderVersion(version={:?}, git={})", + self.version, + self.git + .as_ref() + .map_or("None".to_string(), |g| g.__repr__()) + ) + } +} + +#[pyclass(name = "Provenance", frozen)] +#[derive(Clone, Debug)] +struct PyProvenance { + kernel_builder: PyKernelBuilderVersion, + kernel: Option, +} + +impl From for PyProvenance { + fn from(b: Provenance) -> Self { + Self { + kernel_builder: b.kernel_builder.into(), + kernel: b.kernel.map(Into::into), + } + } +} + +#[pymethods] +impl PyProvenance { + #[getter] + fn kernel_builder(&self) -> PyKernelBuilderVersion { + self.kernel_builder.clone() + } + + #[getter] + fn kernel(&self) -> Option { + self.kernel.clone() + } + + /// Whether either the `kernel-builder` or the kernel source was dirty. + #[getter] + fn dirty(&self) -> bool { + self.kernel_builder.git.as_ref().is_some_and(|g| g.dirty) + || self.kernel.as_ref().is_some_and(|k| k.dirty) + } + + fn __repr__(&self) -> String { + format!( + "Provenance(kernel_builder={}, kernel={})", + self.kernel_builder.__repr__(), + self.kernel + .as_ref() + .map_or("None".to_string(), |k| k.__repr__()) + ) + } +} + /// Parsed `metadata.json` for a kernel build variant. #[pyclass(name = "Metadata", frozen)] #[derive(Clone, Debug)] @@ -201,6 +320,7 @@ struct PyMetadata { python_depends: Vec, backend: PyBackendInfo, digest: Option, + provenance: Option, } impl From for PyMetadata { @@ -215,6 +335,7 @@ impl From for PyMetadata { python_depends: m.python_depends, backend: m.backend.into(), digest: m.digest.map(Into::into), + provenance: m.provenance.map(Into::into), } } } @@ -293,9 +414,14 @@ impl PyMetadata { self.digest.clone() } + #[getter] + fn provenance(&self) -> Option { + self.provenance.clone() + } + fn __repr__(&self) -> String { format!( - "Metadata(id={}, name={:?}, version={:?}, license={:?}, upstream={:?}, source={:?}, python_depends={:?}, backend={}, digest={})", + "Metadata(id={}, name={:?}, version={:?}, license={:?}, upstream={:?}, source={:?}, python_depends={:?}, backend={}, digest={}, provenance={})", self.id, self.name, self.version, @@ -306,7 +432,10 @@ impl PyMetadata { self.backend.__repr__(), self.digest .as_ref() - .map_or("None".to_string(), |sd| sd.__repr__()) + .map_or("None".to_string(), |sd| sd.__repr__()), + self.provenance + .as_ref() + .map_or("None".to_string(), |bi| bi.__repr__()) ) } } @@ -530,6 +659,9 @@ impl PyDigest { fn kernels_data_py(m: &PyBound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/kernels-data/src/metadata.rs b/kernels-data/src/metadata.rs index a55c42c51..d895f7fbb 100644 --- a/kernels-data/src/metadata.rs +++ b/kernels-data/src/metadata.rs @@ -15,6 +15,45 @@ pub struct BackendInfo { pub archs: Option>, } +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub struct GitHash { + pub sha: String, + pub dirty: bool, +} + +/// Provenance of the `kernel-builder` that produced a build. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub struct KernelBuilderVersion { + pub version: String, + /// Commit SHA + dirty state of the `kernel-builder` source, when known. + #[serde(flatten)] + pub git: Option, +} + +/// Provenance of a kernel build: the git state of the `kernel-builder` and of +/// the kernel source it was built from. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub struct Provenance { + /// The `kernel-builder` that produced the build. Always known: it is baked + /// into the `kernel-builder` binary at compile time. + pub kernel_builder: KernelBuilderVersion, + /// Git provenance of the kernel source. `None` when the kernel source was + /// not built from a git repository (so its revision is unknown). + #[serde(skip_serializing_if = "Option::is_none")] + pub kernel: Option, +} + +impl Provenance { + /// Whether either the `kernel-builder` or the kernel source was dirty. + pub fn is_dirty(&self) -> bool { + self.kernel_builder.git.as_ref().is_some_and(|g| g.dirty) + || self.kernel.as_ref().is_some_and(|k| k.dirty) + } +} + /// Kernel metadata. #[derive(Debug, Deserialize, Serialize)] #[serde(rename_all = "kebab-case")] @@ -31,6 +70,8 @@ pub struct Metadata { pub backend: BackendInfo, #[serde(skip_serializing_if = "Option::is_none")] pub digest: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub provenance: Option, } impl Metadata { @@ -58,6 +99,7 @@ impl Metadata { backend_type: backend, }, digest: None, + provenance: None, }) } @@ -86,7 +128,23 @@ mod tests { use crate::config::{Backend, Build, Framework, General, KernelName, TorchNoarch, TvmFfi}; - use super::Metadata; + use super::{GitHash, KernelBuilderVersion, Metadata, Provenance}; + + fn sample_provenance(kernel_builder_dirty: bool, kernel_dirty: bool) -> Provenance { + Provenance { + kernel_builder: KernelBuilderVersion { + version: "0.1.0".to_string(), + git: Some(GitHash { + sha: "a".repeat(40), + dirty: kernel_builder_dirty, + }), + }, + kernel: Some(GitHash { + sha: "b".repeat(40), + dirty: kernel_dirty, + }), + } + } fn torch_noarch_build() -> Build { Build { @@ -171,4 +229,89 @@ mod tests { assert_eq!(metadata.backend.backend_type, Backend::Cuda); assert!(metadata.backend.archs.is_none()); } + + #[test] + fn provenance_is_dirty_reflects_either_source() { + assert!(!sample_provenance(false, false).is_dirty()); + assert!(sample_provenance(true, false).is_dirty()); + assert!(sample_provenance(false, true).is_dirty()); + assert!(sample_provenance(true, true).is_dirty()); + } + + #[test] + fn metadata_without_provenance_serializes_without_key() { + let build = torch_noarch_build(); + let metadata = Metadata::for_backend(&build, "test-id".to_string(), Backend::Cuda).unwrap(); + + let json = serde_json::to_string(&metadata).unwrap(); + assert!(!json.contains("provenance")); + + let parsed: Metadata = serde_json::from_str(&json).unwrap(); + assert!(parsed.provenance.is_none()); + } + + #[test] + fn metadata_round_trips_provenance_in_kebab_case() { + let build = torch_noarch_build(); + let mut metadata = + Metadata::for_backend(&build, "test-id".to_string(), Backend::Cuda).unwrap(); + metadata.provenance = Some(sample_provenance(false, true)); + + let json = serde_json::to_string(&metadata).unwrap(); + assert!(json.contains("\"provenance\"")); + assert!(json.contains("\"kernel-builder\"")); + + let parsed: Metadata = serde_json::from_str(&json).unwrap(); + let provenance = parsed.provenance.expect("provenance should round-trip"); + assert!(provenance.is_dirty()); + assert_eq!(provenance.kernel.unwrap().sha, "b".repeat(40)); + + let kernel_builder = provenance.kernel_builder; + assert_eq!(kernel_builder.version, "0.1.0"); + // The embedded `GitHash` is flattened into the `kernel-builder` object. + let git = kernel_builder + .git + .expect("kernel-builder git should round-trip"); + assert_eq!(git.sha, "a".repeat(40)); + assert!(!git.dirty); + } + + #[test] + fn kernel_builder_version_flattens_git_hash() { + let kernel_builder = KernelBuilderVersion { + version: "0.1.0".to_string(), + git: Some(GitHash { + sha: "c".repeat(40), + dirty: true, + }), + }; + + // The `GitHash` fields are flattened into the same object as `version`. + let value: serde_json::Value = serde_json::to_value(&kernel_builder).unwrap(); + assert_eq!(value["version"], "0.1.0"); + assert_eq!(value["sha"], "c".repeat(40)); + assert_eq!(value["dirty"], true); + assert!(value.get("git").is_none()); + + let parsed: KernelBuilderVersion = serde_json::from_value(value).unwrap(); + let git = parsed.git.expect("git should round-trip"); + assert_eq!(git.sha, "c".repeat(40)); + assert!(git.dirty); + } + + #[test] + fn kernel_builder_version_without_git_round_trips_to_none() { + // When the `kernel-builder` git provenance is unknown, only `version` + // is serialized and it must deserialize back to `git: None`. + let kernel_builder = KernelBuilderVersion { + version: "0.1.0".to_string(), + git: None, + }; + + let json = serde_json::to_string(&kernel_builder).unwrap(); + assert_eq!(json, r#"{"version":"0.1.0"}"#); + + let parsed: KernelBuilderVersion = serde_json::from_str(&json).unwrap(); + assert!(parsed.git.is_none()); + } } diff --git a/nix-builder/lib/build-sets.nix b/nix-builder/lib/build-sets.nix index 348781357..76cada998 100644 --- a/nix-builder/lib/build-sets.nix +++ b/nix-builder/lib/build-sets.nix @@ -1,13 +1,15 @@ { nixpkgs, rust-overlay, + + # Git provenance (`{ sha, dirty }` or `null`) of the `kernel-builder` flake, + # so it can be burned into the binary. + builderProvenance ? null, }: let inherit (nixpkgs) lib; - overlay = import ../nix/overlay.nix; - inherit (import ./torch-version-utils.nix { inherit lib; }) backend flattenSystems @@ -24,7 +26,13 @@ let in builtins.map (buildConfig: buildConfig // { backend = backend buildConfig; }) systemBuildConfigs; - mkBuildSet = import ./mk-build-set.nix { inherit nixpkgs rust-overlay; }; + mkBuildSet = import ./mk-build-set.nix { + inherit + nixpkgs + rust-overlay + builderProvenance + ; + }; in rec { diff --git a/nix-builder/lib/build.nix b/nix-builder/lib/build.nix index 09b4e0cee..f890149d4 100644 --- a/nix-builder/lib/build.nix +++ b/nix-builder/lib/build.nix @@ -113,6 +113,7 @@ rec { rev, doGetKernelCheck, stripRPath ? false, + kernelProvenance, }: let inherit (lib) fileset; @@ -151,6 +152,7 @@ rec { doGetKernelCheck pythonDeps backendPythonDeps + kernelProvenance ; kernelName = kernelConfig.name; } @@ -166,6 +168,7 @@ rec { rev pythonDeps backendPythonDeps + kernelProvenance ; kernelName = kernelConfig.name; @@ -183,6 +186,7 @@ rec { rev pythonDeps backendPythonDeps + kernelProvenance ; torchStableAbiVersion = kernelConfig.torchStableAbiVersionForBackend buildConfig.backend; @@ -199,6 +203,7 @@ rec { doGetKernelCheck, bundleOnly, buildSets, + kernelProvenance, }: let kernelConfig = readKernelConfig path; @@ -212,6 +217,7 @@ rec { kernelConfig rev doGetKernelCheck + kernelProvenance ; stripRPath = true; }; @@ -227,6 +233,7 @@ rec { rev, doGetKernelCheck, buildSets, + kernelProvenance, }: let extensions = mkDistTorchExtensions { @@ -235,6 +242,7 @@ rec { path rev doGetKernelCheck + kernelProvenance ; bundleOnly = true; }; @@ -289,6 +297,8 @@ rec { rev doGetKernelCheck ; + # Dev/test shells do not record build provenance. + kernelProvenance = null; }; in { @@ -339,6 +349,7 @@ rec { buildSets, doGetKernelCheck, pythonCheckInputs, + kernelProvenance, }: let kernelConfig = readKernelConfig path; @@ -355,6 +366,7 @@ rec { kernelConfig rev doGetKernelCheck + kernelProvenance ; }; testPython = @@ -412,6 +424,8 @@ rec { rev doGetKernelCheck ; + # Dev/test shells do not record build provenance. + kernelProvenance = null; }; python = ( pkgs.python3.withPackages ( diff --git a/nix-builder/lib/extension/provenance-flags.nix b/nix-builder/lib/extension/provenance-flags.nix new file mode 100644 index 000000000..bfcd1e4db --- /dev/null +++ b/nix-builder/lib/extension/provenance-flags.nix @@ -0,0 +1,10 @@ +# Assemble `kernel-builder create-pyproject` flags from the kernel's atomic git +# provenance (`{ sha; dirty; }`, or `null` when unknown). The Nix sandbox has no +# `.git`, so the kernel's commit SHA and dirty state are passed in explicitly. +{ + lib, + kernelProvenance, +}: +lib.optionalString (kernelProvenance != null) ( + "--kernel-sha ${kernelProvenance.sha}" + lib.optionalString kernelProvenance.dirty " --kernel-dirty" +) diff --git a/nix-builder/lib/extension/torch/arch.nix b/nix-builder/lib/extension/torch/arch.nix index d23655916..0fa51eaf2 100644 --- a/nix-builder/lib/extension/torch/arch.nix +++ b/nix-builder/lib/extension/torch/arch.nix @@ -71,6 +71,10 @@ # Revision to bake into the ops name. rev, + # Git provenance (`{ sha; dirty; }`, or `null`) of the kernel source, recorded + # in the build metadata. + kernelProvenance, + src, }: @@ -111,6 +115,8 @@ let metalSupport = buildConfig.metal or false; + provenanceFlags = import ../provenance-flags.nix { inherit lib kernelProvenance; }; + in stdenv.mkDerivation (prevAttrs: { @@ -132,7 +138,7 @@ stdenv.mkDerivation (prevAttrs: { mkdir -p $out cp -r --no-preserve=mode ${src}/* $out/ ${pkgs.kernel-builder}/bin/kernel-builder create-pyproject \ - --unique-id ${rev} $out + --unique-id ${rev} ${provenanceFlags} $out ''; preConfigure = diff --git a/nix-builder/lib/extension/torch/no-arch.nix b/nix-builder/lib/extension/torch/no-arch.nix index f89211156..e0f2b132d 100644 --- a/nix-builder/lib/extension/torch/no-arch.nix +++ b/nix-builder/lib/extension/torch/no-arch.nix @@ -41,6 +41,10 @@ pythonDeps, backendPythonDeps, + + # Git provenance (`{ sha; dirty; }`, or `null`) of the kernel source, recorded + # in the build metadata. + kernelProvenance, }: # Extra validation - the environment should correspind to the build config. @@ -68,6 +72,7 @@ let ++ [ torch ]; moduleName = builtins.replaceStrings [ "-" ] [ "_" ] kernelName; metalSupport = buildConfig.metal or false; + provenanceFlags = import ../provenance-flags.nix { inherit lib kernelProvenance; }; in stdenv.mkDerivation (prevAttrs: { @@ -79,7 +84,7 @@ stdenv.mkDerivation (prevAttrs: { mkdir -p $out cp -r --no-preserve=mode ${src}/* $out/ ${pkgs.kernel-builder}/bin/kernel-builder create-pyproject \ - --unique-id ${rev} $out + --unique-id ${rev} ${provenanceFlags} $out ''; framework = "torch"; diff --git a/nix-builder/lib/extension/tvm-ffi/arch.nix b/nix-builder/lib/extension/tvm-ffi/arch.nix index ffc877cd9..ef1e1d2d9 100644 --- a/nix-builder/lib/extension/tvm-ffi/arch.nix +++ b/nix-builder/lib/extension/tvm-ffi/arch.nix @@ -68,6 +68,10 @@ # Revision to bake into the ops name. rev, + # Git provenance (`{ sha; dirty; }`, or `null`) of the kernel source, recorded + # in the build metadata. + kernelProvenance, + src, }: @@ -111,6 +115,8 @@ let metalSupport = buildConfig.metal or false; + provenanceFlags = import ../provenance-flags.nix { inherit lib kernelProvenance; }; + in stdenv.mkDerivation (prevAttrs: { @@ -129,7 +135,7 @@ stdenv.mkDerivation (prevAttrs: { # Generate build files. postPatch = '' kernel-builder create-pyproject \ - --unique-id ${rev} . + --unique-id ${rev} ${provenanceFlags} . ''; preConfigure = diff --git a/nix-builder/lib/flake-provenance.nix b/nix-builder/lib/flake-provenance.nix new file mode 100644 index 000000000..92f0f73ea --- /dev/null +++ b/nix-builder/lib/flake-provenance.nix @@ -0,0 +1,20 @@ +# Extract atomic git provenance (`{ sha; dirty; }`, or `null`) from a flake's +# `self`. The result is `null` when the flake was built from a non-git source +# (no revision is available, e.g. a local `path:`). +{ lib }: +flakeSelf: + +if flakeSelf == null then + null +else if flakeSelf ? rev then + { + sha = flakeSelf.rev; + dirty = false; + } +else if flakeSelf ? dirtyRev then + { + sha = lib.removeSuffix "-dirty" flakeSelf.dirtyRev; + dirty = true; + } +else + null diff --git a/nix-builder/lib/gen-flake-outputs.nix b/nix-builder/lib/gen-flake-outputs.nix index 5409edb3f..99fba216e 100644 --- a/nix-builder/lib/gen-flake-outputs.nix +++ b/nix-builder/lib/gen-flake-outputs.nix @@ -36,6 +36,12 @@ let revUnderscored = builtins.replaceStrings [ "-" ] [ "_" ] flakeRev; + # Git provenance (`{ sha; dirty; }`, or `null`) of the kernel source, recorded + # in the build metadata. The Nix sandbox has no `.git`, so it is derived from + # the kernel flake's `self` here and passed in explicitly. (`kernel-builder`'s + # own provenance is burned into its binary.) + kernelProvenance = import ./flake-provenance.nix { inherit lib; } self; + applicableBuildSets = build.applicableBuildSets { inherit path buildSets; }; kernelConfig = (import ./kernel-config.nix { inherit lib; }) path; @@ -153,12 +159,17 @@ in packages = let bundle = build.mkExtensionBundle { - inherit path doGetKernelCheck; + inherit path doGetKernelCheck kernelProvenance; buildSets = applicableBuildSets; rev = revUnderscored; }; ciTests = build.mkCiTests { - inherit path doGetKernelCheck pythonCheckInputs; + inherit + path + doGetKernelCheck + pythonCheckInputs + kernelProvenance + ; buildSets = applicableBuildSets; rev = revUnderscored; }; @@ -175,7 +186,7 @@ in builtins.map (backend: { name = backend; value = build.mkExtensionBundle { - inherit path doGetKernelCheck; + inherit path doGetKernelCheck kernelProvenance; buildSets = builtins.filter ( set: buildConfigBackend set.buildConfig == backend ) applicableBuildSets; @@ -241,7 +252,7 @@ in ++ (headOrEmpty (setsWithFramework "xpu")); in build.mkExtensionBundle { - inherit path doGetKernelCheck; + inherit path doGetKernelCheck kernelProvenance; buildSets = onePerFramework; rev = revUnderscored; }; @@ -254,7 +265,7 @@ in builtins.map (backend: { name = backend; value = build.mkExtensionBundle { - inherit path doGetKernelCheck; + inherit path doGetKernelCheck kernelProvenance; # It is too costly to build all variants in CI, so we just build one per framework. buildSets = headOrEmpty ( builtins.filter (set: set.buildConfig.framework == backend) buildSetsSorted @@ -277,7 +288,7 @@ in }; redistributable = build.mkDistTorchExtensions { - inherit path doGetKernelCheck; + inherit path doGetKernelCheck kernelProvenance; bundleOnly = false; rev = revUnderscored; buildSets = applicableBuildSets; diff --git a/nix-builder/lib/mk-build-set.nix b/nix-builder/lib/mk-build-set.nix index dd8d54254..b12308250 100644 --- a/nix-builder/lib/mk-build-set.nix +++ b/nix-builder/lib/mk-build-set.nix @@ -1,12 +1,16 @@ { nixpkgs, rust-overlay, + + # Git provenance (`{ sha, dirty }` or `null`) of the `kernel-builder` flake, + # so it can be burned into the binary that the kernels it builds record. + builderProvenance ? null, }: let inherit (nixpkgs) lib; - overlay = import ../overlay.nix; + overlay = import ../overlay.nix { inherit builderProvenance; }; flattenVersion = version: lib.replaceStrings [ "." ] [ "_" ] (lib.versions.pad 2 version); diff --git a/nix-builder/overlay.nix b/nix-builder/overlay.nix index 9fab8d225..b7328dbe6 100644 --- a/nix-builder/overlay.nix +++ b/nix-builder/overlay.nix @@ -1,6 +1,13 @@ +{ + # Git provenance (`{ sha, dirty }` or `null`) of the `kernel-builder` flake, + # burned into the binary. It cannot be detected from `.git` in the build + # sandbox. + builderProvenance ? null, +}: + final: prev: { - kernel-builder = final.callPackage ./pkgs/kernel-builder { }; + kernel-builder = final.callPackage ./pkgs/kernel-builder { inherit builderProvenance; }; cmakeNvccThreadsHook = final.callPackage ./pkgs/cmake-nvcc-threads-hook { }; diff --git a/nix-builder/pkgs/kernel-builder/default.nix b/nix-builder/pkgs/kernel-builder/default.nix index 2fb344851..31d02bcae 100644 --- a/nix-builder/pkgs/kernel-builder/default.nix +++ b/nix-builder/pkgs/kernel-builder/default.nix @@ -5,6 +5,12 @@ pkg-config, libgit2, openssl, + + # Git provenance (`{ sha, dirty }`, or `null` for a non-git source) of the + # `kernel-builder` flake. It is burned into the binary at build time and + # later recorded in the build metadata of the kernels it builds. The build + # sandbox has no `.git`, so `build.rs` cannot detect it. + builderProvenance ? null, }: let @@ -15,76 +21,86 @@ let "hf-kernel-builder" ]; in -rustPlatform.buildRustPackage { - inherit version; - pname = "kernel-builder"; +rustPlatform.buildRustPackage ( + # Supply the git provenance through `built`'s override environment variables + # (`hf_kernel_builder` is the package name with hyphens replaced by + # underscores), which `build.rs` bakes into the binary. When there is no provenance + # information (e.g. non-git source), do not set the variables. + lib.optionalAttrs (builderProvenance != null) { + "BUILT_OVERRIDE_hf_kernel_builder_GIT_COMMIT_HASH" = builderProvenance.sha; + "BUILT_OVERRIDE_hf_kernel_builder_GIT_DIRTY" = if builderProvenance.dirty then "true" else "false"; + } + // { + inherit version; + pname = "kernel-builder"; + + src = + let + sourceFiles = + file: + file.name == "Cargo.toml" + || file.name == "Cargo.lock" + || file.name == "flake.nix" + || file.name == "manylinux-policy.json" + || file.name == "pyproject.toml" + || file.name == "pyproject_universal.toml" + || file.name == "python_dependencies.json" + || file.name == "shim_function_versions.txt" + || file.name == "stable_abi.toml" + || file.name == ".gitattributes" + || file.name == ".gitignore" + || (builtins.any file.hasExt [ + "cmake" + "cpp" + "cu" + "h" + "in" + "md" + "metal" + "mm" + "py" + "rs" + "toml" + ]); + in + import ../crate-dirs.nix { + inherit lib sourceFiles; + }; - src = - let - sourceFiles = - file: - file.name == "Cargo.toml" - || file.name == "Cargo.lock" - || file.name == "flake.nix" - || file.name == "manylinux-policy.json" - || file.name == "pyproject.toml" - || file.name == "pyproject_universal.toml" - || file.name == "python_dependencies.json" - || file.name == "shim_function_versions.txt" - || file.name == "stable_abi.toml" - || file.name == ".gitattributes" - || file.name == ".gitignore" - || (builtins.any file.hasExt [ - "cmake" - "cpp" - "cu" - "h" - "in" - "md" - "metal" - "mm" - "py" - "rs" - "toml" - ]); - in - import ../crate-dirs.nix { - inherit lib sourceFiles; + cargoLock = { + lockFile = ../../../Cargo.lock; }; - cargoLock = { - lockFile = ../../../Cargo.lock; - }; + cargoBuildFlags = cargoFlags; + cargoTestFlags = cargoFlags; - cargoBuildFlags = cargoFlags; - cargoTestFlags = cargoFlags; + # e2e tests look for binary at target/debug/ which doesn't exist in nix + doCheck = false; - # e2e tests look for binary at target/debug/ which doesn't exist in nix - doCheck = false; + nativeBuildInputs = [ + installShellFiles + pkg-config + ]; - nativeBuildInputs = [ - installShellFiles - pkg-config - ]; + buildInputs = [ + libgit2 + openssl.dev + ]; - buildInputs = [ - libgit2 - openssl.dev - ]; + postInstall = '' + for shell in bash fish zsh; do + $out/bin/kernel-builder completions $shell > kernel-builder.$shell + done + installShellCompletion kernel-builder.{bash,fish,zsh} + ''; - postInstall = '' - for shell in bash fish zsh; do - $out/bin/kernel-builder completions $shell > kernel-builder.$shell - done - installShellCompletion kernel-builder.{bash,fish,zsh} - ''; + setupHooks = [ + ./check-kernel-abi-hook.sh + ./check-kernel-build-hook.sh + ]; - setupHooks = [ - ./check-kernel-abi-hook.sh - ./check-kernel-build-hook.sh - ]; - - meta = { - description = "Create cmake build infrastructure from build.toml files"; - }; -} + meta = { + description = "Create cmake build infrastructure from build.toml files"; + }; + } +)