feat: append builder info to detect builds made with uncommitted changes in the builder

docs/source/builder-cli.md

@@ -244,6 +244,8 @@ Generate CMake files for a kernel extension build
 
 * `-f`, `--force` — Force-overwrite existing files
 * `--unique-id <UNIQUE_ID>` — This is an optional unique identifier that is suffixed to the kernel name to avoid name collisions. (e.g. Git SHA)
+* `--kernel-sha <KERNEL_SHA>` — Full commit SHA of the kernel source, recorded in the build metadata. When absent, it is detected from the kernel's git repository (used by Nix builds where the source has no `.git`)
+* `--kernel-dirty` — Mark the kernel source as having uncommitted changes in the build metadata. Only meaningful together with `--kernel-sha`
 
 
 

docs/source/kernel-requirements.md

@@ -81,6 +81,31 @@ metadata. Currently the following top-level keys are supported:
 - `digest` (`Digest`, required): hash digest of the kernel files.
 - `python-depends` (`list[str]`, optional): list of Python dependencies
   from a curated set of Python dependencies.
+- `provenance` (`dict`, optional): provenance of the build, used to flag
+  non-reproducible (dirty) builds. It contains two optional sub-objects:
+  - `kernel-builder`: the `kernel-builder` that produced the build, with its
+    `version` (`str`), the `sha` (`str`) of the `kernel-builder` source it was
+    built from (when known), and a `dirty` (`bool`) flag that is `true` when
+    `kernel-builder` was built from a source tree with uncommitted changes.
+  - `kernel`: the kernel source that was built, with its commit `sha` (`str`)
+    and a `dirty` (`bool`) flag that is `true` when the kernel source had
+    uncommitted changes.
+
+  When either `dirty` flag is set, the kernel was built from uncommitted
+  sources and cannot be reliably reproduced.
+
+  > **Note:** For Nix builds, dirtiness follows Nix's flake tree status, which
+  > also counts **untracked** files (including an uncommitted `flake.lock`).
+  > Commit your `flake.lock` (and avoid stray untracked files) so that clean
+  > builds are not flagged as dirty. Local `create-pyproject` runs only
+  > consider changes to tracked files.
+
+  > **Note:** The kernel `sha`/`dirty` are captured at the moment
+  > `create-pyproject` runs, so they describe the source tree as it was *then*.
+  > Running `create-pyproject` and committing afterwards is bad practice: the
+  > recorded provenance keeps pointing at the pre-commit state (a stale `sha`,
+  > and `dirty: true` if the tree was dirty) even though the committed source
+  > differs. Generate the metadata from the final, committed source instead.
 
 Example `metadata.json`:
 

flake.nix

@@ -27,9 +27,16 @@
       rust-overlay,
     }:
     let
+      # Git provenance (`{ sha, dirty }` or `null`) of the `kernel-builder`
+      # flake itself, so it can be burned into the binary at build time and
+      # recorded in the build metadata of the kernels it builds.
+      builderProvenance = import ./nix-builder/lib/flake-provenance.nix {
+        inherit (nixpkgs) lib;
+      } self;
+
       inherit
         (import ./nix-builder/lib/build-sets.nix {
-          inherit nixpkgs rust-overlay;
+          inherit nixpkgs rust-overlay builderProvenance;
         })
         mkBuildSets
         partitionBuildSetsBySystem

kernel-builder/Cargo.toml

@@ -37,6 +37,7 @@ reqwest = { version = "0.13", features = ["blocking"] }
 walkdir = "2"
 
 [build-dependencies]
+built = { version = "0.8", features = ["git2"] }
 minijinja-embed = "2.5"
 
 [dev-dependencies]

kernel-builder/build.rs

@@ -1,3 +1,4 @@
 fn main() {
     minijinja_embed::embed_templates!("src/pyproject/templates");
+    built::write_built_file().expect("Failed to acquire build-time information");
 }

kernel-builder/src/main.rs

@@ -50,6 +50,13 @@ use validate_builds::check_builds;
 
 use crate::util::parse_and_validate_compat;
 
+/// Build-time metadata gathered by the [`built`](https://crates.io/crates/built)
+/// crate (see `build.rs`), including the `kernel-builder` git provenance that is
+/// recorded in the build metadata of the kernels it builds.
+mod built_info {
+    include!(concat!(env!("OUT_DIR"), "/built.rs"));
+}
+
 #[derive(Args, Debug)]
 struct NixArgs {
     /// Maximum number of parallel Nix build jobs.
@@ -174,6 +181,17 @@ enum Commands {
         /// kernel name to avoid name collisions. (e.g. Git SHA)
         #[arg(long)]
         unique_id: Option<String>,
+
+        /// Full commit SHA of the kernel source, recorded in the build
+        /// metadata. When absent, it is detected from the kernel's git
+        /// repository (used by Nix builds where the source has no `.git`).
+        #[arg(long)]
+        kernel_sha: Option<String>,
+
+        /// Mark the kernel source as having uncommitted changes in the build
+        /// metadata. Only meaningful together with `--kernel-sha`.
+        #[arg(long)]
+        kernel_dirty: bool,
     },
 
     /// Spawn a kernel development shell.
@@ -365,7 +383,16 @@ fn main() -> Result<()> {
             force,
             target_dir,
             unique_id,
-        } => create_pyproject(kernel_dir, target_dir, force, unique_id),
+            kernel_sha,
+            kernel_dirty,
+        } => create_pyproject(
+            kernel_dir,
+            target_dir,
+            force,
+            unique_id,
+            kernel_sha,
+            kernel_dirty,
+        ),
         Commands::Devshell {
             kernel_dir,
             variant,

kernel-builder/src/pyproject/common.rs

@@ -4,7 +4,7 @@ use eyre::Result;
 use itertools::Itertools;
 
 use kernels_data::config::{Backend, Build};
-use kernels_data::metadata::Metadata;
+use kernels_data::metadata::{GitHash, KernelBuilderVersion, Metadata, Provenance};
 
 use crate::pyproject::ops_identifier::KernelIdentifier;
 use crate::pyproject::FileSet;
@@ -20,16 +20,34 @@ pub fn write_compat_py(file_set: &mut FileSet) -> Result<()> {
     Ok(())
 }
 
+/// `kernel-builder` provenance baked in at compile time.
+///
+/// The git provenance is gathered by the `built` crate (see `build.rs`). In
+/// build sandboxes without a `.git` (e.g. Nix), the derivation supplies it via
+/// `built`'s `BUILT_OVERRIDE_hf_kernel_builder_GIT_*` environment variables.
+pub(crate) fn kernel_builder_version() -> KernelBuilderVersion {
+    let git = crate::built_info::GIT_COMMIT_HASH.map(|sha| GitHash {
+        sha: sha.to_owned(),
+        dirty: crate::built_info::GIT_DIRTY.unwrap_or(false),
+    });
+    KernelBuilderVersion {
+        version: env!("CARGO_PKG_VERSION").to_owned(),
+        git,
+    }
+}
+
 pub fn write_metadata(
     build: &Build,
     kernel_id: &KernelIdentifier,
+    provenance: Option<&Provenance>,
     file_set: &mut FileSet,
 ) -> Result<()> {
     for backend in &Backend::all() {
         let writer = file_set.entry(format!("metadata-{backend}.json"));
 
-        let metadata =
+        let mut metadata =
             Metadata::for_backend(build, kernel_id.to_string_for_backend(*backend), *backend)?;
+        metadata.provenance = provenance.cloned();
 
         serde_json::to_writer_pretty(writer, &metadata)?;
     }

kernel-builder/src/pyproject/mod.rs

@@ -6,6 +6,7 @@ use std::{
 
 use eyre::{bail, Result};
 use kernels_data::config::{Build, Framework};
+use kernels_data::metadata::{GitHash, Provenance};
 use minijinja::Environment;
 
 use crate::{
@@ -23,17 +24,21 @@ mod tvm_ffi;
 
 pub use fileset::FileSet;
 
-pub fn create_pyproject_file_set(build: Build, kernel_id: &KernelIdentifier) -> Result<FileSet> {
+pub fn create_pyproject_file_set(
+    build: Build,
+    kernel_id: &KernelIdentifier,
+    provenance: Option<&Provenance>,
+) -> Result<FileSet> {
     let mut env = Environment::new();
     env.set_trim_blocks(true);
     minijinja_embed::load_templates!(&mut env);
 
     let file_set = if matches!(build.framework, Framework::TvmFfi(_)) {
-        tvm_ffi::write_tvm_ffi_ext(&env, &build, kernel_id)?
+        tvm_ffi::write_tvm_ffi_ext(&env, &build, kernel_id, provenance)?
     } else if build.is_noarch() {
-        torch::write_torch_ext_noarch(&env, &build, kernel_id)?
+        torch::write_torch_ext_noarch(&env, &build, kernel_id, provenance)?
     } else {
-        torch::write_torch_ext(&env, &build, kernel_id)?
+        torch::write_torch_ext(&env, &build, kernel_id, provenance)?
     };
 
     Ok(file_set)
@@ -44,12 +49,31 @@ pub fn create_pyproject(
     target_dir: Option<PathBuf>,
     force: bool,
     unique_id: Option<String>,
+    kernel_sha: Option<String>,
+    kernel_dirty: bool,
 ) -> Result<()> {
     let kernel_dir = check_or_infer_kernel_dir(kernel_dir)?;
     let target_dir = check_or_infer_target_dir(&kernel_dir, target_dir)?;
     let build = parse_build(&kernel_dir)?;
+
+    // Assemble build provenance. Prefer an explicitly provided kernel git
+    // provenance (e.g. passed by Nix builds, where the source tree has no
+    // `.git`); fall back to detecting it from the kernel's git repository. The
+    // `kernel-builder` provenance is always the one baked into this binary at
+    // compile time.
+    let kernel = kernel_sha
+        .map(|sha| GitHash {
+            sha,
+            dirty: kernel_dirty,
+        })
+        .or_else(|| ops_identifier::git_hash(&kernel_dir));
+    let provenance = Provenance {
+        kernel_builder: common::kernel_builder_version(),
+        kernel,
+    };
+
     let kernel_id = KernelIdentifier::new(&kernel_dir, build.general.name.python_name(), unique_id);
-    let file_set = create_pyproject_file_set(build, &kernel_id)?;
+    let file_set = create_pyproject_file_set(build, &kernel_id, Some(&provenance))?;
     file_set.write(&target_dir, force)?;
 
     Ok(())
@@ -65,9 +89,10 @@ pub fn clean_pyproject(
     let kernel_dir = check_or_infer_kernel_dir(kernel_dir)?;
     let target_dir = check_or_infer_target_dir(&kernel_dir, target_dir)?;
     let build = parse_build(&kernel_dir)?;
+    // Provenance is irrelevant when computing the set of files to clean.
     let kernel_id = KernelIdentifier::new(&kernel_dir, build.general.name.python_name(), unique_id);
 
-    let generated_files = create_pyproject_file_set(build, &kernel_id)?.into_names();
+    let generated_files = create_pyproject_file_set(build, &kernel_id, None)?.into_names();
 
     if generated_files.is_empty() {
         eprintln!("No generated artifacts found to clean.");

kernel-builder/src/pyproject/ops_identifier.rs

@@ -3,6 +3,7 @@ use std::path::Path;
 use eyre::{Result, WrapErr};
 use git2::Repository;
 use kernels_data::config::Backend;
+use kernels_data::metadata::GitHash;
 use rand::Rng;
 
 pub fn random_identifier() -> String {
@@ -28,6 +29,25 @@ pub fn git_identifier(target_dir: impl AsRef<Path>) -> Result<String> {
     Ok(if dirty { format!("{rev}_dirty") } else { rev })
 }
 
+pub fn git_hash(target_dir: impl AsRef<Path>) -> Option<GitHash> {
+    let repo = Repository::discover(target_dir.as_ref()).ok()?;
+    let head = repo.head().ok()?;
+    let commit = head.peel_to_commit().ok()?;
+    let sha = commit.id().to_string();
+
+    let mut status_options = git2::StatusOptions::new();
+    status_options.include_untracked(false); // Ignore untracked files (like generated CMake files)
+    status_options.exclude_submodules(true);
+    let dirty = !repo.statuses(Some(&mut status_options)).ok()?.is_empty();
+
+    Some(GitHash { sha, dirty })
+}
+
+/// Uniquely identifies a kernel for the purpose of ops-name generation.
+///
+/// This type is intentionally small and atomic: it carries only what is
+/// needed to name a kernel's ops. Build provenance (git SHA, dirty flags,
+/// `kernel-builder` version) is handled separately when writing metadata.
 pub struct KernelIdentifier {
     name: String,
     unique_id: String,
@@ -63,3 +83,23 @@ impl KernelIdentifier {
         &self.unique_id
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn kernel_identifier_uses_provided_unique_id() {
+        let tmp = tempfile::tempdir().unwrap();
+        let id = KernelIdentifier::new(tmp.path(), "relu".to_owned(), Some("rev123".to_owned()));
+
+        assert_eq!(id.unique_id(), "rev123");
+        assert_eq!(id.to_string_for_backend(Backend::Cuda), "_relu_cuda_rev123");
+    }
+
+    #[test]
+    fn git_hash_is_none_in_non_git_dir() {
+        let tmp = tempfile::tempdir().unwrap();
+        assert!(git_hash(tmp.path()).is_none());
+    }
+}

kernel-builder/src/pyproject/torch/mod.rs

@@ -4,6 +4,7 @@ use std::path::PathBuf;
 use eyre::{bail, Context, Result};
 use itertools::Itertools;
 use kernels_data::config::{Backend, Build, General, Torch};
+use kernels_data::metadata::Provenance;
 use minijinja::context;
 
 use crate::pyproject::common::{
@@ -271,6 +272,7 @@ pub fn write_torch_ext(
     env: &minijinja::Environment,
     build: &Build,
     kernel_id: &KernelIdentifier,
+    provenance: Option<&Provenance>,
 ) -> Result<FileSet> {
     let torch_ext = match build.framework.torch() {
         Some(torch_ext) => torch_ext,
@@ -296,7 +298,7 @@ pub fn write_torch_ext(
 
     write_torch_registration_macros(&mut file_set)?;
 
-    write_metadata(build, kernel_id, &mut file_set)?;
+    write_metadata(build, kernel_id, provenance, &mut file_set)?;
 
     Ok(file_set)
 }