From 03795978d693a6e0872f6a36187de55888744cfd Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Fri, 19 Jun 2026 12:05:27 +0530 Subject: [PATCH 01/21] feat: record git status. --- docs/source/kernel-requirements.md | 14 ++ flake.nix | 10 ++ kernel-builder/build.rs | 49 ++++++ kernel-builder/src/main.rs | 37 ++++- kernel-builder/src/pyproject/common.rs | 22 ++- kernel-builder/src/pyproject/mod.rs | 34 ++++- .../src/pyproject/ops_identifier.rs | 102 ++++++++++++- kernel-builder/src/upload.rs | 120 ++++++++++++++- kernels-data/bindings/python/kernels_data.pyi | 63 ++++++++ kernels-data/bindings/python/src/lib.rs | 143 +++++++++++++++++- kernels-data/src/metadata.rs | 113 ++++++++++++++ kernels/src/kernels/utils.py | 24 +++ kernels/tests/test_dirty_build.py | 120 +++++++++++++++ nix-builder/lib/build.nix | 10 ++ nix-builder/lib/extension/torch/arch.nix | 6 +- nix-builder/lib/extension/torch/no-arch.nix | 6 +- nix-builder/lib/extension/tvm-ffi/arch.nix | 6 +- nix-builder/lib/gen-flake-outputs.nix | 43 +++++- 18 files changed, 899 insertions(+), 23 deletions(-) create mode 100644 kernels/tests/test_dirty_build.py diff --git a/docs/source/kernel-requirements.md b/docs/source/kernel-requirements.md index eac3ead6e..8757defaa 100644 --- a/docs/source/kernel-requirements.md +++ b/docs/source/kernel-requirements.md @@ -81,6 +81,20 @@ metadata. Currently the following top-level keys are supported: - `digest` (`Digest`, required): hash digest of the kernel files. - `python-depends` (`list[str]`, optional): list of Python dependencies from a curated set of Python dependencies. +- `build-info` (`dict`, optional): provenance of the build, used to flag + non-reproducible (dirty) builds. It contains two optional sub-objects: + - `kernel-builder`: the `kernel-builder` that produced the build, with its + `version` (`str`), the `sha` (`str`) of the `kernel-builder` source it was + built from (when known), and a `dirty` (`bool`) flag that is `true` when + `kernel-builder` was built from a source tree with uncommitted changes. + - `kernel`: the kernel source that was built, with its commit `sha` (`str`) + and a `dirty` (`bool`) flag that is `true` when the kernel source had + uncommitted changes. + + When either `dirty` flag is set, the kernel was built from uncommitted + sources and cannot be reliably reproduced. `kernels` warns when loading such + a build, `kernel-builder` warns when uploading it, and a warning banner is + added to the Hub repository's README. Example `metadata.json`: diff --git a/flake.nix b/flake.nix index aa4b29c57..00c034799 100644 --- a/flake.nix +++ b/flake.nix @@ -37,6 +37,14 @@ ; inherit (import ./nix-builder/lib/cache.nix) mkForCache; + # Git provenance of `kernel-builder` itself, recorded in the build + # metadata of kernels it builds. `self` here is the `kernel-builder` + # flake (as resolved in the consuming kernel's lock). It is captured + # here because the `self` argument of `genKernelFlakeOutputs` below + # shadows it with the *kernel's* `self`. + builderRev = self.rev or self.dirtyRev or null; + builderDirty = !(self ? rev); + systems = with flake-utils.lib.system; [ aarch64-darwin aarch64-linux @@ -109,6 +117,8 @@ path rev self + builderRev + builderDirty doGetKernelCheck pythonCheckInputs pythonNativeCheckInputs diff --git a/kernel-builder/build.rs b/kernel-builder/build.rs index 3acdfe018..4022e5dff 100644 --- a/kernel-builder/build.rs +++ b/kernel-builder/build.rs @@ -1,3 +1,52 @@ +use std::process::Command; + fn main() { minijinja_embed::embed_templates!("src/pyproject/templates"); + emit_git_info(); +} + +fn emit_git_info() { + let sha = env_var("KERNEL_BUILDER_GIT_SHA").or_else(|| git_output(&["rev-parse", "HEAD"])); + if let Some(sha) = sha { + println!("cargo:rustc-env=KERNEL_BUILDER_GIT_SHA={sha}"); + } + + let dirty = match env_var("KERNEL_BUILDER_GIT_DIRTY") { + Some(value) => is_truthy(&value), + // Only consider tracked files; untracked files (e.g. generated build + // artifacts) should not mark the build as dirty. + None => git_output(&["status", "--porcelain", "--untracked-files=no"]) + .map(|out| !out.is_empty()) + .unwrap_or(false), + }; + println!( + "cargo:rustc-env=KERNEL_BUILDER_GIT_DIRTY={}", + if dirty { "1" } else { "0" } + ); + + println!("cargo:rerun-if-env-changed=KERNEL_BUILDER_GIT_SHA"); + println!("cargo:rerun-if-env-changed=KERNEL_BUILDER_GIT_DIRTY"); + println!("cargo:rerun-if-changed=../.git/HEAD"); + println!("cargo:rerun-if-changed=../.git/index"); +} + +fn env_var(name: &str) -> Option { + std::env::var(name).ok().filter(|s| !s.is_empty()) +} + +fn is_truthy(value: &str) -> bool { + value == "1" || value.eq_ignore_ascii_case("true") +} + +fn git_output(args: &[&str]) -> Option { + let output = Command::new("git").args(args).output().ok()?; + if !output.status.success() { + return None; + } + let s = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if s.is_empty() { + None + } else { + Some(s) + } } diff --git a/kernel-builder/src/main.rs b/kernel-builder/src/main.rs index 1a7a7a8bc..c26c2e483 100644 --- a/kernel-builder/src/main.rs +++ b/kernel-builder/src/main.rs @@ -174,6 +174,28 @@ enum Commands { /// kernel name to avoid name collisions. (e.g. Git SHA) #[arg(long)] unique_id: Option, + + /// Full commit SHA of the kernel source, recorded in the build + /// metadata. When absent, it is detected from the kernel's git + /// repository (used by Nix builds where the source has no `.git`). + #[arg(long)] + kernel_sha: Option, + + /// Mark the kernel source as having uncommitted changes in the build + /// metadata. Only meaningful together with `--kernel-sha`. + #[arg(long)] + kernel_dirty: bool, + + /// Full commit SHA of the `kernel-builder` source, recorded in the + /// build metadata. When absent, the SHA baked in at compile time is + /// used (Nix builds pass this since the sandbox has no `.git`). + #[arg(long)] + kernel_builder_sha: Option, + + /// Mark `kernel-builder` as having uncommitted changes in the build + /// metadata. Only meaningful together with `--kernel-builder-sha`. + #[arg(long)] + kernel_builder_dirty: bool, }, /// Spawn a kernel development shell. @@ -365,7 +387,20 @@ fn main() -> Result<()> { force, target_dir, unique_id, - } => create_pyproject(kernel_dir, target_dir, force, unique_id), + kernel_sha, + kernel_dirty, + kernel_builder_sha, + kernel_builder_dirty, + } => create_pyproject( + kernel_dir, + target_dir, + force, + unique_id, + kernel_sha, + kernel_dirty, + kernel_builder_sha, + kernel_builder_dirty, + ), Commands::Devshell { kernel_dir, variant, diff --git a/kernel-builder/src/pyproject/common.rs b/kernel-builder/src/pyproject/common.rs index c03ddb2e4..16ba5d13b 100644 --- a/kernel-builder/src/pyproject/common.rs +++ b/kernel-builder/src/pyproject/common.rs @@ -4,7 +4,7 @@ use eyre::Result; use itertools::Itertools; use kernels_data::config::{Backend, General}; -use kernels_data::metadata::{BackendInfo, Metadata}; +use kernels_data::metadata::{BackendInfo, BuildInfo, KernelBuilderInfo, Metadata}; use crate::pyproject::ops_identifier::KernelIdentifier; use crate::pyproject::FileSet; @@ -20,11 +20,30 @@ pub fn write_compat_py(file_set: &mut FileSet) -> Result<()> { Ok(()) } +fn kernel_builder_info() -> KernelBuilderInfo { + KernelBuilderInfo { + version: env!("CARGO_PKG_VERSION").to_owned(), + sha: option_env!("KERNEL_BUILDER_GIT_SHA").map(str::to_owned), + dirty: matches!(option_env!("KERNEL_BUILDER_GIT_DIRTY"), Some("1")), + } +} + pub fn write_metadata( general: &General, kernel_id: &KernelIdentifier, file_set: &mut FileSet, ) -> Result<()> { + // Prefer externally-provided `kernel-builder` provenance (e.g. from Nix), + // falling back to the provenance baked in at compile time. + let kernel_builder = kernel_id + .kernel_builder() + .cloned() + .unwrap_or_else(kernel_builder_info); + let build_info = BuildInfo { + kernel_builder: Some(kernel_builder), + kernel: kernel_id.git_info().cloned(), + }; + for backend in &Backend::all() { let writer = file_set.entry(format!("metadata-{backend}.json")); @@ -51,6 +70,7 @@ pub fn write_metadata( backend_type: *backend, }, digest: None, + build_info: Some(build_info.clone()), }; serde_json::to_writer_pretty(writer, &metadata)?; diff --git a/kernel-builder/src/pyproject/mod.rs b/kernel-builder/src/pyproject/mod.rs index 5582d5b6a..be33e6bed 100644 --- a/kernel-builder/src/pyproject/mod.rs +++ b/kernel-builder/src/pyproject/mod.rs @@ -6,6 +6,7 @@ use std::{ use eyre::{bail, Result}; use kernels_data::config::{Build, Framework}; +use kernels_data::metadata::{GitInfo, KernelBuilderInfo}; use minijinja::Environment; use crate::{ @@ -39,16 +40,38 @@ pub fn create_pyproject_file_set(build: Build, kernel_id: &KernelIdentifier) -> Ok(file_set) } +#[allow(clippy::too_many_arguments)] pub fn create_pyproject( kernel_dir: Option, target_dir: Option, force: bool, unique_id: Option, + kernel_sha: Option, + kernel_dirty: bool, + kernel_builder_sha: Option, + kernel_builder_dirty: bool, ) -> Result<()> { let kernel_dir = check_or_infer_kernel_dir(kernel_dir)?; let target_dir = check_or_infer_target_dir(&kernel_dir, target_dir)?; let build = parse_build(&kernel_dir)?; - let kernel_id = KernelIdentifier::new(&kernel_dir, build.general.name.python_name(), unique_id); + let git_override = kernel_sha.map(|sha| GitInfo { + sha, + dirty: kernel_dirty, + }); + // The version is always that of the running `kernel-builder`; only the + // git provenance is supplied externally (e.g. by Nix). + let kernel_builder_override = kernel_builder_sha.map(|sha| KernelBuilderInfo { + version: env!("CARGO_PKG_VERSION").to_owned(), + sha: Some(sha), + dirty: kernel_builder_dirty, + }); + let kernel_id = KernelIdentifier::new( + &kernel_dir, + build.general.name.python_name(), + unique_id, + git_override, + kernel_builder_override, + ); let file_set = create_pyproject_file_set(build, &kernel_id)?; file_set.write(&target_dir, force)?; @@ -65,7 +88,14 @@ pub fn clean_pyproject( let kernel_dir = check_or_infer_kernel_dir(kernel_dir)?; let target_dir = check_or_infer_target_dir(&kernel_dir, target_dir)?; let build = parse_build(&kernel_dir)?; - let kernel_id = KernelIdentifier::new(&kernel_dir, build.general.name.python_name(), unique_id); + // Provenance is irrelevant when computing the set of files to clean. + let kernel_id = KernelIdentifier::new( + &kernel_dir, + build.general.name.python_name(), + unique_id, + None, + None, + ); let generated_files = create_pyproject_file_set(build, &kernel_id)?.into_names(); diff --git a/kernel-builder/src/pyproject/ops_identifier.rs b/kernel-builder/src/pyproject/ops_identifier.rs index 46157260a..5c8b7ecf0 100644 --- a/kernel-builder/src/pyproject/ops_identifier.rs +++ b/kernel-builder/src/pyproject/ops_identifier.rs @@ -3,6 +3,7 @@ use std::path::Path; use eyre::{Result, WrapErr}; use git2::Repository; use kernels_data::config::Backend; +use kernels_data::metadata::{GitInfo, KernelBuilderInfo}; use rand::Rng; pub fn random_identifier() -> String { @@ -28,9 +29,25 @@ pub fn git_identifier(target_dir: impl AsRef) -> Result { Ok(if dirty { format!("{rev}_dirty") } else { rev }) } +pub fn git_info(target_dir: impl AsRef) -> Option { + let repo = Repository::discover(target_dir.as_ref()).ok()?; + let head = repo.head().ok()?; + let commit = head.peel_to_commit().ok()?; + let sha = commit.id().to_string(); + + let mut status_options = git2::StatusOptions::new(); + status_options.include_untracked(false); // Ignore untracked files (like generated CMake files) + status_options.exclude_submodules(true); + let dirty = !repo.statuses(Some(&mut status_options)).ok()?.is_empty(); + + Some(GitInfo { sha, dirty }) +} + pub struct KernelIdentifier { name: String, unique_id: String, + git_info: Option, + kernel_builder: Option, } impl KernelIdentifier { @@ -41,19 +58,44 @@ impl KernelIdentifier { /// source revision. If this identefier is not provided, a Git short /// hash is extracted from the repository of `target_dir`. If that /// fails, a random identifier is generated. - pub fn new(kernel_dir: impl AsRef, name: String, unique_id: Option) -> Self { + pub fn new( + kernel_dir: impl AsRef, + name: String, + unique_id: Option, + git_override: Option, + kernel_builder: Option, + ) -> Self { + // Prefer an explicitly provided git provenance (e.g. passed by Nix + // builds, where the source tree has no `.git`); fall back to detecting + // it from the kernel's git repository. + let git_info = git_override.or_else(|| git_info(kernel_dir.as_ref())); let unique_id = unique_id.unwrap_or_else(|| match git_identifier(kernel_dir.as_ref()) { Ok(rev) => rev, Err(_) => random_identifier(), }); - Self { name, unique_id } + Self { + name, + unique_id, + git_info, + kernel_builder, + } } pub fn name(&self) -> &str { &self.name } + pub fn git_info(&self) -> Option<&GitInfo> { + self.git_info.as_ref() + } + + /// Externally-provided `kernel-builder` provenance, when available. When + /// `None`, the compile-time baked provenance is used instead. + pub fn kernel_builder(&self) -> Option<&KernelBuilderInfo> { + self.kernel_builder.as_ref() + } + /// Create the kernel identifier string for a given backend. pub fn to_string_for_backend(&self, backend: Backend) -> String { format!("_{}_{}_{}", self.name, backend, self.unique_id) @@ -63,3 +105,59 @@ impl KernelIdentifier { &self.unique_id } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn explicit_provenance_overrides_are_stored() { + let tmp = tempfile::tempdir().unwrap(); + let git = GitInfo { + sha: "a".repeat(40), + dirty: true, + }; + let kernel_builder = KernelBuilderInfo { + version: "0.16.0-dev0".to_owned(), + sha: Some("b".repeat(40)), + dirty: false, + }; + + let id = KernelIdentifier::new( + tmp.path(), + "relu".to_owned(), + Some("rev123".to_owned()), + Some(git), + Some(kernel_builder), + ); + + assert_eq!(id.unique_id(), "rev123"); + assert_eq!(id.to_string_for_backend(Backend::Cuda), "_relu_cuda_rev123"); + + let git = id.git_info().expect("kernel git info"); + assert_eq!(git.sha, "a".repeat(40)); + assert!(git.dirty); + + let kb = id.kernel_builder().expect("kernel-builder info"); + assert_eq!(kb.sha.as_deref(), Some(&"b".repeat(40)[..])); + assert!(!kb.dirty); + } + + #[test] + fn no_provenance_in_non_git_dir() { + let tmp = tempfile::tempdir().unwrap(); + let id = KernelIdentifier::new( + tmp.path(), + "relu".to_owned(), + Some("x".to_owned()), + None, + None, + ); + + // No overrides supplied and the temp dir is not a git repository, so + // no kernel provenance is recorded. The kernel-builder provenance is + // filled in later from the compile-time default in `write_metadata`. + assert!(id.git_info().is_none()); + assert!(id.kernel_builder().is_none()); + } +} diff --git a/kernel-builder/src/upload.rs b/kernel-builder/src/upload.rs index 2113c2f33..ebfa06d2b 100644 --- a/kernel-builder/src/upload.rs +++ b/kernel-builder/src/upload.rs @@ -150,6 +150,11 @@ fn run_upload_typed(args: UploadArgs) -> Result<()> { build_dir.display() ); + let dirty_build = detect_dirty_build(&variants); + if dirty_build { + eprintln!("{DIRTY_BUILD_WARNING}"); + } + let (repo_id, branch) = get_repo_and_branch(&kernel_dir, args.repo_id, args.branch, &variants)?; let repo_url = match api @@ -207,6 +212,7 @@ fn run_upload_typed(args: UploadArgs) -> Result<()> { collect_readme_commit_ops( &kernel_dir, + dirty_build, operations_by_branch .entry(MAIN_BRANCH.to_owned()) .or_default(), @@ -363,16 +369,51 @@ fn collect_benchmark_commit_ops( } /// Collect README commit operation: upload build/CARD.md as README.md. -fn collect_readme_commit_ops(kernel_dir: &Path, operations: &mut Vec) { +fn collect_readme_commit_ops( + kernel_dir: &Path, + dirty_build: bool, + operations: &mut Vec, +) { let Ok(card_path) = discover_build_file(kernel_dir, "CARD.md") else { return; }; + + let source = if dirty_build { + match fs::read_to_string(&card_path) { + Ok(content) => AddSource::Bytes(prepend_dirty_banner(&content).into_bytes()), + Err(err) => { + eprintln!( + "Warning: cannot read `{}` to add dirty-build banner: {err}", + card_path.display() + ); + AddSource::File(card_path) + } + } + } else { + AddSource::File(card_path) + }; + operations.push(CommitOperation::Add { path_in_repo: "README.md".to_owned(), - source: AddSource::File(card_path), + source, }); } +fn prepend_dirty_banner(content: &str) -> String { + let banner = format!("{DIRTY_BUILD_BANNER}\n\n"); + + // Keep the YAML front matter (delimited by `---`) at the very top. + if let Some(rest) = content.strip_prefix("---\n") { + if let Some(end) = rest.find("\n---\n") { + let split = "---\n".len() + end + "\n---\n".len(); + let (front_matter, body) = content.split_at(split); + return format!("{front_matter}\n{banner}{}", body.trim_start_matches('\n')); + } + } + + format!("{banner}{content}") +} + /// Collect build artifact commit operations: add variant files, delete stale ones. fn collect_build_commit_ops( build_dir: &Path, @@ -456,6 +497,29 @@ fn discover_build_file( ); } +const DIRTY_BUILD_WARNING: &str = "\ +Warning: one or more build variants were built from uncommitted (dirty) \ +sources.\n Such builds are not reproducible. A warning banner will be \ +added to the\n repository README."; + +/// Markdown banner prepended to the Hub README for dirty builds. +const DIRTY_BUILD_BANNER: &str = "> [!WARNING]\n\ + > This kernel was built from uncommitted (dirty) sources and is therefore \ + **not reproducible**. It should not be relied upon for production use."; + +fn detect_dirty_build(variants: &[PathBuf]) -> bool { + variants.iter().any(|variant| { + let metadata_path = variant.join("metadata.json"); + let Ok(file) = File::open(&metadata_path) else { + return false; + }; + match Metadata::from_reader(BufReader::new(file)) { + Ok(metadata) => metadata.build_info.as_ref().is_some_and(|bi| bi.is_dirty()), + Err(_) => false, + } + }) +} + /// Determine the branch name (`v{version}`) from variant metadata. fn detect_branch_from_metadata(variants: &[PathBuf]) -> Result> { let mut versions: HashSet = HashSet::new(); @@ -513,22 +577,68 @@ mod tests { fs::write(kernel_dir.join("build/CARD.md"), "# Readme").unwrap(); let mut operations = vec![]; - collect_readme_commit_ops(kernel_dir, &mut operations); + collect_readme_commit_ops(kernel_dir, false, &mut operations); assert_eq!(operations.len(), 1); match &operations[0] { - CommitOperation::Add { path_in_repo, .. } => { + CommitOperation::Add { + path_in_repo, + source, + } => { assert_eq!(path_in_repo, "README.md"); + assert!(matches!(source, AddSource::File(_))); } _ => panic!("Expected Add operation"), } } + #[test] + fn test_collect_readme_commit_ops_dirty() { + let temp_dir = tempfile::tempdir().unwrap(); + let kernel_dir = temp_dir.path(); + + fs::create_dir_all(kernel_dir.join("build")).unwrap(); + fs::write(kernel_dir.join("build/CARD.md"), "# Readme").unwrap(); + + let mut operations = vec![]; + collect_readme_commit_ops(kernel_dir, true, &mut operations); + + assert_eq!(operations.len(), 1); + match &operations[0] { + CommitOperation::Add { + path_in_repo, + source, + } => { + assert_eq!(path_in_repo, "README.md"); + match source { + AddSource::Bytes(bytes) => { + let content = String::from_utf8(bytes.clone()).unwrap(); + assert!(content.contains("[!WARNING]")); + assert!(content.contains("# Readme")); + } + _ => panic!("Expected Bytes source for dirty build"), + } + } + _ => panic!("Expected Add operation"), + } + } + + #[test] + fn test_prepend_dirty_banner_with_front_matter() { + let content = "---\nlicense: apache-2.0\n---\n# Title\n\nBody."; + let result = prepend_dirty_banner(content); + assert!(result.starts_with("---\nlicense: apache-2.0\n---\n")); + assert!(result.contains("[!WARNING]")); + let banner_pos = result.find("[!WARNING]").unwrap(); + let title_pos = result.find("# Title").unwrap(); + assert!(banner_pos < title_pos); + } + #[test] fn test_collect_readme_commit_ops_no_card() { let temp_dir = tempfile::tempdir().unwrap(); let mut operations = vec![]; - collect_readme_commit_ops(temp_dir.path(), &mut operations); + collect_readme_commit_ops(temp_dir.path(), false, &mut operations); assert!(operations.is_empty()); } diff --git a/kernels-data/bindings/python/kernels_data.pyi b/kernels-data/bindings/python/kernels_data.pyi index f0397c8e4..9feb36eb0 100644 --- a/kernels-data/bindings/python/kernels_data.pyi +++ b/kernels-data/bindings/python/kernels_data.pyi @@ -7,7 +7,10 @@ from typing import Optional, final __all__ = [ "Backend", "BackendInfo", + "BuildInfo", "DigestAlgorithm", + "GitInfo", + "KernelBuilderInfo", "KernelName", "Metadata", "Digest", @@ -63,6 +66,64 @@ class BackendInfo: def __repr__(self) -> str: ... +@final +class GitInfo: + """Git provenance (commit SHA and dirty state) of a source tree.""" + + @property + def sha(self) -> str: + """Full 40-character commit SHA.""" + ... + + @property + def dirty(self) -> bool: + """Whether the working tree had uncommitted changes to tracked files.""" + ... + + def __repr__(self) -> str: ... + +@final +class KernelBuilderInfo: + """Provenance of the `kernel-builder` that produced a build.""" + + @property + def version(self) -> str: + """`kernel-builder` package version.""" + ... + + @property + def sha(self) -> Optional[str]: + """Commit SHA of the `kernel-builder` source, when known.""" + ... + + @property + def dirty(self) -> bool: + """Whether `kernel-builder` was built from a dirty source tree.""" + ... + + def __repr__(self) -> str: ... + +@final +class BuildInfo: + """Build provenance: git state of the `kernel-builder` and kernel source.""" + + @property + def kernel_builder(self) -> Optional[KernelBuilderInfo]: + """Provenance of the `kernel-builder` that produced the build.""" + ... + + @property + def kernel(self) -> Optional[GitInfo]: + """Git provenance of the kernel source that was built.""" + ... + + @property + def dirty(self) -> bool: + """Whether either the `kernel-builder` or the kernel source was dirty.""" + ... + + def __repr__(self) -> str: ... + @final class Version: """A dotted numeric version (e.g. `12.8.0`). @@ -263,4 +324,6 @@ class Metadata: def backend(self) -> BackendInfo: ... @property def digest(self) -> Optional[Digest]: ... + @property + def build_info(self) -> Optional[BuildInfo]: ... def __repr__(self) -> str: ... diff --git a/kernels-data/bindings/python/src/lib.rs b/kernels-data/bindings/python/src/lib.rs index 25d234caa..a34518c9b 100644 --- a/kernels-data/bindings/python/src/lib.rs +++ b/kernels-data/bindings/python/src/lib.rs @@ -6,7 +6,7 @@ use std::str::FromStr; use kernels_data::config::{Backend, KernelName}; use kernels_data::digest::{Digest, DigestAlgorithm, DigestViolation}; -use kernels_data::metadata::{BackendInfo, Metadata}; +use kernels_data::metadata::{BackendInfo, BuildInfo, GitInfo, KernelBuilderInfo, Metadata}; use kernels_data::version::Version; use pyo3::Bound as PyBound; use pyo3::exceptions::{PyException, PyOSError, PyRuntimeError, PyValueError}; @@ -188,6 +188,130 @@ impl PyBackendInfo { } } +#[pyclass(name = "GitInfo", frozen)] +#[derive(Clone, Debug)] +struct PyGitInfo { + sha: String, + dirty: bool, +} + +impl From for PyGitInfo { + fn from(g: GitInfo) -> Self { + Self { + sha: g.sha, + dirty: g.dirty, + } + } +} + +#[pymethods] +impl PyGitInfo { + #[getter] + fn sha(&self) -> &str { + &self.sha + } + + #[getter] + fn dirty(&self) -> bool { + self.dirty + } + + fn __repr__(&self) -> String { + format!("GitInfo(sha={:?}, dirty={})", self.sha, self.dirty) + } +} + +#[pyclass(name = "KernelBuilderInfo", frozen)] +#[derive(Clone, Debug)] +struct PyKernelBuilderInfo { + version: String, + sha: Option, + dirty: bool, +} + +impl From for PyKernelBuilderInfo { + fn from(kb: KernelBuilderInfo) -> Self { + Self { + version: kb.version, + sha: kb.sha, + dirty: kb.dirty, + } + } +} + +#[pymethods] +impl PyKernelBuilderInfo { + #[getter] + fn version(&self) -> &str { + &self.version + } + + #[getter] + fn sha(&self) -> Option<&str> { + self.sha.as_deref() + } + + #[getter] + fn dirty(&self) -> bool { + self.dirty + } + + fn __repr__(&self) -> String { + format!( + "KernelBuilderInfo(version={:?}, sha={:?}, dirty={})", + self.version, self.sha, self.dirty + ) + } +} + +#[pyclass(name = "BuildInfo", frozen)] +#[derive(Clone, Debug)] +struct PyBuildInfo { + kernel_builder: Option, + kernel: Option, +} + +impl From for PyBuildInfo { + fn from(b: BuildInfo) -> Self { + Self { + kernel_builder: b.kernel_builder.map(Into::into), + kernel: b.kernel.map(Into::into), + } + } +} + +#[pymethods] +impl PyBuildInfo { + #[getter] + fn kernel_builder(&self) -> Option { + self.kernel_builder.clone() + } + + #[getter] + fn kernel(&self) -> Option { + self.kernel.clone() + } + + /// Whether either the `kernel-builder` or the kernel source was dirty. + #[getter] + fn dirty(&self) -> bool { + self.kernel_builder.as_ref().is_some_and(|kb| kb.dirty) + || self.kernel.as_ref().is_some_and(|k| k.dirty) + } + + fn __repr__(&self) -> String { + format!( + "BuildInfo(kernel_builder={}, kernel={})", + self.kernel_builder + .as_ref() + .map_or("None".to_string(), |kb| kb.__repr__()), + self.kernel + .as_ref() + .map_or("None".to_string(), |k| k.__repr__()) + ) + } +} + /// Parsed `metadata.json` for a kernel build variant. #[pyclass(name = "Metadata", frozen)] #[derive(Clone, Debug)] @@ -201,6 +325,7 @@ struct PyMetadata { python_depends: Vec, backend: PyBackendInfo, digest: Option, + build_info: Option, } impl From for PyMetadata { @@ -215,6 +340,7 @@ impl From for PyMetadata { python_depends: m.python_depends, backend: m.backend.into(), digest: m.digest.map(Into::into), + build_info: m.build_info.map(Into::into), } } } @@ -293,9 +419,14 @@ impl PyMetadata { self.digest.clone() } + #[getter] + fn build_info(&self) -> Option { + self.build_info.clone() + } + fn __repr__(&self) -> String { format!( - "Metadata(id={}, name={:?}, version={:?}, license={:?}, upstream={:?}, source={:?}, python_depends={:?}, backend={}, digest={})", + "Metadata(id={}, name={:?}, version={:?}, license={:?}, upstream={:?}, source={:?}, python_depends={:?}, backend={}, digest={}, build_info={})", self.id, self.name, self.version, @@ -306,7 +437,10 @@ impl PyMetadata { self.backend.__repr__(), self.digest .as_ref() - .map_or("None".to_string(), |sd| sd.__repr__()) + .map_or("None".to_string(), |sd| sd.__repr__()), + self.build_info + .as_ref() + .map_or("None".to_string(), |bi| bi.__repr__()) ) } } @@ -530,6 +664,9 @@ impl PyDigest { fn kernels_data_py(m: &PyBound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/kernels-data/src/metadata.rs b/kernels-data/src/metadata.rs index 109c93fdc..bb88f68b0 100644 --- a/kernels-data/src/metadata.rs +++ b/kernels-data/src/metadata.rs @@ -15,6 +15,40 @@ pub struct BackendInfo { pub archs: Option>, } +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub struct GitInfo { + pub sha: String, + pub dirty: bool, +} + +/// Provenance of the `kernel-builder` that produced a build. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub struct KernelBuilderInfo { + pub version: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub sha: Option, + pub dirty: bool, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub struct BuildInfo { + #[serde(skip_serializing_if = "Option::is_none")] + pub kernel_builder: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub kernel: Option, +} + +impl BuildInfo { + /// Whether either the `kernel-builder` or the kernel source was dirty. + pub fn is_dirty(&self) -> bool { + self.kernel_builder.as_ref().is_some_and(|kb| kb.dirty) + || self.kernel.as_ref().is_some_and(|k| k.dirty) + } +} + /// Kernel metadata. #[derive(Debug, Deserialize, Serialize)] #[serde(rename_all = "kebab-case")] @@ -31,6 +65,8 @@ pub struct Metadata { pub backend: BackendInfo, #[serde(skip_serializing_if = "Option::is_none")] pub digest: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub build_info: Option, } impl Metadata { @@ -52,3 +88,80 @@ impl FromStr for Metadata { Ok(serde_json::from_str(s)?) } } + +#[cfg(test)] +mod tests { + use super::*; + + const METADATA_NO_BUILD_INFO: &str = r#"{ + "name": "relu", + "id": "_relu_cuda_abc1234", + "version": 1, + "license": "Apache-2.0", + "python-depends": [], + "backend": { "type": "cuda" } + }"#; + + const METADATA_WITH_BUILD_INFO: &str = r#"{ + "name": "relu", + "id": "_relu_cuda_abc1234", + "version": 1, + "license": "Apache-2.0", + "python-depends": [], + "backend": { "type": "cuda" }, + "build-info": { + "kernel-builder": { "version": "0.16.0-dev0", "sha": "1111111111111111111111111111111111111111", "dirty": true }, + "kernel": { "sha": "2222222222222222222222222222222222222222", "dirty": false } + } + }"#; + + #[test] + fn parses_metadata_without_build_info() { + let metadata: Metadata = METADATA_NO_BUILD_INFO.parse().unwrap(); + assert!(metadata.build_info.is_none()); + } + + #[test] + fn parses_and_reports_dirty_build_info() { + let metadata: Metadata = METADATA_WITH_BUILD_INFO.parse().unwrap(); + let build_info = metadata.build_info.expect("build-info should be present"); + assert!(build_info.is_dirty()); + + let kernel_builder = build_info.kernel_builder.unwrap(); + assert_eq!(kernel_builder.version, "0.16.0-dev0"); + assert!(kernel_builder.dirty); + assert_eq!(kernel_builder.sha.as_deref(), Some(&"1".repeat(40)[..])); + + let kernel = build_info.kernel.unwrap(); + assert!(!kernel.dirty); + assert_eq!(kernel.sha, "2".repeat(40)); + } + + #[test] + fn build_info_round_trips_with_kebab_case_keys() { + let metadata: Metadata = METADATA_WITH_BUILD_INFO.parse().unwrap(); + let json = serde_json::to_string(&metadata).unwrap(); + assert!(json.contains("\"build-info\"")); + assert!(json.contains("\"kernel-builder\"")); + + // Re-parsing the serialized form yields the same dirtiness. + let reparsed: Metadata = json.parse().unwrap(); + assert!(reparsed.build_info.unwrap().is_dirty()); + } + + #[test] + fn build_info_is_not_dirty_when_all_clean() { + let build_info = BuildInfo { + kernel_builder: Some(KernelBuilderInfo { + version: "0.16.0".to_owned(), + sha: None, + dirty: false, + }), + kernel: Some(GitInfo { + sha: "abc".to_owned(), + dirty: false, + }), + }; + assert!(!build_info.is_dirty()); + } +} diff --git a/kernels/src/kernels/utils.py b/kernels/src/kernels/utils.py index c54a59227..650443f16 100644 --- a/kernels/src/kernels/utils.py +++ b/kernels/src/kernels/utils.py @@ -204,6 +204,28 @@ def _validate_variant_dependencies(variant_path: Path) -> None: validate_dependencies(metadata.name.python_name, metadata.python_depends, _backend()) +def _warn_if_dirty_build(metadata: Metadata, repo_info: "RepoInfo | None") -> None: + build_info = metadata.build_info + if build_info is None or not build_info.dirty: + return + + where = f" of `{repo_info.repo_id}`" if repo_info is not None else "" + reasons = [] + kernel_builder = build_info.kernel_builder + if kernel_builder is not None and kernel_builder.dirty: + reasons.append("the `kernel-builder` had uncommitted changes") + kernel = build_info.kernel + if kernel is not None and kernel.dirty: + reasons.append("the kernel source had uncommitted changes") + + warnings.warn( + f"Kernel `{metadata.id}`{where} was built from a dirty source tree " + f"({' and '.join(reasons)}). Such builds are not reproducible and " + f"should not be relied upon.", + stacklevel=2, + ) + + def _import_from_path(variant_path: Path, repo_info: RepoInfo | None = None) -> ModuleType: if (loaded_kernel := _loaded_kernels.get(variant_path)) is not None: return loaded_kernel.module @@ -211,6 +233,8 @@ def _import_from_path(variant_path: Path, repo_info: RepoInfo | None = None) -> metadata = Metadata.read_from_file(variant_path / "metadata.json") module_name = metadata.name.python_name + _warn_if_dirty_build(metadata, repo_info) + file_path = variant_path / "__init__.py" if not file_path.exists(): file_path = variant_path / module_name / "__init__.py" diff --git a/kernels/tests/test_dirty_build.py b/kernels/tests/test_dirty_build.py new file mode 100644 index 000000000..e2a98f80a --- /dev/null +++ b/kernels/tests/test_dirty_build.py @@ -0,0 +1,120 @@ +import json +import warnings + +import pytest +from kernels_data import Metadata + +from kernels.utils import RepoInfo, _import_from_path, _loaded_kernels, _warn_if_dirty_build + + +def _metadata(build_info: dict | None) -> Metadata: + data: dict = { + "name": "relu", + "id": "_relu_cuda_abc1234", + "version": 1, + "license": "Apache-2.0", + "python-depends": [], + "backend": {"type": "cuda"}, + } + if build_info is not None: + data["build-info"] = build_info + return Metadata.from_bytes(json.dumps(data).encode("utf-8")) + + +def _clean_builder() -> dict: + return {"version": "0.16.0-dev0", "sha": "a" * 40, "dirty": False} + + +def _dirty_builder() -> dict: + return {"version": "0.16.0-dev0", "sha": "a" * 40, "dirty": True} + + +def _clean_kernel() -> dict: + return {"sha": "b" * 40, "dirty": False} + + +def _dirty_kernel() -> dict: + return {"sha": "b" * 40, "dirty": True} + + +@pytest.fixture +def fresh_registry(): + """Run the test against a clean loaded-kernel registry, restore on teardown.""" + saved = _loaded_kernels.copy() + _loaded_kernels.clear() + yield + _loaded_kernels.clear() + _loaded_kernels.update(saved) + + +def _warnings_for(metadata: Metadata, repo_info: RepoInfo | None = None) -> list[str]: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + _warn_if_dirty_build(metadata, repo_info) + return [str(w.message) for w in caught] + + +def test_no_build_info_does_not_warn(): + assert _warnings_for(_metadata(None)) == [] + + +def test_clean_build_does_not_warn(): + build_info = {"kernel-builder": _clean_builder(), "kernel": _clean_kernel()} + assert _warnings_for(_metadata(build_info)) == [] + + +def test_dirty_kernel_builder_warns(): + build_info = {"kernel-builder": _dirty_builder(), "kernel": _clean_kernel()} + messages = _warnings_for(_metadata(build_info)) + assert len(messages) == 1 + assert "`kernel-builder` had uncommitted changes" in messages[0] + assert "kernel source had uncommitted changes" not in messages[0] + + +def test_dirty_kernel_source_warns(): + build_info = {"kernel-builder": _clean_builder(), "kernel": _dirty_kernel()} + messages = _warnings_for(_metadata(build_info)) + assert len(messages) == 1 + assert "kernel source had uncommitted changes" in messages[0] + assert "`kernel-builder` had uncommitted changes" not in messages[0] + + +def test_both_dirty_warns_with_both_reasons(): + build_info = {"kernel-builder": _dirty_builder(), "kernel": _dirty_kernel()} + messages = _warnings_for(_metadata(build_info)) + assert len(messages) == 1 + assert "`kernel-builder` had uncommitted changes" in messages[0] + assert "kernel source had uncommitted changes" in messages[0] + + +def test_warning_includes_repo_id_when_available(): + build_info = {"kernel-builder": _dirty_builder(), "kernel": _clean_kernel()} + messages = _warnings_for( + _metadata(build_info), RepoInfo(repo_id="acme/relu", revision="main") + ) + assert "acme/relu" in messages[0] + + +def test_import_from_path_warns_on_dirty_build(tmp_path, fresh_registry): + variant_path = tmp_path / "variant" + (variant_path / "relu").mkdir(parents=True) + (variant_path / "relu" / "__init__.py").write_text("VALUE = 42\n") + + metadata = { + "name": "relu", + "id": "_relu_cuda_abc1234", + "version": 1, + "license": "Apache-2.0", + "python-depends": [], + "backend": {"type": "cuda"}, + "build-info": {"kernel-builder": _dirty_builder(), "kernel": _clean_kernel()}, + } + (variant_path / "metadata.json").write_text(json.dumps(metadata)) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + module = _import_from_path(variant_path) + + assert module.VALUE == 42 + dirty_warnings = [w for w in caught if "dirty source tree" in str(w.message)] + assert len(dirty_warnings) == 1 diff --git a/nix-builder/lib/build.nix b/nix-builder/lib/build.nix index 4b49871db..a2eeaaa16 100644 --- a/nix-builder/lib/build.nix +++ b/nix-builder/lib/build.nix @@ -113,6 +113,7 @@ rec { rev, doGetKernelCheck, stripRPath ? false, + provenanceArgs ? "", }: let inherit (lib) fileset; @@ -151,6 +152,7 @@ rec { doGetKernelCheck pythonDeps backendPythonDeps + provenanceArgs ; kernelName = kernelConfig.name; } @@ -166,6 +168,7 @@ rec { rev pythonDeps backendPythonDeps + provenanceArgs ; kernelName = kernelConfig.name; @@ -183,6 +186,7 @@ rec { rev pythonDeps backendPythonDeps + provenanceArgs ; inherit (kernelConfig) torchStableAbiVersion; @@ -199,6 +203,7 @@ rec { doGetKernelCheck, bundleOnly, buildSets, + provenanceArgs ? "", }: let kernelConfig = readKernelConfig path; @@ -212,6 +217,7 @@ rec { kernelConfig rev doGetKernelCheck + provenanceArgs ; stripRPath = true; }; @@ -227,6 +233,7 @@ rec { rev, doGetKernelCheck, buildSets, + provenanceArgs ? "", }: let extensions = mkDistTorchExtensions { @@ -235,6 +242,7 @@ rec { path rev doGetKernelCheck + provenanceArgs ; bundleOnly = true; }; @@ -334,6 +342,7 @@ rec { buildSets, doGetKernelCheck, pythonCheckInputs, + provenanceArgs ? "", }: let kernelConfig = readKernelConfig path; @@ -350,6 +359,7 @@ rec { kernelConfig rev doGetKernelCheck + provenanceArgs ; }; testPython = diff --git a/nix-builder/lib/extension/torch/arch.nix b/nix-builder/lib/extension/torch/arch.nix index d23655916..d16df70f2 100644 --- a/nix-builder/lib/extension/torch/arch.nix +++ b/nix-builder/lib/extension/torch/arch.nix @@ -71,6 +71,10 @@ # Revision to bake into the ops name. rev, + # Extra `create-pyproject` flags recording git provenance (commit SHA and + # dirty state) of the kernel source and `kernel-builder`. + provenanceArgs ? "", + src, }: @@ -132,7 +136,7 @@ stdenv.mkDerivation (prevAttrs: { mkdir -p $out cp -r --no-preserve=mode ${src}/* $out/ ${pkgs.kernel-builder}/bin/kernel-builder create-pyproject \ - --unique-id ${rev} $out + --unique-id ${rev} ${provenanceArgs} $out ''; preConfigure = diff --git a/nix-builder/lib/extension/torch/no-arch.nix b/nix-builder/lib/extension/torch/no-arch.nix index f89211156..c3264734a 100644 --- a/nix-builder/lib/extension/torch/no-arch.nix +++ b/nix-builder/lib/extension/torch/no-arch.nix @@ -41,6 +41,10 @@ pythonDeps, backendPythonDeps, + + # Extra `create-pyproject` flags recording git provenance (commit SHA and + # dirty state) of the kernel source and `kernel-builder`. + provenanceArgs ? "", }: # Extra validation - the environment should correspind to the build config. @@ -79,7 +83,7 @@ stdenv.mkDerivation (prevAttrs: { mkdir -p $out cp -r --no-preserve=mode ${src}/* $out/ ${pkgs.kernel-builder}/bin/kernel-builder create-pyproject \ - --unique-id ${rev} $out + --unique-id ${rev} ${provenanceArgs} $out ''; framework = "torch"; diff --git a/nix-builder/lib/extension/tvm-ffi/arch.nix b/nix-builder/lib/extension/tvm-ffi/arch.nix index ffc877cd9..6c32a238e 100644 --- a/nix-builder/lib/extension/tvm-ffi/arch.nix +++ b/nix-builder/lib/extension/tvm-ffi/arch.nix @@ -68,6 +68,10 @@ # Revision to bake into the ops name. rev, + # Extra `create-pyproject` flags recording git provenance (commit SHA and + # dirty state) of the kernel source and `kernel-builder`. + provenanceArgs ? "", + src, }: @@ -129,7 +133,7 @@ stdenv.mkDerivation (prevAttrs: { # Generate build files. postPatch = '' kernel-builder create-pyproject \ - --unique-id ${rev} . + --unique-id ${rev} ${provenanceArgs} . ''; preConfigure = diff --git a/nix-builder/lib/gen-flake-outputs.nix b/nix-builder/lib/gen-flake-outputs.nix index 5409edb3f..be8d07a79 100644 --- a/nix-builder/lib/gen-flake-outputs.nix +++ b/nix-builder/lib/gen-flake-outputs.nix @@ -11,6 +11,12 @@ rev ? null, self ? null, + # Git provenance of the `kernel-builder` itself (the flake revision it was + # evaluated from), recorded in the build metadata. `null`/`false` when + # `kernel-builder` is used from a non-git source (e.g. a local `path:`). + builderRev ? null, + builderDirty ? false, + doGetKernelCheck, pythonCheckInputs, pythonNativeCheckInputs, @@ -36,6 +42,31 @@ let revUnderscored = builtins.replaceStrings [ "-" ] [ "_" ] flakeRev; + # Extra `kernel-builder create-pyproject` flags that record the full git + # provenance (commit SHA + dirty state) of both the kernel source and + # `kernel-builder` in the build metadata. The Nix sandbox has no `.git`, so + # this information has to be passed in explicitly. + provenanceArgs = + let + kernelSha = + if self == null then + null + else if self ? rev then + self.rev + else if self ? dirtyRev then + lib.removeSuffix "-dirty" self.dirtyRev + else + null; + kernelDirty = self != null && !(self ? rev); + builderSha = if builderRev == null then null else lib.removeSuffix "-dirty" builderRev; + parts = + lib.optional (kernelSha != null) "--kernel-sha ${kernelSha}" + ++ lib.optional (kernelSha != null && kernelDirty) "--kernel-dirty" + ++ lib.optional (builderSha != null) "--kernel-builder-sha ${builderSha}" + ++ lib.optional (builderSha != null && builderDirty) "--kernel-builder-dirty"; + in + lib.concatStringsSep " " parts; + applicableBuildSets = build.applicableBuildSets { inherit path buildSets; }; kernelConfig = (import ./kernel-config.nix { inherit lib; }) path; @@ -153,12 +184,12 @@ in packages = let bundle = build.mkExtensionBundle { - inherit path doGetKernelCheck; + inherit path doGetKernelCheck provenanceArgs; buildSets = applicableBuildSets; rev = revUnderscored; }; ciTests = build.mkCiTests { - inherit path doGetKernelCheck pythonCheckInputs; + inherit path doGetKernelCheck pythonCheckInputs provenanceArgs; buildSets = applicableBuildSets; rev = revUnderscored; }; @@ -175,7 +206,7 @@ in builtins.map (backend: { name = backend; value = build.mkExtensionBundle { - inherit path doGetKernelCheck; + inherit path doGetKernelCheck provenanceArgs; buildSets = builtins.filter ( set: buildConfigBackend set.buildConfig == backend ) applicableBuildSets; @@ -241,7 +272,7 @@ in ++ (headOrEmpty (setsWithFramework "xpu")); in build.mkExtensionBundle { - inherit path doGetKernelCheck; + inherit path doGetKernelCheck provenanceArgs; buildSets = onePerFramework; rev = revUnderscored; }; @@ -254,7 +285,7 @@ in builtins.map (backend: { name = backend; value = build.mkExtensionBundle { - inherit path doGetKernelCheck; + inherit path doGetKernelCheck provenanceArgs; # It is too costly to build all variants in CI, so we just build one per framework. buildSets = headOrEmpty ( builtins.filter (set: set.buildConfig.framework == backend) buildSetsSorted @@ -277,7 +308,7 @@ in }; redistributable = build.mkDistTorchExtensions { - inherit path doGetKernelCheck; + inherit path doGetKernelCheck provenanceArgs; bundleOnly = false; rev = revUnderscored; buildSets = applicableBuildSets; From a6fa6384598afdc112a559bb425b556226dc498c Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Fri, 19 Jun 2026 12:51:09 +0530 Subject: [PATCH 02/21] up --- docs/source/kernel-requirements.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/source/kernel-requirements.md b/docs/source/kernel-requirements.md index 8757defaa..8e83ea655 100644 --- a/docs/source/kernel-requirements.md +++ b/docs/source/kernel-requirements.md @@ -96,6 +96,12 @@ metadata. Currently the following top-level keys are supported: a build, `kernel-builder` warns when uploading it, and a warning banner is added to the Hub repository's README. + > **Note:** For Nix builds, dirtiness follows Nix's flake tree status, which + > also counts **untracked** files (including an uncommitted `flake.lock`). + > Commit your `flake.lock` (and avoid stray untracked files) so that clean + > builds are not flagged as dirty. Local `create-pyproject` runs only + > consider changes to tracked files. + Example `metadata.json`: ```json From cd9493021f741d5996e264d37179d585191997c8 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Fri, 19 Jun 2026 14:03:49 +0530 Subject: [PATCH 03/21] docs: regenerate builder-cli.md for create-pyproject provenance flags Adds the --kernel-sha/--kernel-dirty and --kernel-builder-sha/ --kernel-builder-dirty flags to the generated CLI reference, fixing the check-cli-docs CI job. Co-Authored-By: Claude Opus 4.8 --- docs/source/builder-cli.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/builder-cli.md b/docs/source/builder-cli.md index a4c80025b..5a1503284 100644 --- a/docs/source/builder-cli.md +++ b/docs/source/builder-cli.md @@ -244,6 +244,10 @@ Generate CMake files for a kernel extension build * `-f`, `--force` — Force-overwrite existing files * `--unique-id ` — This is an optional unique identifier that is suffixed to the kernel name to avoid name collisions. (e.g. Git SHA) +* `--kernel-sha ` — Full commit SHA of the kernel source, recorded in the build metadata. When absent, it is detected from the kernel's git repository (used by Nix builds where the source has no `.git`) +* `--kernel-dirty` — Mark the kernel source as having uncommitted changes in the build metadata. Only meaningful together with `--kernel-sha` +* `--kernel-builder-sha ` — Full commit SHA of the `kernel-builder` source, recorded in the build metadata. When absent, the SHA baked in at compile time is used (Nix builds pass this since the sandbox has no `.git`) +* `--kernel-builder-dirty` — Mark `kernel-builder` as having uncommitted changes in the build metadata. Only meaningful together with `--kernel-builder-sha` From 27a2009f74ee36e6e27dd1f75234c76250a7833b Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Fri, 19 Jun 2026 14:17:50 +0530 Subject: [PATCH 04/21] nix: format gen-flake-outputs.nix with nixfmt Fixes the `nix fmt --ci` formatting check. Co-Authored-By: Claude Opus 4.8 --- nix-builder/lib/gen-flake-outputs.nix | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/nix-builder/lib/gen-flake-outputs.nix b/nix-builder/lib/gen-flake-outputs.nix index be8d07a79..e710695a6 100644 --- a/nix-builder/lib/gen-flake-outputs.nix +++ b/nix-builder/lib/gen-flake-outputs.nix @@ -189,7 +189,12 @@ in rev = revUnderscored; }; ciTests = build.mkCiTests { - inherit path doGetKernelCheck pythonCheckInputs provenanceArgs; + inherit + path + doGetKernelCheck + pythonCheckInputs + provenanceArgs + ; buildSets = applicableBuildSets; rev = revUnderscored; }; From b6e800eec402bd49f3d0e4d4238f5eefc117bc2b Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Fri, 19 Jun 2026 14:48:32 +0530 Subject: [PATCH 05/21] formatting --- kernels/tests/test_dirty_build.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/kernels/tests/test_dirty_build.py b/kernels/tests/test_dirty_build.py index e2a98f80a..c55c7639e 100644 --- a/kernels/tests/test_dirty_build.py +++ b/kernels/tests/test_dirty_build.py @@ -89,9 +89,7 @@ def test_both_dirty_warns_with_both_reasons(): def test_warning_includes_repo_id_when_available(): build_info = {"kernel-builder": _dirty_builder(), "kernel": _clean_kernel()} - messages = _warnings_for( - _metadata(build_info), RepoInfo(repo_id="acme/relu", revision="main") - ) + messages = _warnings_for(_metadata(build_info), RepoInfo(repo_id="acme/relu", revision="main")) assert "acme/relu" in messages[0] From d497ce93263ee3e69c8490f16b7b263bcec1996a Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Mon, 6 Jul 2026 14:22:30 +0530 Subject: [PATCH 06/21] Fix build_info wiring dropped during merge conflict resolution The conflict resolution in 7c1176e left `Metadata::for_backend` without the `build_info` field (E0063) and `write_metadata` building a `BuildInfo` that was never attached to the emitted metadata. Restore both, which unblocks every Rust / maturin / binding / nix CI job. Co-Authored-By: Claude Opus 4.8 (1M context) --- kernel-builder/src/pyproject/common.rs | 5 +++-- kernels-data/src/metadata.rs | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/kernel-builder/src/pyproject/common.rs b/kernel-builder/src/pyproject/common.rs index 92bf61fcd..92bd874ad 100644 --- a/kernel-builder/src/pyproject/common.rs +++ b/kernel-builder/src/pyproject/common.rs @@ -4,7 +4,7 @@ use eyre::Result; use itertools::Itertools; use kernels_data::config::{Backend, Build}; -use kernels_data::metadata::Metadata; +use kernels_data::metadata::{BuildInfo, KernelBuilderInfo, Metadata}; use crate::pyproject::ops_identifier::KernelIdentifier; use crate::pyproject::FileSet; @@ -47,8 +47,9 @@ pub fn write_metadata( for backend in &Backend::all() { let writer = file_set.entry(format!("metadata-{backend}.json")); - let metadata = + let mut metadata = Metadata::for_backend(build, kernel_id.to_string_for_backend(*backend), *backend)?; + metadata.build_info = Some(build_info.clone()); serde_json::to_writer_pretty(writer, &metadata)?; } diff --git a/kernels-data/src/metadata.rs b/kernels-data/src/metadata.rs index 8f513808c..ad31be9c0 100644 --- a/kernels-data/src/metadata.rs +++ b/kernels-data/src/metadata.rs @@ -94,6 +94,7 @@ impl Metadata { backend_type: backend, }, digest: None, + build_info: None, }) } From e52cdc2ea5e7060c22641de8336b0587726e4b47 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Mon, 6 Jul 2026 14:31:09 +0530 Subject: [PATCH 07/21] Scope PR to metadata capture only; defer dirty-build warnings Focus this PR on recording build provenance (kernel + kernel-builder git SHA and dirty flags) in metadata, plus the Python bindings and Nix plumbing to populate it. The three warning surfaces are deferred to follow-up PRs: - load-time warning in `kernels.utils` - upload-time warning + Hub README banner in `kernel-builder` upload - (docs updated to drop the promise of those warnings for now) Also restores Rust serde/is_dirty tests for the metadata types that were dropped during the earlier merge conflict resolution. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/source/kernel-requirements.md | 4 +- kernel-builder/src/upload.rs | 120 ++--------------------------- kernels-data/src/metadata.rs | 57 +++++++++++++- kernels/src/kernels/utils.py | 24 ------ kernels/tests/test_dirty_build.py | 118 ---------------------------- 5 files changed, 62 insertions(+), 261 deletions(-) delete mode 100644 kernels/tests/test_dirty_build.py diff --git a/docs/source/kernel-requirements.md b/docs/source/kernel-requirements.md index 6aaa26110..473dbfcc5 100644 --- a/docs/source/kernel-requirements.md +++ b/docs/source/kernel-requirements.md @@ -92,9 +92,7 @@ metadata. Currently the following top-level keys are supported: uncommitted changes. When either `dirty` flag is set, the kernel was built from uncommitted - sources and cannot be reliably reproduced. `kernels` warns when loading such - a build, `kernel-builder` warns when uploading it, and a warning banner is - added to the Hub repository's README. + sources and cannot be reliably reproduced. > **Note:** For Nix builds, dirtiness follows Nix's flake tree status, which > also counts **untracked** files (including an uncommitted `flake.lock`). diff --git a/kernel-builder/src/upload.rs b/kernel-builder/src/upload.rs index ae67615fd..89c6c59ab 100644 --- a/kernel-builder/src/upload.rs +++ b/kernel-builder/src/upload.rs @@ -150,11 +150,6 @@ fn run_upload_typed(args: UploadArgs) -> Result<()> { build_dir.display() ); - let dirty_build = detect_dirty_build(&variants); - if dirty_build { - eprintln!("{DIRTY_BUILD_WARNING}"); - } - let (repo_id, branch) = get_repo_and_branch(&kernel_dir, args.repo_id, args.branch, &variants)?; let repo_url = match api @@ -212,7 +207,6 @@ fn run_upload_typed(args: UploadArgs) -> Result<()> { collect_readme_commit_ops( &kernel_dir, - dirty_build, operations_by_branch .entry(MAIN_BRANCH.to_owned()) .or_default(), @@ -369,51 +363,16 @@ fn collect_benchmark_commit_ops( } /// Collect README commit operation: upload build/CARD.md as README.md. -fn collect_readme_commit_ops( - kernel_dir: &Path, - dirty_build: bool, - operations: &mut Vec, -) { +fn collect_readme_commit_ops(kernel_dir: &Path, operations: &mut Vec) { let Ok(card_path) = discover_build_file(kernel_dir, "CARD.md") else { return; }; - - let source = if dirty_build { - match fs::read_to_string(&card_path) { - Ok(content) => AddSource::Bytes(prepend_dirty_banner(&content).into_bytes()), - Err(err) => { - eprintln!( - "Warning: cannot read `{}` to add dirty-build banner: {err}", - card_path.display() - ); - AddSource::File(card_path) - } - } - } else { - AddSource::File(card_path) - }; - operations.push(CommitOperation::Add { path_in_repo: "README.md".to_owned(), - source, + source: AddSource::File(card_path), }); } -fn prepend_dirty_banner(content: &str) -> String { - let banner = format!("{DIRTY_BUILD_BANNER}\n\n"); - - // Keep the YAML front matter (delimited by `---`) at the very top. - if let Some(rest) = content.strip_prefix("---\n") { - if let Some(end) = rest.find("\n---\n") { - let split = "---\n".len() + end + "\n---\n".len(); - let (front_matter, body) = content.split_at(split); - return format!("{front_matter}\n{banner}{}", body.trim_start_matches('\n')); - } - } - - format!("{banner}{content}") -} - /// Collect build artifact commit operations: add variant files, delete stale ones. fn collect_build_commit_ops( build_dir: &Path, @@ -497,29 +456,6 @@ fn discover_build_file( ); } -const DIRTY_BUILD_WARNING: &str = "\ -Warning: one or more build variants were built from uncommitted (dirty) \ -sources.\n Such builds are not reproducible. A warning banner will be \ -added to the\n repository README."; - -/// Markdown banner prepended to the Hub README for dirty builds. -const DIRTY_BUILD_BANNER: &str = "> [!WARNING]\n\ - > This kernel was built from uncommitted (dirty) sources and is therefore \ - **not reproducible**. It should not be relied upon for production use."; - -fn detect_dirty_build(variants: &[PathBuf]) -> bool { - variants.iter().any(|variant| { - let metadata_path = variant.join("metadata.json"); - let Ok(file) = File::open(&metadata_path) else { - return false; - }; - match Metadata::from_reader(BufReader::new(file)) { - Ok(metadata) => metadata.build_info.as_ref().is_some_and(|bi| bi.is_dirty()), - Err(_) => false, - } - }) -} - /// Determine the branch name (`v{version}`) from variant metadata. fn detect_branch_from_metadata(variants: &[PathBuf]) -> Result> { let mut versions: HashSet = HashSet::new(); @@ -577,68 +513,22 @@ mod tests { fs::write(kernel_dir.join("build/CARD.md"), "# Readme").unwrap(); let mut operations = vec![]; - collect_readme_commit_ops(kernel_dir, false, &mut operations); + collect_readme_commit_ops(kernel_dir, &mut operations); assert_eq!(operations.len(), 1); match &operations[0] { - CommitOperation::Add { - path_in_repo, - source, - } => { + CommitOperation::Add { path_in_repo, .. } => { assert_eq!(path_in_repo, "README.md"); - assert!(matches!(source, AddSource::File(_))); } _ => panic!("Expected Add operation"), } } - #[test] - fn test_collect_readme_commit_ops_dirty() { - let temp_dir = tempfile::tempdir().unwrap(); - let kernel_dir = temp_dir.path(); - - fs::create_dir_all(kernel_dir.join("build")).unwrap(); - fs::write(kernel_dir.join("build/CARD.md"), "# Readme").unwrap(); - - let mut operations = vec![]; - collect_readme_commit_ops(kernel_dir, true, &mut operations); - - assert_eq!(operations.len(), 1); - match &operations[0] { - CommitOperation::Add { - path_in_repo, - source, - } => { - assert_eq!(path_in_repo, "README.md"); - match source { - AddSource::Bytes(bytes) => { - let content = String::from_utf8(bytes.clone()).unwrap(); - assert!(content.contains("[!WARNING]")); - assert!(content.contains("# Readme")); - } - _ => panic!("Expected Bytes source for dirty build"), - } - } - _ => panic!("Expected Add operation"), - } - } - - #[test] - fn test_prepend_dirty_banner_with_front_matter() { - let content = "---\nlicense: apache-2.0\n---\n# Title\n\nBody."; - let result = prepend_dirty_banner(content); - assert!(result.starts_with("---\nlicense: apache-2.0\n---\n")); - assert!(result.contains("[!WARNING]")); - let banner_pos = result.find("[!WARNING]").unwrap(); - let title_pos = result.find("# Title").unwrap(); - assert!(banner_pos < title_pos); - } - #[test] fn test_collect_readme_commit_ops_no_card() { let temp_dir = tempfile::tempdir().unwrap(); let mut operations = vec![]; - collect_readme_commit_ops(temp_dir.path(), false, &mut operations); + collect_readme_commit_ops(temp_dir.path(), &mut operations); assert!(operations.is_empty()); } diff --git a/kernels-data/src/metadata.rs b/kernels-data/src/metadata.rs index ad31be9c0..fd877b623 100644 --- a/kernels-data/src/metadata.rs +++ b/kernels-data/src/metadata.rs @@ -123,7 +123,21 @@ mod tests { use crate::config::{Backend, Build, Framework, General, KernelName, TorchNoarch, TvmFfi}; - use super::Metadata; + use super::{BuildInfo, GitInfo, KernelBuilderInfo, Metadata}; + + fn sample_build_info(kernel_builder_dirty: bool, kernel_dirty: bool) -> BuildInfo { + BuildInfo { + kernel_builder: Some(KernelBuilderInfo { + version: "0.1.0".to_string(), + sha: Some("a".repeat(40)), + dirty: kernel_builder_dirty, + }), + kernel: Some(GitInfo { + sha: "b".repeat(40), + dirty: kernel_dirty, + }), + } + } fn torch_noarch_build() -> Build { Build { @@ -208,4 +222,45 @@ mod tests { assert_eq!(metadata.backend.backend_type, Backend::Cuda); assert!(metadata.backend.archs.is_none()); } + + #[test] + fn build_info_is_dirty_reflects_either_source() { + assert!(!sample_build_info(false, false).is_dirty()); + assert!(sample_build_info(true, false).is_dirty()); + assert!(sample_build_info(false, true).is_dirty()); + assert!(sample_build_info(true, true).is_dirty()); + } + + #[test] + fn metadata_without_build_info_serializes_without_key() { + let build = torch_noarch_build(); + let metadata = Metadata::for_backend(&build, "test-id".to_string(), Backend::Cuda).unwrap(); + + let json = serde_json::to_string(&metadata).unwrap(); + assert!(!json.contains("build-info")); + + let parsed: Metadata = serde_json::from_str(&json).unwrap(); + assert!(parsed.build_info.is_none()); + } + + #[test] + fn metadata_round_trips_build_info_in_kebab_case() { + let build = torch_noarch_build(); + let mut metadata = + Metadata::for_backend(&build, "test-id".to_string(), Backend::Cuda).unwrap(); + metadata.build_info = Some(sample_build_info(false, true)); + + let json = serde_json::to_string(&metadata).unwrap(); + assert!(json.contains("\"build-info\"")); + assert!(json.contains("\"kernel-builder\"")); + + let parsed: Metadata = serde_json::from_str(&json).unwrap(); + let build_info = parsed.build_info.expect("build-info should round-trip"); + assert!(build_info.is_dirty()); + assert_eq!(build_info.kernel.unwrap().sha, "b".repeat(40)); + + let kernel_builder = build_info.kernel_builder.unwrap(); + assert_eq!(kernel_builder.version, "0.1.0"); + assert!(!kernel_builder.dirty); + } } diff --git a/kernels/src/kernels/utils.py b/kernels/src/kernels/utils.py index 650443f16..c54a59227 100644 --- a/kernels/src/kernels/utils.py +++ b/kernels/src/kernels/utils.py @@ -204,28 +204,6 @@ def _validate_variant_dependencies(variant_path: Path) -> None: validate_dependencies(metadata.name.python_name, metadata.python_depends, _backend()) -def _warn_if_dirty_build(metadata: Metadata, repo_info: "RepoInfo | None") -> None: - build_info = metadata.build_info - if build_info is None or not build_info.dirty: - return - - where = f" of `{repo_info.repo_id}`" if repo_info is not None else "" - reasons = [] - kernel_builder = build_info.kernel_builder - if kernel_builder is not None and kernel_builder.dirty: - reasons.append("the `kernel-builder` had uncommitted changes") - kernel = build_info.kernel - if kernel is not None and kernel.dirty: - reasons.append("the kernel source had uncommitted changes") - - warnings.warn( - f"Kernel `{metadata.id}`{where} was built from a dirty source tree " - f"({' and '.join(reasons)}). Such builds are not reproducible and " - f"should not be relied upon.", - stacklevel=2, - ) - - def _import_from_path(variant_path: Path, repo_info: RepoInfo | None = None) -> ModuleType: if (loaded_kernel := _loaded_kernels.get(variant_path)) is not None: return loaded_kernel.module @@ -233,8 +211,6 @@ def _import_from_path(variant_path: Path, repo_info: RepoInfo | None = None) -> metadata = Metadata.read_from_file(variant_path / "metadata.json") module_name = metadata.name.python_name - _warn_if_dirty_build(metadata, repo_info) - file_path = variant_path / "__init__.py" if not file_path.exists(): file_path = variant_path / module_name / "__init__.py" diff --git a/kernels/tests/test_dirty_build.py b/kernels/tests/test_dirty_build.py deleted file mode 100644 index c55c7639e..000000000 --- a/kernels/tests/test_dirty_build.py +++ /dev/null @@ -1,118 +0,0 @@ -import json -import warnings - -import pytest -from kernels_data import Metadata - -from kernels.utils import RepoInfo, _import_from_path, _loaded_kernels, _warn_if_dirty_build - - -def _metadata(build_info: dict | None) -> Metadata: - data: dict = { - "name": "relu", - "id": "_relu_cuda_abc1234", - "version": 1, - "license": "Apache-2.0", - "python-depends": [], - "backend": {"type": "cuda"}, - } - if build_info is not None: - data["build-info"] = build_info - return Metadata.from_bytes(json.dumps(data).encode("utf-8")) - - -def _clean_builder() -> dict: - return {"version": "0.16.0-dev0", "sha": "a" * 40, "dirty": False} - - -def _dirty_builder() -> dict: - return {"version": "0.16.0-dev0", "sha": "a" * 40, "dirty": True} - - -def _clean_kernel() -> dict: - return {"sha": "b" * 40, "dirty": False} - - -def _dirty_kernel() -> dict: - return {"sha": "b" * 40, "dirty": True} - - -@pytest.fixture -def fresh_registry(): - """Run the test against a clean loaded-kernel registry, restore on teardown.""" - saved = _loaded_kernels.copy() - _loaded_kernels.clear() - yield - _loaded_kernels.clear() - _loaded_kernels.update(saved) - - -def _warnings_for(metadata: Metadata, repo_info: RepoInfo | None = None) -> list[str]: - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - _warn_if_dirty_build(metadata, repo_info) - return [str(w.message) for w in caught] - - -def test_no_build_info_does_not_warn(): - assert _warnings_for(_metadata(None)) == [] - - -def test_clean_build_does_not_warn(): - build_info = {"kernel-builder": _clean_builder(), "kernel": _clean_kernel()} - assert _warnings_for(_metadata(build_info)) == [] - - -def test_dirty_kernel_builder_warns(): - build_info = {"kernel-builder": _dirty_builder(), "kernel": _clean_kernel()} - messages = _warnings_for(_metadata(build_info)) - assert len(messages) == 1 - assert "`kernel-builder` had uncommitted changes" in messages[0] - assert "kernel source had uncommitted changes" not in messages[0] - - -def test_dirty_kernel_source_warns(): - build_info = {"kernel-builder": _clean_builder(), "kernel": _dirty_kernel()} - messages = _warnings_for(_metadata(build_info)) - assert len(messages) == 1 - assert "kernel source had uncommitted changes" in messages[0] - assert "`kernel-builder` had uncommitted changes" not in messages[0] - - -def test_both_dirty_warns_with_both_reasons(): - build_info = {"kernel-builder": _dirty_builder(), "kernel": _dirty_kernel()} - messages = _warnings_for(_metadata(build_info)) - assert len(messages) == 1 - assert "`kernel-builder` had uncommitted changes" in messages[0] - assert "kernel source had uncommitted changes" in messages[0] - - -def test_warning_includes_repo_id_when_available(): - build_info = {"kernel-builder": _dirty_builder(), "kernel": _clean_kernel()} - messages = _warnings_for(_metadata(build_info), RepoInfo(repo_id="acme/relu", revision="main")) - assert "acme/relu" in messages[0] - - -def test_import_from_path_warns_on_dirty_build(tmp_path, fresh_registry): - variant_path = tmp_path / "variant" - (variant_path / "relu").mkdir(parents=True) - (variant_path / "relu" / "__init__.py").write_text("VALUE = 42\n") - - metadata = { - "name": "relu", - "id": "_relu_cuda_abc1234", - "version": 1, - "license": "Apache-2.0", - "python-depends": [], - "backend": {"type": "cuda"}, - "build-info": {"kernel-builder": _dirty_builder(), "kernel": _clean_kernel()}, - } - (variant_path / "metadata.json").write_text(json.dumps(metadata)) - - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - module = _import_from_path(variant_path) - - assert module.VALUE == 42 - dirty_warnings = [w for w in caught if "dirty source tree" in str(w.message)] - assert len(dirty_warnings) == 1 From 56fba60b5efa61e027bc372544d49fbc9e16eb2c Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Mon, 6 Jul 2026 14:43:25 +0530 Subject: [PATCH 08/21] keep op identifier atomic. --- kernel-builder/src/pyproject/common.rs | 17 +--- kernel-builder/src/pyproject/mod.rs | 73 +++++++++-------- .../src/pyproject/ops_identifier.rs | 82 +++---------------- kernel-builder/src/pyproject/torch/mod.rs | 4 +- kernel-builder/src/pyproject/torch/noarch.rs | 4 +- kernel-builder/src/pyproject/tvm_ffi/mod.rs | 4 +- 6 files changed, 65 insertions(+), 119 deletions(-) diff --git a/kernel-builder/src/pyproject/common.rs b/kernel-builder/src/pyproject/common.rs index 92bd874ad..9c6f224cd 100644 --- a/kernel-builder/src/pyproject/common.rs +++ b/kernel-builder/src/pyproject/common.rs @@ -20,7 +20,8 @@ pub fn write_compat_py(file_set: &mut FileSet) -> Result<()> { Ok(()) } -fn kernel_builder_info() -> KernelBuilderInfo { +/// `kernel-builder` provenance baked in at compile time. +pub(crate) fn kernel_builder_info() -> KernelBuilderInfo { KernelBuilderInfo { version: env!("CARGO_PKG_VERSION").to_owned(), sha: option_env!("KERNEL_BUILDER_GIT_SHA").map(str::to_owned), @@ -31,25 +32,15 @@ fn kernel_builder_info() -> KernelBuilderInfo { pub fn write_metadata( build: &Build, kernel_id: &KernelIdentifier, + build_info: Option<&BuildInfo>, file_set: &mut FileSet, ) -> Result<()> { - // Prefer externally-provided `kernel-builder` provenance (e.g. from Nix), - // falling back to the provenance baked in at compile time. - let kernel_builder = kernel_id - .kernel_builder() - .cloned() - .unwrap_or_else(kernel_builder_info); - let build_info = BuildInfo { - kernel_builder: Some(kernel_builder), - kernel: kernel_id.git_info().cloned(), - }; - for backend in &Backend::all() { let writer = file_set.entry(format!("metadata-{backend}.json")); let mut metadata = Metadata::for_backend(build, kernel_id.to_string_for_backend(*backend), *backend)?; - metadata.build_info = Some(build_info.clone()); + metadata.build_info = build_info.cloned(); serde_json::to_writer_pretty(writer, &metadata)?; } diff --git a/kernel-builder/src/pyproject/mod.rs b/kernel-builder/src/pyproject/mod.rs index be33e6bed..d11b5674a 100644 --- a/kernel-builder/src/pyproject/mod.rs +++ b/kernel-builder/src/pyproject/mod.rs @@ -6,7 +6,7 @@ use std::{ use eyre::{bail, Result}; use kernels_data::config::{Build, Framework}; -use kernels_data::metadata::{GitInfo, KernelBuilderInfo}; +use kernels_data::metadata::{BuildInfo, GitInfo, KernelBuilderInfo}; use minijinja::Environment; use crate::{ @@ -24,17 +24,21 @@ mod tvm_ffi; pub use fileset::FileSet; -pub fn create_pyproject_file_set(build: Build, kernel_id: &KernelIdentifier) -> Result { +pub fn create_pyproject_file_set( + build: Build, + kernel_id: &KernelIdentifier, + build_info: Option<&BuildInfo>, +) -> Result { let mut env = Environment::new(); env.set_trim_blocks(true); minijinja_embed::load_templates!(&mut env); let file_set = if matches!(build.framework, Framework::TvmFfi(_)) { - tvm_ffi::write_tvm_ffi_ext(&env, &build, kernel_id)? + tvm_ffi::write_tvm_ffi_ext(&env, &build, kernel_id, build_info)? } else if build.is_noarch() { - torch::write_torch_ext_noarch(&env, &build, kernel_id)? + torch::write_torch_ext_noarch(&env, &build, kernel_id, build_info)? } else { - torch::write_torch_ext(&env, &build, kernel_id)? + torch::write_torch_ext(&env, &build, kernel_id, build_info)? }; Ok(file_set) @@ -54,25 +58,34 @@ pub fn create_pyproject( let kernel_dir = check_or_infer_kernel_dir(kernel_dir)?; let target_dir = check_or_infer_target_dir(&kernel_dir, target_dir)?; let build = parse_build(&kernel_dir)?; - let git_override = kernel_sha.map(|sha| GitInfo { - sha, - dirty: kernel_dirty, - }); - // The version is always that of the running `kernel-builder`; only the - // git provenance is supplied externally (e.g. by Nix). - let kernel_builder_override = kernel_builder_sha.map(|sha| KernelBuilderInfo { - version: env!("CARGO_PKG_VERSION").to_owned(), - sha: Some(sha), - dirty: kernel_builder_dirty, - }); - let kernel_id = KernelIdentifier::new( - &kernel_dir, - build.general.name.python_name(), - unique_id, - git_override, - kernel_builder_override, - ); - let file_set = create_pyproject_file_set(build, &kernel_id)?; + + // Assemble build provenance. Prefer an explicitly provided git provenance + // (e.g. passed by Nix builds, where the source tree has no `.git`); fall + // back to detecting it from the kernel's git repository. + let kernel = kernel_sha + .map(|sha| GitInfo { + sha, + dirty: kernel_dirty, + }) + .or_else(|| ops_identifier::git_info(&kernel_dir)); + // The version is always that of the running `kernel-builder`; only the git + // provenance may be supplied externally (e.g. by Nix). Otherwise fall back + // to the provenance baked in at compile time. + let kernel_builder = match kernel_builder_sha { + Some(sha) => KernelBuilderInfo { + version: env!("CARGO_PKG_VERSION").to_owned(), + sha: Some(sha), + dirty: kernel_builder_dirty, + }, + None => common::kernel_builder_info(), + }; + let build_info = BuildInfo { + kernel_builder: Some(kernel_builder), + kernel, + }; + + let kernel_id = KernelIdentifier::new(&kernel_dir, build.general.name.python_name(), unique_id); + let file_set = create_pyproject_file_set(build, &kernel_id, Some(&build_info))?; file_set.write(&target_dir, force)?; Ok(()) @@ -89,15 +102,9 @@ pub fn clean_pyproject( let target_dir = check_or_infer_target_dir(&kernel_dir, target_dir)?; let build = parse_build(&kernel_dir)?; // Provenance is irrelevant when computing the set of files to clean. - let kernel_id = KernelIdentifier::new( - &kernel_dir, - build.general.name.python_name(), - unique_id, - None, - None, - ); - - let generated_files = create_pyproject_file_set(build, &kernel_id)?.into_names(); + let kernel_id = KernelIdentifier::new(&kernel_dir, build.general.name.python_name(), unique_id); + + let generated_files = create_pyproject_file_set(build, &kernel_id, None)?.into_names(); if generated_files.is_empty() { eprintln!("No generated artifacts found to clean."); diff --git a/kernel-builder/src/pyproject/ops_identifier.rs b/kernel-builder/src/pyproject/ops_identifier.rs index 5c8b7ecf0..8dfe0fd2a 100644 --- a/kernel-builder/src/pyproject/ops_identifier.rs +++ b/kernel-builder/src/pyproject/ops_identifier.rs @@ -3,7 +3,7 @@ use std::path::Path; use eyre::{Result, WrapErr}; use git2::Repository; use kernels_data::config::Backend; -use kernels_data::metadata::{GitInfo, KernelBuilderInfo}; +use kernels_data::metadata::GitInfo; use rand::Rng; pub fn random_identifier() -> String { @@ -43,11 +43,14 @@ pub fn git_info(target_dir: impl AsRef) -> Option { Some(GitInfo { sha, dirty }) } +/// Uniquely identifies a kernel for the purpose of ops-name generation. +/// +/// This type is intentionally small and atomic: it carries only what is +/// needed to name a kernel's ops. Build provenance (git SHA, dirty flags, +/// `kernel-builder` version) is handled separately when writing metadata. pub struct KernelIdentifier { name: String, unique_id: String, - git_info: Option, - kernel_builder: Option, } impl KernelIdentifier { @@ -58,44 +61,19 @@ impl KernelIdentifier { /// source revision. If this identefier is not provided, a Git short /// hash is extracted from the repository of `target_dir`. If that /// fails, a random identifier is generated. - pub fn new( - kernel_dir: impl AsRef, - name: String, - unique_id: Option, - git_override: Option, - kernel_builder: Option, - ) -> Self { - // Prefer an explicitly provided git provenance (e.g. passed by Nix - // builds, where the source tree has no `.git`); fall back to detecting - // it from the kernel's git repository. - let git_info = git_override.or_else(|| git_info(kernel_dir.as_ref())); + pub fn new(kernel_dir: impl AsRef, name: String, unique_id: Option) -> Self { let unique_id = unique_id.unwrap_or_else(|| match git_identifier(kernel_dir.as_ref()) { Ok(rev) => rev, Err(_) => random_identifier(), }); - Self { - name, - unique_id, - git_info, - kernel_builder, - } + Self { name, unique_id } } pub fn name(&self) -> &str { &self.name } - pub fn git_info(&self) -> Option<&GitInfo> { - self.git_info.as_ref() - } - - /// Externally-provided `kernel-builder` provenance, when available. When - /// `None`, the compile-time baked provenance is used instead. - pub fn kernel_builder(&self) -> Option<&KernelBuilderInfo> { - self.kernel_builder.as_ref() - } - /// Create the kernel identifier string for a given backend. pub fn to_string_for_backend(&self, backend: Backend) -> String { format!("_{}_{}_{}", self.name, backend, self.unique_id) @@ -111,53 +89,17 @@ mod tests { use super::*; #[test] - fn explicit_provenance_overrides_are_stored() { + fn kernel_identifier_uses_provided_unique_id() { let tmp = tempfile::tempdir().unwrap(); - let git = GitInfo { - sha: "a".repeat(40), - dirty: true, - }; - let kernel_builder = KernelBuilderInfo { - version: "0.16.0-dev0".to_owned(), - sha: Some("b".repeat(40)), - dirty: false, - }; - - let id = KernelIdentifier::new( - tmp.path(), - "relu".to_owned(), - Some("rev123".to_owned()), - Some(git), - Some(kernel_builder), - ); + let id = KernelIdentifier::new(tmp.path(), "relu".to_owned(), Some("rev123".to_owned())); assert_eq!(id.unique_id(), "rev123"); assert_eq!(id.to_string_for_backend(Backend::Cuda), "_relu_cuda_rev123"); - - let git = id.git_info().expect("kernel git info"); - assert_eq!(git.sha, "a".repeat(40)); - assert!(git.dirty); - - let kb = id.kernel_builder().expect("kernel-builder info"); - assert_eq!(kb.sha.as_deref(), Some(&"b".repeat(40)[..])); - assert!(!kb.dirty); } #[test] - fn no_provenance_in_non_git_dir() { + fn git_info_is_none_in_non_git_dir() { let tmp = tempfile::tempdir().unwrap(); - let id = KernelIdentifier::new( - tmp.path(), - "relu".to_owned(), - Some("x".to_owned()), - None, - None, - ); - - // No overrides supplied and the temp dir is not a git repository, so - // no kernel provenance is recorded. The kernel-builder provenance is - // filled in later from the compile-time default in `write_metadata`. - assert!(id.git_info().is_none()); - assert!(id.kernel_builder().is_none()); + assert!(git_info(tmp.path()).is_none()); } } diff --git a/kernel-builder/src/pyproject/torch/mod.rs b/kernel-builder/src/pyproject/torch/mod.rs index 11dcb1ee4..f57a09186 100644 --- a/kernel-builder/src/pyproject/torch/mod.rs +++ b/kernel-builder/src/pyproject/torch/mod.rs @@ -4,6 +4,7 @@ use std::path::PathBuf; use eyre::{bail, Context, Result}; use itertools::Itertools; use kernels_data::config::{Backend, Build, General, Torch}; +use kernels_data::metadata::BuildInfo; use minijinja::context; use crate::pyproject::common::{ @@ -271,6 +272,7 @@ pub fn write_torch_ext( env: &minijinja::Environment, build: &Build, kernel_id: &KernelIdentifier, + build_info: Option<&BuildInfo>, ) -> Result { let torch_ext = match build.framework.torch() { Some(torch_ext) => torch_ext, @@ -296,7 +298,7 @@ pub fn write_torch_ext( write_torch_registration_macros(&mut file_set)?; - write_metadata(build, kernel_id, &mut file_set)?; + write_metadata(build, kernel_id, build_info, &mut file_set)?; Ok(file_set) } diff --git a/kernel-builder/src/pyproject/torch/noarch.rs b/kernel-builder/src/pyproject/torch/noarch.rs index 1fbe8d523..a7cb70b90 100644 --- a/kernel-builder/src/pyproject/torch/noarch.rs +++ b/kernel-builder/src/pyproject/torch/noarch.rs @@ -4,6 +4,7 @@ use eyre::{Context, Result}; use itertools::Itertools; use kernels_data::config::TorchNoarch; use kernels_data::config::{Backend, Build, General}; +use kernels_data::metadata::BuildInfo; use minijinja::{context, Environment}; use crate::pyproject::common::write_compat_py; @@ -17,6 +18,7 @@ pub fn write_torch_ext_noarch( env: &Environment, build: &Build, kernel_id: &KernelIdentifier, + build_info: Option<&BuildInfo>, ) -> Result { let mut file_set = FileSet::default(); @@ -34,7 +36,7 @@ pub fn write_torch_ext_noarch( &mut file_set, )?; write_setup_py(&mut file_set)?; - write_metadata(build, kernel_id, &mut file_set)?; + write_metadata(build, kernel_id, build_info, &mut file_set)?; Ok(file_set) } diff --git a/kernel-builder/src/pyproject/tvm_ffi/mod.rs b/kernel-builder/src/pyproject/tvm_ffi/mod.rs index 410ceb531..c7a8281da 100644 --- a/kernel-builder/src/pyproject/tvm_ffi/mod.rs +++ b/kernel-builder/src/pyproject/tvm_ffi/mod.rs @@ -3,6 +3,7 @@ use std::io::Write; use eyre::{bail, Context, Result}; use itertools::Itertools; use kernels_data::config::{Backend, Build, General, TvmFfi}; +use kernels_data::metadata::BuildInfo; use minijinja::{context, Environment}; use crate::pyproject::common::{ @@ -42,6 +43,7 @@ pub fn write_tvm_ffi_ext( env: &Environment, build: &Build, kernel_id: &KernelIdentifier, + build_info: Option<&BuildInfo>, ) -> Result { let tvm_ffi_ext = match build.framework.tvm_ffi() { Some(torch_ext) => torch_ext, @@ -65,7 +67,7 @@ pub fn write_tvm_ffi_ext( write_pyproject_toml(env, &build.general, &mut file_set)?; - write_metadata(build, kernel_id, &mut file_set)?; + write_metadata(build, kernel_id, build_info, &mut file_set)?; Ok(file_set) } From 0c2b0ea2dd74dd32541f8e91a492bc982f619465 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Mon, 6 Jul 2026 15:10:52 +0530 Subject: [PATCH 09/21] burn --- docs/source/builder-cli.md | 2 - flake.nix | 24 ++++----- kernel-builder/Cargo.toml | 1 + kernel-builder/build.rs | 54 +++++++++++++-------- kernel-builder/src/main.rs | 15 ------ kernel-builder/src/pyproject/mod.rs | 26 +++------- nix-builder/lib/build-sets.nix | 13 ++++- nix-builder/lib/gen-flake-outputs.nix | 19 ++------ nix-builder/lib/mk-build-set.nix | 12 +++++ nix-builder/pkgs/kernel-builder/default.nix | 13 +++++ 10 files changed, 96 insertions(+), 83 deletions(-) diff --git a/docs/source/builder-cli.md b/docs/source/builder-cli.md index 5a1503284..02a5f7aec 100644 --- a/docs/source/builder-cli.md +++ b/docs/source/builder-cli.md @@ -246,8 +246,6 @@ Generate CMake files for a kernel extension build * `--unique-id ` — This is an optional unique identifier that is suffixed to the kernel name to avoid name collisions. (e.g. Git SHA) * `--kernel-sha ` — Full commit SHA of the kernel source, recorded in the build metadata. When absent, it is detected from the kernel's git repository (used by Nix builds where the source has no `.git`) * `--kernel-dirty` — Mark the kernel source as having uncommitted changes in the build metadata. Only meaningful together with `--kernel-sha` -* `--kernel-builder-sha ` — Full commit SHA of the `kernel-builder` source, recorded in the build metadata. When absent, the SHA baked in at compile time is used (Nix builds pass this since the sandbox has no `.git`) -* `--kernel-builder-dirty` — Mark `kernel-builder` as having uncommitted changes in the build metadata. Only meaningful together with `--kernel-builder-sha` diff --git a/flake.nix b/flake.nix index 00c034799..e2cdd8429 100644 --- a/flake.nix +++ b/flake.nix @@ -27,9 +27,21 @@ rust-overlay, }: let + # Git provenance of `kernel-builder` itself, burned into the binary at + # build time and recorded in the build metadata of the kernels it builds. + # `self` here is the `kernel-builder` flake (as resolved in the consuming + # kernel's lock). + builderRev = self.rev or self.dirtyRev or null; + builderDirty = !(self ? rev); + inherit (import ./nix-builder/lib/build-sets.nix { - inherit nixpkgs rust-overlay; + inherit + nixpkgs + rust-overlay + builderRev + builderDirty + ; }) mkBuildSets partitionBuildSetsBySystem @@ -37,14 +49,6 @@ ; inherit (import ./nix-builder/lib/cache.nix) mkForCache; - # Git provenance of `kernel-builder` itself, recorded in the build - # metadata of kernels it builds. `self` here is the `kernel-builder` - # flake (as resolved in the consuming kernel's lock). It is captured - # here because the `self` argument of `genKernelFlakeOutputs` below - # shadows it with the *kernel's* `self`. - builderRev = self.rev or self.dirtyRev or null; - builderDirty = !(self ? rev); - systems = with flake-utils.lib.system; [ aarch64-darwin aarch64-linux @@ -117,8 +121,6 @@ path rev self - builderRev - builderDirty doGetKernelCheck pythonCheckInputs pythonNativeCheckInputs diff --git a/kernel-builder/Cargo.toml b/kernel-builder/Cargo.toml index 1763a6eb7..663706224 100644 --- a/kernel-builder/Cargo.toml +++ b/kernel-builder/Cargo.toml @@ -37,6 +37,7 @@ reqwest = { version = "0.13", features = ["blocking"] } walkdir = "2" [build-dependencies] +git2 = "0.20" minijinja-embed = "2.5" [dev-dependencies] diff --git a/kernel-builder/build.rs b/kernel-builder/build.rs index 4022e5dff..e81db6604 100644 --- a/kernel-builder/build.rs +++ b/kernel-builder/build.rs @@ -1,23 +1,31 @@ -use std::process::Command; - fn main() { minijinja_embed::embed_templates!("src/pyproject/templates"); emit_git_info(); } +struct GitInfo { + sha: String, + dirty: bool, +} + +/// Burn the `kernel-builder` git provenance into the binary at compile time. +/// +/// The provenance is read from the git repository using `git2` (rather than +/// shelling out to `git`, which is more robust in build sandboxes). When the +/// build happens without a `.git` (e.g. the Nix sandbox), the derivation passes +/// the provenance through the `KERNEL_BUILDER_GIT_SHA` / `KERNEL_BUILDER_GIT_DIRTY` +/// environment variables, which take precedence. fn emit_git_info() { - let sha = env_var("KERNEL_BUILDER_GIT_SHA").or_else(|| git_output(&["rev-parse", "HEAD"])); + let git = git_info(); + + let sha = env_var("KERNEL_BUILDER_GIT_SHA").or_else(|| git.as_ref().map(|g| g.sha.clone())); if let Some(sha) = sha { println!("cargo:rustc-env=KERNEL_BUILDER_GIT_SHA={sha}"); } let dirty = match env_var("KERNEL_BUILDER_GIT_DIRTY") { Some(value) => is_truthy(&value), - // Only consider tracked files; untracked files (e.g. generated build - // artifacts) should not mark the build as dirty. - None => git_output(&["status", "--porcelain", "--untracked-files=no"]) - .map(|out| !out.is_empty()) - .unwrap_or(false), + None => git.map(|g| g.dirty).unwrap_or(false), }; println!( "cargo:rustc-env=KERNEL_BUILDER_GIT_DIRTY={}", @@ -30,6 +38,23 @@ fn emit_git_info() { println!("cargo:rerun-if-changed=../.git/index"); } +/// Detect the current commit SHA and dirty state via `git2`. +fn git_info() -> Option { + let repo = git2::Repository::discover(".").ok()?; + let head = repo.head().ok()?; + let commit = head.peel_to_commit().ok()?; + let sha = commit.id().to_string(); + + let mut status_options = git2::StatusOptions::new(); + // Only consider tracked files; untracked files (e.g. generated build + // artifacts) should not mark the build as dirty. + status_options.include_untracked(false); + status_options.exclude_submodules(true); + let dirty = !repo.statuses(Some(&mut status_options)).ok()?.is_empty(); + + Some(GitInfo { sha, dirty }) +} + fn env_var(name: &str) -> Option { std::env::var(name).ok().filter(|s| !s.is_empty()) } @@ -37,16 +62,3 @@ fn env_var(name: &str) -> Option { fn is_truthy(value: &str) -> bool { value == "1" || value.eq_ignore_ascii_case("true") } - -fn git_output(args: &[&str]) -> Option { - let output = Command::new("git").args(args).output().ok()?; - if !output.status.success() { - return None; - } - let s = String::from_utf8_lossy(&output.stdout).trim().to_string(); - if s.is_empty() { - None - } else { - Some(s) - } -} diff --git a/kernel-builder/src/main.rs b/kernel-builder/src/main.rs index 76bad5350..9c07c5a67 100644 --- a/kernel-builder/src/main.rs +++ b/kernel-builder/src/main.rs @@ -185,17 +185,6 @@ enum Commands { /// metadata. Only meaningful together with `--kernel-sha`. #[arg(long)] kernel_dirty: bool, - - /// Full commit SHA of the `kernel-builder` source, recorded in the - /// build metadata. When absent, the SHA baked in at compile time is - /// used (Nix builds pass this since the sandbox has no `.git`). - #[arg(long)] - kernel_builder_sha: Option, - - /// Mark `kernel-builder` as having uncommitted changes in the build - /// metadata. Only meaningful together with `--kernel-builder-sha`. - #[arg(long)] - kernel_builder_dirty: bool, }, /// Spawn a kernel development shell. @@ -389,8 +378,6 @@ fn main() -> Result<()> { unique_id, kernel_sha, kernel_dirty, - kernel_builder_sha, - kernel_builder_dirty, } => create_pyproject( kernel_dir, target_dir, @@ -398,8 +385,6 @@ fn main() -> Result<()> { unique_id, kernel_sha, kernel_dirty, - kernel_builder_sha, - kernel_builder_dirty, ), Commands::Devshell { kernel_dir, diff --git a/kernel-builder/src/pyproject/mod.rs b/kernel-builder/src/pyproject/mod.rs index d11b5674a..fed9cc7b0 100644 --- a/kernel-builder/src/pyproject/mod.rs +++ b/kernel-builder/src/pyproject/mod.rs @@ -6,7 +6,7 @@ use std::{ use eyre::{bail, Result}; use kernels_data::config::{Build, Framework}; -use kernels_data::metadata::{BuildInfo, GitInfo, KernelBuilderInfo}; +use kernels_data::metadata::{BuildInfo, GitInfo}; use minijinja::Environment; use crate::{ @@ -44,7 +44,6 @@ pub fn create_pyproject_file_set( Ok(file_set) } -#[allow(clippy::too_many_arguments)] pub fn create_pyproject( kernel_dir: Option, target_dir: Option, @@ -52,35 +51,24 @@ pub fn create_pyproject( unique_id: Option, kernel_sha: Option, kernel_dirty: bool, - kernel_builder_sha: Option, - kernel_builder_dirty: bool, ) -> Result<()> { let kernel_dir = check_or_infer_kernel_dir(kernel_dir)?; let target_dir = check_or_infer_target_dir(&kernel_dir, target_dir)?; let build = parse_build(&kernel_dir)?; - // Assemble build provenance. Prefer an explicitly provided git provenance - // (e.g. passed by Nix builds, where the source tree has no `.git`); fall - // back to detecting it from the kernel's git repository. + // Assemble build provenance. Prefer an explicitly provided kernel git + // provenance (e.g. passed by Nix builds, where the source tree has no + // `.git`); fall back to detecting it from the kernel's git repository. The + // `kernel-builder` provenance is always the one baked into this binary at + // compile time. let kernel = kernel_sha .map(|sha| GitInfo { sha, dirty: kernel_dirty, }) .or_else(|| ops_identifier::git_info(&kernel_dir)); - // The version is always that of the running `kernel-builder`; only the git - // provenance may be supplied externally (e.g. by Nix). Otherwise fall back - // to the provenance baked in at compile time. - let kernel_builder = match kernel_builder_sha { - Some(sha) => KernelBuilderInfo { - version: env!("CARGO_PKG_VERSION").to_owned(), - sha: Some(sha), - dirty: kernel_builder_dirty, - }, - None => common::kernel_builder_info(), - }; let build_info = BuildInfo { - kernel_builder: Some(kernel_builder), + kernel_builder: Some(common::kernel_builder_info()), kernel, }; diff --git a/nix-builder/lib/build-sets.nix b/nix-builder/lib/build-sets.nix index 348781357..cbf6c36f4 100644 --- a/nix-builder/lib/build-sets.nix +++ b/nix-builder/lib/build-sets.nix @@ -1,6 +1,10 @@ { nixpkgs, rust-overlay, + + # Git provenance of `kernel-builder` itself, burned into the binary. + builderRev ? null, + builderDirty ? false, }: let @@ -24,7 +28,14 @@ let in builtins.map (buildConfig: buildConfig // { backend = backend buildConfig; }) systemBuildConfigs; - mkBuildSet = import ./mk-build-set.nix { inherit nixpkgs rust-overlay; }; + mkBuildSet = import ./mk-build-set.nix { + inherit + nixpkgs + rust-overlay + builderRev + builderDirty + ; + }; in rec { diff --git a/nix-builder/lib/gen-flake-outputs.nix b/nix-builder/lib/gen-flake-outputs.nix index e710695a6..3ba1a2f2e 100644 --- a/nix-builder/lib/gen-flake-outputs.nix +++ b/nix-builder/lib/gen-flake-outputs.nix @@ -11,12 +11,6 @@ rev ? null, self ? null, - # Git provenance of the `kernel-builder` itself (the flake revision it was - # evaluated from), recorded in the build metadata. `null`/`false` when - # `kernel-builder` is used from a non-git source (e.g. a local `path:`). - builderRev ? null, - builderDirty ? false, - doGetKernelCheck, pythonCheckInputs, pythonNativeCheckInputs, @@ -42,10 +36,10 @@ let revUnderscored = builtins.replaceStrings [ "-" ] [ "_" ] flakeRev; - # Extra `kernel-builder create-pyproject` flags that record the full git - # provenance (commit SHA + dirty state) of both the kernel source and - # `kernel-builder` in the build metadata. The Nix sandbox has no `.git`, so - # this information has to be passed in explicitly. + # Extra `kernel-builder create-pyproject` flags that record the git provenance + # (commit SHA + dirty state) of the kernel source in the build metadata. The + # Nix sandbox has no `.git`, so this information has to be passed in + # explicitly. (`kernel-builder`'s own provenance is burned into its binary.) provenanceArgs = let kernelSha = @@ -58,12 +52,9 @@ let else null; kernelDirty = self != null && !(self ? rev); - builderSha = if builderRev == null then null else lib.removeSuffix "-dirty" builderRev; parts = lib.optional (kernelSha != null) "--kernel-sha ${kernelSha}" - ++ lib.optional (kernelSha != null && kernelDirty) "--kernel-dirty" - ++ lib.optional (builderSha != null) "--kernel-builder-sha ${builderSha}" - ++ lib.optional (builderSha != null && builderDirty) "--kernel-builder-dirty"; + ++ lib.optional (kernelSha != null && kernelDirty) "--kernel-dirty"; in lib.concatStringsSep " " parts; diff --git a/nix-builder/lib/mk-build-set.nix b/nix-builder/lib/mk-build-set.nix index dd8d54254..465e9ceb6 100644 --- a/nix-builder/lib/mk-build-set.nix +++ b/nix-builder/lib/mk-build-set.nix @@ -1,6 +1,11 @@ { nixpkgs, rust-overlay, + + # Git provenance of `kernel-builder` itself, burned into the binary so that + # the kernels it builds record which `kernel-builder` produced them. + builderRev ? null, + builderDirty ? false, }: let @@ -8,6 +13,12 @@ let overlay = import ../overlay.nix; + # Inject the `kernel-builder` git provenance into the package. It cannot be + # detected from `.git` in the build sandbox, so it is passed through here. + builderProvenanceOverlay = _final: prev: { + kernel-builder = prev.kernel-builder.override { inherit builderRev builderDirty; }; + }; + flattenVersion = version: lib.replaceStrings [ "." ] [ "_" ] (lib.versions.pad 2 version); overlayForTorchVersion = torchVersion: sourceBuild: self: super: { @@ -127,6 +138,7 @@ let inherit config system; overlays = [ overlay + builderProvenanceOverlay rust-overlay.overlays.default ] ++ backendOverlay diff --git a/nix-builder/pkgs/kernel-builder/default.nix b/nix-builder/pkgs/kernel-builder/default.nix index 2fb344851..2fbcdcdbd 100644 --- a/nix-builder/pkgs/kernel-builder/default.nix +++ b/nix-builder/pkgs/kernel-builder/default.nix @@ -5,6 +5,13 @@ pkg-config, libgit2, openssl, + + # Git provenance of `kernel-builder` itself. Burned into the binary at build + # time (the build sandbox has no `.git`, so `build.rs` cannot detect it) and + # later recorded in the build metadata of the kernels it builds. `null`/`false` + # when `kernel-builder` is built from a non-git source (e.g. a local `path:`). + builderRev ? null, + builderDirty ? false, }: let @@ -19,6 +26,12 @@ rustPlatform.buildRustPackage { inherit version; pname = "kernel-builder"; + # Consumed by `kernel-builder/build.rs` and baked into the binary. + KERNEL_BUILDER_GIT_SHA = lib.optionalString (builderRev != null) ( + lib.removeSuffix "-dirty" builderRev + ); + KERNEL_BUILDER_GIT_DIRTY = if builderDirty then "1" else "0"; + src = let sourceFiles = From 9b6705c932a49e0f39eff4a1b242154fb8ef249d Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Mon, 6 Jul 2026 15:49:57 +0530 Subject: [PATCH 10/21] use crates --- Cargo.lock | 28 +++++++++- kernel-builder/Cargo.toml | 2 +- kernel-builder/build.rs | 62 +-------------------- kernel-builder/src/main.rs | 7 +++ kernel-builder/src/pyproject/common.rs | 8 ++- nix-builder/pkgs/kernel-builder/default.nix | 12 ++-- 6 files changed, 47 insertions(+), 72 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bd0e44db2..16797e9a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -246,6 +246,15 @@ dependencies = [ "serde", ] +[[package]] +name = "built" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c0e531d93d39c34eef561e929e8a7f86d77a5af08aac4f6d6e39976c51858e9" +dependencies = [ + "git2 0.21.0", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -1026,6 +1035,18 @@ dependencies = [ "url", ] +[[package]] +name = "git2" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e" +dependencies = [ + "bitflags", + "libc", + "libgit2-sys", + "log", +] + [[package]] name = "globset" version = "0.4.18" @@ -1125,12 +1146,13 @@ name = "hf-kernel-builder" version = "0.17.0-dev0" dependencies = [ "base32", + "built", "clap", "clap-markdown", "clap_complete", "dirs", "eyre", - "git2", + "git2 0.20.4", "hf-hub", "indicatif", "itertools 0.13.0", @@ -1693,9 +1715,9 @@ checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libgit2-sys" -version = "0.18.3+1.9.2" +version = "0.18.5+1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9b3acc4b91781bb0b3386669d325163746af5f6e4f73e6d2d630e09a35f3487" +checksum = "005d6ae6eac1912906073e069f7db60b1fa98e052a68227824afe3e3a1c59ca2" dependencies = [ "cc", "libc", diff --git a/kernel-builder/Cargo.toml b/kernel-builder/Cargo.toml index 663706224..59a92fdfe 100644 --- a/kernel-builder/Cargo.toml +++ b/kernel-builder/Cargo.toml @@ -37,7 +37,7 @@ reqwest = { version = "0.13", features = ["blocking"] } walkdir = "2" [build-dependencies] -git2 = "0.20" +built = { version = "0.8", features = ["git2"] } minijinja-embed = "2.5" [dev-dependencies] diff --git a/kernel-builder/build.rs b/kernel-builder/build.rs index e81db6604..6347861a2 100644 --- a/kernel-builder/build.rs +++ b/kernel-builder/build.rs @@ -1,64 +1,4 @@ fn main() { minijinja_embed::embed_templates!("src/pyproject/templates"); - emit_git_info(); -} - -struct GitInfo { - sha: String, - dirty: bool, -} - -/// Burn the `kernel-builder` git provenance into the binary at compile time. -/// -/// The provenance is read from the git repository using `git2` (rather than -/// shelling out to `git`, which is more robust in build sandboxes). When the -/// build happens without a `.git` (e.g. the Nix sandbox), the derivation passes -/// the provenance through the `KERNEL_BUILDER_GIT_SHA` / `KERNEL_BUILDER_GIT_DIRTY` -/// environment variables, which take precedence. -fn emit_git_info() { - let git = git_info(); - - let sha = env_var("KERNEL_BUILDER_GIT_SHA").or_else(|| git.as_ref().map(|g| g.sha.clone())); - if let Some(sha) = sha { - println!("cargo:rustc-env=KERNEL_BUILDER_GIT_SHA={sha}"); - } - - let dirty = match env_var("KERNEL_BUILDER_GIT_DIRTY") { - Some(value) => is_truthy(&value), - None => git.map(|g| g.dirty).unwrap_or(false), - }; - println!( - "cargo:rustc-env=KERNEL_BUILDER_GIT_DIRTY={}", - if dirty { "1" } else { "0" } - ); - - println!("cargo:rerun-if-env-changed=KERNEL_BUILDER_GIT_SHA"); - println!("cargo:rerun-if-env-changed=KERNEL_BUILDER_GIT_DIRTY"); - println!("cargo:rerun-if-changed=../.git/HEAD"); - println!("cargo:rerun-if-changed=../.git/index"); -} - -/// Detect the current commit SHA and dirty state via `git2`. -fn git_info() -> Option { - let repo = git2::Repository::discover(".").ok()?; - let head = repo.head().ok()?; - let commit = head.peel_to_commit().ok()?; - let sha = commit.id().to_string(); - - let mut status_options = git2::StatusOptions::new(); - // Only consider tracked files; untracked files (e.g. generated build - // artifacts) should not mark the build as dirty. - status_options.include_untracked(false); - status_options.exclude_submodules(true); - let dirty = !repo.statuses(Some(&mut status_options)).ok()?.is_empty(); - - Some(GitInfo { sha, dirty }) -} - -fn env_var(name: &str) -> Option { - std::env::var(name).ok().filter(|s| !s.is_empty()) -} - -fn is_truthy(value: &str) -> bool { - value == "1" || value.eq_ignore_ascii_case("true") + built::write_built_file().expect("Failed to acquire build-time information"); } diff --git a/kernel-builder/src/main.rs b/kernel-builder/src/main.rs index 9c07c5a67..9f544214e 100644 --- a/kernel-builder/src/main.rs +++ b/kernel-builder/src/main.rs @@ -50,6 +50,13 @@ use validate_builds::check_builds; use crate::util::parse_and_validate_compat; +/// Build-time metadata gathered by the [`built`](https://crates.io/crates/built) +/// crate (see `build.rs`), including the `kernel-builder` git provenance that is +/// recorded in the build metadata of the kernels it builds. +mod built_info { + include!(concat!(env!("OUT_DIR"), "/built.rs")); +} + #[derive(Args, Debug)] struct NixArgs { /// Maximum number of parallel Nix build jobs. diff --git a/kernel-builder/src/pyproject/common.rs b/kernel-builder/src/pyproject/common.rs index 9c6f224cd..e89abc24c 100644 --- a/kernel-builder/src/pyproject/common.rs +++ b/kernel-builder/src/pyproject/common.rs @@ -21,11 +21,15 @@ pub fn write_compat_py(file_set: &mut FileSet) -> Result<()> { } /// `kernel-builder` provenance baked in at compile time. +/// +/// The git provenance is gathered by the `built` crate (see `build.rs`). In +/// build sandboxes without a `.git` (e.g. Nix), the derivation supplies it via +/// `built`'s `BUILT_OVERRIDE_hf_kernel_builder_GIT_*` environment variables. pub(crate) fn kernel_builder_info() -> KernelBuilderInfo { KernelBuilderInfo { version: env!("CARGO_PKG_VERSION").to_owned(), - sha: option_env!("KERNEL_BUILDER_GIT_SHA").map(str::to_owned), - dirty: matches!(option_env!("KERNEL_BUILDER_GIT_DIRTY"), Some("1")), + sha: crate::built_info::GIT_COMMIT_HASH.map(str::to_owned), + dirty: crate::built_info::GIT_DIRTY.unwrap_or(false), } } diff --git a/nix-builder/pkgs/kernel-builder/default.nix b/nix-builder/pkgs/kernel-builder/default.nix index 2fbcdcdbd..a40c8b5be 100644 --- a/nix-builder/pkgs/kernel-builder/default.nix +++ b/nix-builder/pkgs/kernel-builder/default.nix @@ -26,11 +26,13 @@ rustPlatform.buildRustPackage { inherit version; pname = "kernel-builder"; - # Consumed by `kernel-builder/build.rs` and baked into the binary. - KERNEL_BUILDER_GIT_SHA = lib.optionalString (builderRev != null) ( - lib.removeSuffix "-dirty" builderRev - ); - KERNEL_BUILDER_GIT_DIRTY = if builderDirty then "1" else "0"; + # The build sandbox has no `.git`, so the `built` crate cannot detect the + # `kernel-builder` git provenance. Supply it through `built`'s override + # environment variables (`hf_kernel_builder` is the package name with hyphens + # replaced by underscores), which `build.rs` bakes into the binary. + "BUILT_OVERRIDE_hf_kernel_builder_GIT_COMMIT_HASH" = + if builderRev == null then "BUILT_OVERRIDE_NONE" else lib.removeSuffix "-dirty" builderRev; + "BUILT_OVERRIDE_hf_kernel_builder_GIT_DIRTY" = if builderDirty then "true" else "false"; src = let From 2662a65e9a3a2c3c432a4f59c280b515320c23d3 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Mon, 6 Jul 2026 15:55:00 +0530 Subject: [PATCH 11/21] GitHash. --- kernel-builder/src/pyproject/mod.rs | 6 +++--- .../src/pyproject/ops_identifier.rs | 10 +++++----- kernels-data/bindings/python/kernels_data.pyi | 6 +++--- kernels-data/bindings/python/src/lib.rs | 20 +++++++++---------- kernels-data/src/metadata.rs | 8 ++++---- 5 files changed, 25 insertions(+), 25 deletions(-) diff --git a/kernel-builder/src/pyproject/mod.rs b/kernel-builder/src/pyproject/mod.rs index fed9cc7b0..411a7aade 100644 --- a/kernel-builder/src/pyproject/mod.rs +++ b/kernel-builder/src/pyproject/mod.rs @@ -6,7 +6,7 @@ use std::{ use eyre::{bail, Result}; use kernels_data::config::{Build, Framework}; -use kernels_data::metadata::{BuildInfo, GitInfo}; +use kernels_data::metadata::{BuildInfo, GitHash}; use minijinja::Environment; use crate::{ @@ -62,11 +62,11 @@ pub fn create_pyproject( // `kernel-builder` provenance is always the one baked into this binary at // compile time. let kernel = kernel_sha - .map(|sha| GitInfo { + .map(|sha| GitHash { sha, dirty: kernel_dirty, }) - .or_else(|| ops_identifier::git_info(&kernel_dir)); + .or_else(|| ops_identifier::git_hash(&kernel_dir)); let build_info = BuildInfo { kernel_builder: Some(common::kernel_builder_info()), kernel, diff --git a/kernel-builder/src/pyproject/ops_identifier.rs b/kernel-builder/src/pyproject/ops_identifier.rs index 8dfe0fd2a..dae82b68a 100644 --- a/kernel-builder/src/pyproject/ops_identifier.rs +++ b/kernel-builder/src/pyproject/ops_identifier.rs @@ -3,7 +3,7 @@ use std::path::Path; use eyre::{Result, WrapErr}; use git2::Repository; use kernels_data::config::Backend; -use kernels_data::metadata::GitInfo; +use kernels_data::metadata::GitHash; use rand::Rng; pub fn random_identifier() -> String { @@ -29,7 +29,7 @@ pub fn git_identifier(target_dir: impl AsRef) -> Result { Ok(if dirty { format!("{rev}_dirty") } else { rev }) } -pub fn git_info(target_dir: impl AsRef) -> Option { +pub fn git_hash(target_dir: impl AsRef) -> Option { let repo = Repository::discover(target_dir.as_ref()).ok()?; let head = repo.head().ok()?; let commit = head.peel_to_commit().ok()?; @@ -40,7 +40,7 @@ pub fn git_info(target_dir: impl AsRef) -> Option { status_options.exclude_submodules(true); let dirty = !repo.statuses(Some(&mut status_options)).ok()?.is_empty(); - Some(GitInfo { sha, dirty }) + Some(GitHash { sha, dirty }) } /// Uniquely identifies a kernel for the purpose of ops-name generation. @@ -98,8 +98,8 @@ mod tests { } #[test] - fn git_info_is_none_in_non_git_dir() { + fn git_hash_is_none_in_non_git_dir() { let tmp = tempfile::tempdir().unwrap(); - assert!(git_info(tmp.path()).is_none()); + assert!(git_hash(tmp.path()).is_none()); } } diff --git a/kernels-data/bindings/python/kernels_data.pyi b/kernels-data/bindings/python/kernels_data.pyi index 9feb36eb0..d7930a718 100644 --- a/kernels-data/bindings/python/kernels_data.pyi +++ b/kernels-data/bindings/python/kernels_data.pyi @@ -9,7 +9,7 @@ __all__ = [ "BackendInfo", "BuildInfo", "DigestAlgorithm", - "GitInfo", + "GitHash", "KernelBuilderInfo", "KernelName", "Metadata", @@ -67,7 +67,7 @@ class BackendInfo: def __repr__(self) -> str: ... @final -class GitInfo: +class GitHash: """Git provenance (commit SHA and dirty state) of a source tree.""" @property @@ -113,7 +113,7 @@ class BuildInfo: ... @property - def kernel(self) -> Optional[GitInfo]: + def kernel(self) -> Optional[GitHash]: """Git provenance of the kernel source that was built.""" ... diff --git a/kernels-data/bindings/python/src/lib.rs b/kernels-data/bindings/python/src/lib.rs index a34518c9b..728a1fb1e 100644 --- a/kernels-data/bindings/python/src/lib.rs +++ b/kernels-data/bindings/python/src/lib.rs @@ -6,7 +6,7 @@ use std::str::FromStr; use kernels_data::config::{Backend, KernelName}; use kernels_data::digest::{Digest, DigestAlgorithm, DigestViolation}; -use kernels_data::metadata::{BackendInfo, BuildInfo, GitInfo, KernelBuilderInfo, Metadata}; +use kernels_data::metadata::{BackendInfo, BuildInfo, GitHash, KernelBuilderInfo, Metadata}; use kernels_data::version::Version; use pyo3::Bound as PyBound; use pyo3::exceptions::{PyException, PyOSError, PyRuntimeError, PyValueError}; @@ -188,15 +188,15 @@ impl PyBackendInfo { } } -#[pyclass(name = "GitInfo", frozen)] +#[pyclass(name = "GitHash", frozen)] #[derive(Clone, Debug)] -struct PyGitInfo { +struct PyGitHash { sha: String, dirty: bool, } -impl From for PyGitInfo { - fn from(g: GitInfo) -> Self { +impl From for PyGitHash { + fn from(g: GitHash) -> Self { Self { sha: g.sha, dirty: g.dirty, @@ -205,7 +205,7 @@ impl From for PyGitInfo { } #[pymethods] -impl PyGitInfo { +impl PyGitHash { #[getter] fn sha(&self) -> &str { &self.sha @@ -217,7 +217,7 @@ impl PyGitInfo { } fn __repr__(&self) -> String { - format!("GitInfo(sha={:?}, dirty={})", self.sha, self.dirty) + format!("GitHash(sha={:?}, dirty={})", self.sha, self.dirty) } } @@ -268,7 +268,7 @@ impl PyKernelBuilderInfo { #[derive(Clone, Debug)] struct PyBuildInfo { kernel_builder: Option, - kernel: Option, + kernel: Option, } impl From for PyBuildInfo { @@ -288,7 +288,7 @@ impl PyBuildInfo { } #[getter] - fn kernel(&self) -> Option { + fn kernel(&self) -> Option { self.kernel.clone() } @@ -665,7 +665,7 @@ fn kernels_data_py(m: &PyBound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; - m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/kernels-data/src/metadata.rs b/kernels-data/src/metadata.rs index fd877b623..ef9e89790 100644 --- a/kernels-data/src/metadata.rs +++ b/kernels-data/src/metadata.rs @@ -17,7 +17,7 @@ pub struct BackendInfo { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "kebab-case")] -pub struct GitInfo { +pub struct GitHash { pub sha: String, pub dirty: bool, } @@ -38,7 +38,7 @@ pub struct BuildInfo { #[serde(skip_serializing_if = "Option::is_none")] pub kernel_builder: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub kernel: Option, + pub kernel: Option, } impl BuildInfo { @@ -123,7 +123,7 @@ mod tests { use crate::config::{Backend, Build, Framework, General, KernelName, TorchNoarch, TvmFfi}; - use super::{BuildInfo, GitInfo, KernelBuilderInfo, Metadata}; + use super::{BuildInfo, GitHash, KernelBuilderInfo, Metadata}; fn sample_build_info(kernel_builder_dirty: bool, kernel_dirty: bool) -> BuildInfo { BuildInfo { @@ -132,7 +132,7 @@ mod tests { sha: Some("a".repeat(40)), dirty: kernel_builder_dirty, }), - kernel: Some(GitInfo { + kernel: Some(GitHash { sha: "b".repeat(40), dirty: kernel_dirty, }), From a58e16fe215127c3eef873c0944657480d9d3681 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Mon, 6 Jul 2026 16:03:35 +0530 Subject: [PATCH 12/21] GitHash into KernelBuilderInfo. --- kernel-builder/src/pyproject/common.rs | 9 ++- kernels-data/bindings/python/kernels_data.pyi | 9 +-- kernels-data/bindings/python/src/lib.rs | 28 ++++----- kernels-data/src/metadata.rs | 63 ++++++++++++++++--- 4 files changed, 78 insertions(+), 31 deletions(-) diff --git a/kernel-builder/src/pyproject/common.rs b/kernel-builder/src/pyproject/common.rs index e89abc24c..b2042436f 100644 --- a/kernel-builder/src/pyproject/common.rs +++ b/kernel-builder/src/pyproject/common.rs @@ -4,7 +4,7 @@ use eyre::Result; use itertools::Itertools; use kernels_data::config::{Backend, Build}; -use kernels_data::metadata::{BuildInfo, KernelBuilderInfo, Metadata}; +use kernels_data::metadata::{BuildInfo, GitHash, KernelBuilderInfo, Metadata}; use crate::pyproject::ops_identifier::KernelIdentifier; use crate::pyproject::FileSet; @@ -26,10 +26,13 @@ pub fn write_compat_py(file_set: &mut FileSet) -> Result<()> { /// build sandboxes without a `.git` (e.g. Nix), the derivation supplies it via /// `built`'s `BUILT_OVERRIDE_hf_kernel_builder_GIT_*` environment variables. pub(crate) fn kernel_builder_info() -> KernelBuilderInfo { + let git = crate::built_info::GIT_COMMIT_HASH.map(|sha| GitHash { + sha: sha.to_owned(), + dirty: crate::built_info::GIT_DIRTY.unwrap_or(false), + }); KernelBuilderInfo { version: env!("CARGO_PKG_VERSION").to_owned(), - sha: crate::built_info::GIT_COMMIT_HASH.map(str::to_owned), - dirty: crate::built_info::GIT_DIRTY.unwrap_or(false), + git, } } diff --git a/kernels-data/bindings/python/kernels_data.pyi b/kernels-data/bindings/python/kernels_data.pyi index d7930a718..391a677c9 100644 --- a/kernels-data/bindings/python/kernels_data.pyi +++ b/kernels-data/bindings/python/kernels_data.pyi @@ -92,13 +92,8 @@ class KernelBuilderInfo: ... @property - def sha(self) -> Optional[str]: - """Commit SHA of the `kernel-builder` source, when known.""" - ... - - @property - def dirty(self) -> bool: - """Whether `kernel-builder` was built from a dirty source tree.""" + def git(self) -> Optional[GitHash]: + """Commit SHA and dirty state of the `kernel-builder` source, when known.""" ... def __repr__(self) -> str: ... diff --git a/kernels-data/bindings/python/src/lib.rs b/kernels-data/bindings/python/src/lib.rs index 728a1fb1e..bebf0954a 100644 --- a/kernels-data/bindings/python/src/lib.rs +++ b/kernels-data/bindings/python/src/lib.rs @@ -225,16 +225,14 @@ impl PyGitHash { #[derive(Clone, Debug)] struct PyKernelBuilderInfo { version: String, - sha: Option, - dirty: bool, + git: Option, } impl From for PyKernelBuilderInfo { fn from(kb: KernelBuilderInfo) -> Self { Self { version: kb.version, - sha: kb.sha, - dirty: kb.dirty, + git: kb.git.map(Into::into), } } } @@ -246,20 +244,19 @@ impl PyKernelBuilderInfo { &self.version } + /// Commit SHA + dirty state of the `kernel-builder` source, when known. #[getter] - fn sha(&self) -> Option<&str> { - self.sha.as_deref() - } - - #[getter] - fn dirty(&self) -> bool { - self.dirty + fn git(&self) -> Option { + self.git.clone() } fn __repr__(&self) -> String { format!( - "KernelBuilderInfo(version={:?}, sha={:?}, dirty={})", - self.version, self.sha, self.dirty + "KernelBuilderInfo(version={:?}, git={})", + self.version, + self.git + .as_ref() + .map_or("None".to_string(), |g| g.__repr__()) ) } } @@ -295,7 +292,10 @@ impl PyBuildInfo { /// Whether either the `kernel-builder` or the kernel source was dirty. #[getter] fn dirty(&self) -> bool { - self.kernel_builder.as_ref().is_some_and(|kb| kb.dirty) + self.kernel_builder + .as_ref() + .and_then(|kb| kb.git.as_ref()) + .is_some_and(|g| g.dirty) || self.kernel.as_ref().is_some_and(|k| k.dirty) } diff --git a/kernels-data/src/metadata.rs b/kernels-data/src/metadata.rs index ef9e89790..cb2a19e9d 100644 --- a/kernels-data/src/metadata.rs +++ b/kernels-data/src/metadata.rs @@ -27,9 +27,9 @@ pub struct GitHash { #[serde(rename_all = "kebab-case")] pub struct KernelBuilderInfo { pub version: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub sha: Option, - pub dirty: bool, + /// Commit SHA + dirty state of the `kernel-builder` source, when known. + #[serde(flatten)] + pub git: Option, } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -44,7 +44,10 @@ pub struct BuildInfo { impl BuildInfo { /// Whether either the `kernel-builder` or the kernel source was dirty. pub fn is_dirty(&self) -> bool { - self.kernel_builder.as_ref().is_some_and(|kb| kb.dirty) + self.kernel_builder + .as_ref() + .and_then(|kb| kb.git.as_ref()) + .is_some_and(|g| g.dirty) || self.kernel.as_ref().is_some_and(|k| k.dirty) } } @@ -129,8 +132,10 @@ mod tests { BuildInfo { kernel_builder: Some(KernelBuilderInfo { version: "0.1.0".to_string(), - sha: Some("a".repeat(40)), - dirty: kernel_builder_dirty, + git: Some(GitHash { + sha: "a".repeat(40), + dirty: kernel_builder_dirty, + }), }), kernel: Some(GitHash { sha: "b".repeat(40), @@ -261,6 +266,50 @@ mod tests { let kernel_builder = build_info.kernel_builder.unwrap(); assert_eq!(kernel_builder.version, "0.1.0"); - assert!(!kernel_builder.dirty); + // The embedded `GitHash` is flattened into the `kernel-builder` object. + let git = kernel_builder + .git + .expect("kernel-builder git should round-trip"); + assert_eq!(git.sha, "a".repeat(40)); + assert!(!git.dirty); + } + + #[test] + fn kernel_builder_info_flattens_git_hash() { + let kernel_builder = KernelBuilderInfo { + version: "0.1.0".to_string(), + git: Some(GitHash { + sha: "c".repeat(40), + dirty: true, + }), + }; + + // The `GitHash` fields are flattened into the same object as `version`. + let value: serde_json::Value = serde_json::to_value(&kernel_builder).unwrap(); + assert_eq!(value["version"], "0.1.0"); + assert_eq!(value["sha"], "c".repeat(40)); + assert_eq!(value["dirty"], true); + assert!(value.get("git").is_none()); + + let parsed: KernelBuilderInfo = serde_json::from_value(value).unwrap(); + let git = parsed.git.expect("git should round-trip"); + assert_eq!(git.sha, "c".repeat(40)); + assert!(git.dirty); + } + + #[test] + fn kernel_builder_info_without_git_round_trips_to_none() { + // When the `kernel-builder` git provenance is unknown, only `version` + // is serialized and it must deserialize back to `git: None`. + let kernel_builder = KernelBuilderInfo { + version: "0.1.0".to_string(), + git: None, + }; + + let json = serde_json::to_string(&kernel_builder).unwrap(); + assert_eq!(json, r#"{"version":"0.1.0"}"#); + + let parsed: KernelBuilderInfo = serde_json::from_str(&json).unwrap(); + assert!(parsed.git.is_none()); } } From 2eb585e8799f8310fef814a8ef87bf1f3d6c7150 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Mon, 6 Jul 2026 16:12:48 +0530 Subject: [PATCH 13/21] KernelBuilderVersion --- kernel-builder/src/pyproject/common.rs | 6 +++--- kernel-builder/src/pyproject/mod.rs | 2 +- kernels-data/bindings/python/kernels_data.pyi | 6 +++--- kernels-data/bindings/python/src/lib.rs | 20 +++++++++---------- kernels-data/src/metadata.rs | 20 +++++++++---------- 5 files changed, 27 insertions(+), 27 deletions(-) diff --git a/kernel-builder/src/pyproject/common.rs b/kernel-builder/src/pyproject/common.rs index b2042436f..dc5c1285b 100644 --- a/kernel-builder/src/pyproject/common.rs +++ b/kernel-builder/src/pyproject/common.rs @@ -4,7 +4,7 @@ use eyre::Result; use itertools::Itertools; use kernels_data::config::{Backend, Build}; -use kernels_data::metadata::{BuildInfo, GitHash, KernelBuilderInfo, Metadata}; +use kernels_data::metadata::{BuildInfo, GitHash, KernelBuilderVersion, Metadata}; use crate::pyproject::ops_identifier::KernelIdentifier; use crate::pyproject::FileSet; @@ -25,12 +25,12 @@ pub fn write_compat_py(file_set: &mut FileSet) -> Result<()> { /// The git provenance is gathered by the `built` crate (see `build.rs`). In /// build sandboxes without a `.git` (e.g. Nix), the derivation supplies it via /// `built`'s `BUILT_OVERRIDE_hf_kernel_builder_GIT_*` environment variables. -pub(crate) fn kernel_builder_info() -> KernelBuilderInfo { +pub(crate) fn kernel_builder_version() -> KernelBuilderVersion { let git = crate::built_info::GIT_COMMIT_HASH.map(|sha| GitHash { sha: sha.to_owned(), dirty: crate::built_info::GIT_DIRTY.unwrap_or(false), }); - KernelBuilderInfo { + KernelBuilderVersion { version: env!("CARGO_PKG_VERSION").to_owned(), git, } diff --git a/kernel-builder/src/pyproject/mod.rs b/kernel-builder/src/pyproject/mod.rs index 411a7aade..07f08bf58 100644 --- a/kernel-builder/src/pyproject/mod.rs +++ b/kernel-builder/src/pyproject/mod.rs @@ -68,7 +68,7 @@ pub fn create_pyproject( }) .or_else(|| ops_identifier::git_hash(&kernel_dir)); let build_info = BuildInfo { - kernel_builder: Some(common::kernel_builder_info()), + kernel_builder: Some(common::kernel_builder_version()), kernel, }; diff --git a/kernels-data/bindings/python/kernels_data.pyi b/kernels-data/bindings/python/kernels_data.pyi index 391a677c9..6f7e4833a 100644 --- a/kernels-data/bindings/python/kernels_data.pyi +++ b/kernels-data/bindings/python/kernels_data.pyi @@ -10,7 +10,7 @@ __all__ = [ "BuildInfo", "DigestAlgorithm", "GitHash", - "KernelBuilderInfo", + "KernelBuilderVersion", "KernelName", "Metadata", "Digest", @@ -83,7 +83,7 @@ class GitHash: def __repr__(self) -> str: ... @final -class KernelBuilderInfo: +class KernelBuilderVersion: """Provenance of the `kernel-builder` that produced a build.""" @property @@ -103,7 +103,7 @@ class BuildInfo: """Build provenance: git state of the `kernel-builder` and kernel source.""" @property - def kernel_builder(self) -> Optional[KernelBuilderInfo]: + def kernel_builder(self) -> Optional[KernelBuilderVersion]: """Provenance of the `kernel-builder` that produced the build.""" ... diff --git a/kernels-data/bindings/python/src/lib.rs b/kernels-data/bindings/python/src/lib.rs index bebf0954a..537cc3d88 100644 --- a/kernels-data/bindings/python/src/lib.rs +++ b/kernels-data/bindings/python/src/lib.rs @@ -6,7 +6,7 @@ use std::str::FromStr; use kernels_data::config::{Backend, KernelName}; use kernels_data::digest::{Digest, DigestAlgorithm, DigestViolation}; -use kernels_data::metadata::{BackendInfo, BuildInfo, GitHash, KernelBuilderInfo, Metadata}; +use kernels_data::metadata::{BackendInfo, BuildInfo, GitHash, KernelBuilderVersion, Metadata}; use kernels_data::version::Version; use pyo3::Bound as PyBound; use pyo3::exceptions::{PyException, PyOSError, PyRuntimeError, PyValueError}; @@ -221,15 +221,15 @@ impl PyGitHash { } } -#[pyclass(name = "KernelBuilderInfo", frozen)] +#[pyclass(name = "KernelBuilderVersion", frozen)] #[derive(Clone, Debug)] -struct PyKernelBuilderInfo { +struct PyKernelBuilderVersion { version: String, git: Option, } -impl From for PyKernelBuilderInfo { - fn from(kb: KernelBuilderInfo) -> Self { +impl From for PyKernelBuilderVersion { + fn from(kb: KernelBuilderVersion) -> Self { Self { version: kb.version, git: kb.git.map(Into::into), @@ -238,7 +238,7 @@ impl From for PyKernelBuilderInfo { } #[pymethods] -impl PyKernelBuilderInfo { +impl PyKernelBuilderVersion { #[getter] fn version(&self) -> &str { &self.version @@ -252,7 +252,7 @@ impl PyKernelBuilderInfo { fn __repr__(&self) -> String { format!( - "KernelBuilderInfo(version={:?}, git={})", + "KernelBuilderVersion(version={:?}, git={})", self.version, self.git .as_ref() @@ -264,7 +264,7 @@ impl PyKernelBuilderInfo { #[pyclass(name = "BuildInfo", frozen)] #[derive(Clone, Debug)] struct PyBuildInfo { - kernel_builder: Option, + kernel_builder: Option, kernel: Option, } @@ -280,7 +280,7 @@ impl From for PyBuildInfo { #[pymethods] impl PyBuildInfo { #[getter] - fn kernel_builder(&self) -> Option { + fn kernel_builder(&self) -> Option { self.kernel_builder.clone() } @@ -666,7 +666,7 @@ fn kernels_data_py(m: &PyBound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; - m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/kernels-data/src/metadata.rs b/kernels-data/src/metadata.rs index cb2a19e9d..b76f12d9a 100644 --- a/kernels-data/src/metadata.rs +++ b/kernels-data/src/metadata.rs @@ -25,7 +25,7 @@ pub struct GitHash { /// Provenance of the `kernel-builder` that produced a build. #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "kebab-case")] -pub struct KernelBuilderInfo { +pub struct KernelBuilderVersion { pub version: String, /// Commit SHA + dirty state of the `kernel-builder` source, when known. #[serde(flatten)] @@ -36,7 +36,7 @@ pub struct KernelBuilderInfo { #[serde(rename_all = "kebab-case")] pub struct BuildInfo { #[serde(skip_serializing_if = "Option::is_none")] - pub kernel_builder: Option, + pub kernel_builder: Option, #[serde(skip_serializing_if = "Option::is_none")] pub kernel: Option, } @@ -126,11 +126,11 @@ mod tests { use crate::config::{Backend, Build, Framework, General, KernelName, TorchNoarch, TvmFfi}; - use super::{BuildInfo, GitHash, KernelBuilderInfo, Metadata}; + use super::{BuildInfo, GitHash, KernelBuilderVersion, Metadata}; fn sample_build_info(kernel_builder_dirty: bool, kernel_dirty: bool) -> BuildInfo { BuildInfo { - kernel_builder: Some(KernelBuilderInfo { + kernel_builder: Some(KernelBuilderVersion { version: "0.1.0".to_string(), git: Some(GitHash { sha: "a".repeat(40), @@ -275,8 +275,8 @@ mod tests { } #[test] - fn kernel_builder_info_flattens_git_hash() { - let kernel_builder = KernelBuilderInfo { + fn kernel_builder_version_flattens_git_hash() { + let kernel_builder = KernelBuilderVersion { version: "0.1.0".to_string(), git: Some(GitHash { sha: "c".repeat(40), @@ -291,17 +291,17 @@ mod tests { assert_eq!(value["dirty"], true); assert!(value.get("git").is_none()); - let parsed: KernelBuilderInfo = serde_json::from_value(value).unwrap(); + let parsed: KernelBuilderVersion = serde_json::from_value(value).unwrap(); let git = parsed.git.expect("git should round-trip"); assert_eq!(git.sha, "c".repeat(40)); assert!(git.dirty); } #[test] - fn kernel_builder_info_without_git_round_trips_to_none() { + fn kernel_builder_version_without_git_round_trips_to_none() { // When the `kernel-builder` git provenance is unknown, only `version` // is serialized and it must deserialize back to `git: None`. - let kernel_builder = KernelBuilderInfo { + let kernel_builder = KernelBuilderVersion { version: "0.1.0".to_string(), git: None, }; @@ -309,7 +309,7 @@ mod tests { let json = serde_json::to_string(&kernel_builder).unwrap(); assert_eq!(json, r#"{"version":"0.1.0"}"#); - let parsed: KernelBuilderInfo = serde_json::from_str(&json).unwrap(); + let parsed: KernelBuilderVersion = serde_json::from_str(&json).unwrap(); assert!(parsed.git.is_none()); } } From 83b1c10c2a35bf7406482a3656a67342aa2b4766 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Mon, 6 Jul 2026 16:28:45 +0530 Subject: [PATCH 14/21] use provenance in place of buildinfo. --- docs/source/kernel-requirements.md | 2 +- kernel-builder/src/pyproject/common.rs | 6 +-- kernel-builder/src/pyproject/mod.rs | 14 +++--- kernel-builder/src/pyproject/torch/mod.rs | 6 +-- kernel-builder/src/pyproject/torch/noarch.rs | 6 +-- kernel-builder/src/pyproject/tvm_ffi/mod.rs | 6 +-- kernels-data/bindings/python/kernels_data.pyi | 6 +-- kernels-data/bindings/python/src/lib.rs | 28 +++++------ kernels-data/src/metadata.rs | 46 ++++++++++--------- 9 files changed, 61 insertions(+), 59 deletions(-) diff --git a/docs/source/kernel-requirements.md b/docs/source/kernel-requirements.md index 473dbfcc5..dc14e4a26 100644 --- a/docs/source/kernel-requirements.md +++ b/docs/source/kernel-requirements.md @@ -81,7 +81,7 @@ metadata. Currently the following top-level keys are supported: - `digest` (`Digest`, required): hash digest of the kernel files. - `python-depends` (`list[str]`, optional): list of Python dependencies from a curated set of Python dependencies. -- `build-info` (`dict`, optional): provenance of the build, used to flag +- `provenance` (`dict`, optional): provenance of the build, used to flag non-reproducible (dirty) builds. It contains two optional sub-objects: - `kernel-builder`: the `kernel-builder` that produced the build, with its `version` (`str`), the `sha` (`str`) of the `kernel-builder` source it was diff --git a/kernel-builder/src/pyproject/common.rs b/kernel-builder/src/pyproject/common.rs index dc5c1285b..87ea2fdf5 100644 --- a/kernel-builder/src/pyproject/common.rs +++ b/kernel-builder/src/pyproject/common.rs @@ -4,7 +4,7 @@ use eyre::Result; use itertools::Itertools; use kernels_data::config::{Backend, Build}; -use kernels_data::metadata::{BuildInfo, GitHash, KernelBuilderVersion, Metadata}; +use kernels_data::metadata::{GitHash, KernelBuilderVersion, Metadata, Provenance}; use crate::pyproject::ops_identifier::KernelIdentifier; use crate::pyproject::FileSet; @@ -39,7 +39,7 @@ pub(crate) fn kernel_builder_version() -> KernelBuilderVersion { pub fn write_metadata( build: &Build, kernel_id: &KernelIdentifier, - build_info: Option<&BuildInfo>, + provenance: Option<&Provenance>, file_set: &mut FileSet, ) -> Result<()> { for backend in &Backend::all() { @@ -47,7 +47,7 @@ pub fn write_metadata( let mut metadata = Metadata::for_backend(build, kernel_id.to_string_for_backend(*backend), *backend)?; - metadata.build_info = build_info.cloned(); + metadata.provenance = provenance.cloned(); serde_json::to_writer_pretty(writer, &metadata)?; } diff --git a/kernel-builder/src/pyproject/mod.rs b/kernel-builder/src/pyproject/mod.rs index 07f08bf58..6d7b83243 100644 --- a/kernel-builder/src/pyproject/mod.rs +++ b/kernel-builder/src/pyproject/mod.rs @@ -6,7 +6,7 @@ use std::{ use eyre::{bail, Result}; use kernels_data::config::{Build, Framework}; -use kernels_data::metadata::{BuildInfo, GitHash}; +use kernels_data::metadata::{GitHash, Provenance}; use minijinja::Environment; use crate::{ @@ -27,18 +27,18 @@ pub use fileset::FileSet; pub fn create_pyproject_file_set( build: Build, kernel_id: &KernelIdentifier, - build_info: Option<&BuildInfo>, + provenance: Option<&Provenance>, ) -> Result { let mut env = Environment::new(); env.set_trim_blocks(true); minijinja_embed::load_templates!(&mut env); let file_set = if matches!(build.framework, Framework::TvmFfi(_)) { - tvm_ffi::write_tvm_ffi_ext(&env, &build, kernel_id, build_info)? + tvm_ffi::write_tvm_ffi_ext(&env, &build, kernel_id, provenance)? } else if build.is_noarch() { - torch::write_torch_ext_noarch(&env, &build, kernel_id, build_info)? + torch::write_torch_ext_noarch(&env, &build, kernel_id, provenance)? } else { - torch::write_torch_ext(&env, &build, kernel_id, build_info)? + torch::write_torch_ext(&env, &build, kernel_id, provenance)? }; Ok(file_set) @@ -67,13 +67,13 @@ pub fn create_pyproject( dirty: kernel_dirty, }) .or_else(|| ops_identifier::git_hash(&kernel_dir)); - let build_info = BuildInfo { + let provenance = Provenance { kernel_builder: Some(common::kernel_builder_version()), kernel, }; let kernel_id = KernelIdentifier::new(&kernel_dir, build.general.name.python_name(), unique_id); - let file_set = create_pyproject_file_set(build, &kernel_id, Some(&build_info))?; + let file_set = create_pyproject_file_set(build, &kernel_id, Some(&provenance))?; file_set.write(&target_dir, force)?; Ok(()) diff --git a/kernel-builder/src/pyproject/torch/mod.rs b/kernel-builder/src/pyproject/torch/mod.rs index f57a09186..08f5d7ca4 100644 --- a/kernel-builder/src/pyproject/torch/mod.rs +++ b/kernel-builder/src/pyproject/torch/mod.rs @@ -4,7 +4,7 @@ use std::path::PathBuf; use eyre::{bail, Context, Result}; use itertools::Itertools; use kernels_data::config::{Backend, Build, General, Torch}; -use kernels_data::metadata::BuildInfo; +use kernels_data::metadata::Provenance; use minijinja::context; use crate::pyproject::common::{ @@ -272,7 +272,7 @@ pub fn write_torch_ext( env: &minijinja::Environment, build: &Build, kernel_id: &KernelIdentifier, - build_info: Option<&BuildInfo>, + provenance: Option<&Provenance>, ) -> Result { let torch_ext = match build.framework.torch() { Some(torch_ext) => torch_ext, @@ -298,7 +298,7 @@ pub fn write_torch_ext( write_torch_registration_macros(&mut file_set)?; - write_metadata(build, kernel_id, build_info, &mut file_set)?; + write_metadata(build, kernel_id, provenance, &mut file_set)?; Ok(file_set) } diff --git a/kernel-builder/src/pyproject/torch/noarch.rs b/kernel-builder/src/pyproject/torch/noarch.rs index a7cb70b90..5cbe86f60 100644 --- a/kernel-builder/src/pyproject/torch/noarch.rs +++ b/kernel-builder/src/pyproject/torch/noarch.rs @@ -4,7 +4,7 @@ use eyre::{Context, Result}; use itertools::Itertools; use kernels_data::config::TorchNoarch; use kernels_data::config::{Backend, Build, General}; -use kernels_data::metadata::BuildInfo; +use kernels_data::metadata::Provenance; use minijinja::{context, Environment}; use crate::pyproject::common::write_compat_py; @@ -18,7 +18,7 @@ pub fn write_torch_ext_noarch( env: &Environment, build: &Build, kernel_id: &KernelIdentifier, - build_info: Option<&BuildInfo>, + provenance: Option<&Provenance>, ) -> Result { let mut file_set = FileSet::default(); @@ -36,7 +36,7 @@ pub fn write_torch_ext_noarch( &mut file_set, )?; write_setup_py(&mut file_set)?; - write_metadata(build, kernel_id, build_info, &mut file_set)?; + write_metadata(build, kernel_id, provenance, &mut file_set)?; Ok(file_set) } diff --git a/kernel-builder/src/pyproject/tvm_ffi/mod.rs b/kernel-builder/src/pyproject/tvm_ffi/mod.rs index c7a8281da..e08565a27 100644 --- a/kernel-builder/src/pyproject/tvm_ffi/mod.rs +++ b/kernel-builder/src/pyproject/tvm_ffi/mod.rs @@ -3,7 +3,7 @@ use std::io::Write; use eyre::{bail, Context, Result}; use itertools::Itertools; use kernels_data::config::{Backend, Build, General, TvmFfi}; -use kernels_data::metadata::BuildInfo; +use kernels_data::metadata::Provenance; use minijinja::{context, Environment}; use crate::pyproject::common::{ @@ -43,7 +43,7 @@ pub fn write_tvm_ffi_ext( env: &Environment, build: &Build, kernel_id: &KernelIdentifier, - build_info: Option<&BuildInfo>, + provenance: Option<&Provenance>, ) -> Result { let tvm_ffi_ext = match build.framework.tvm_ffi() { Some(torch_ext) => torch_ext, @@ -67,7 +67,7 @@ pub fn write_tvm_ffi_ext( write_pyproject_toml(env, &build.general, &mut file_set)?; - write_metadata(build, kernel_id, build_info, &mut file_set)?; + write_metadata(build, kernel_id, provenance, &mut file_set)?; Ok(file_set) } diff --git a/kernels-data/bindings/python/kernels_data.pyi b/kernels-data/bindings/python/kernels_data.pyi index 6f7e4833a..f8c6ff989 100644 --- a/kernels-data/bindings/python/kernels_data.pyi +++ b/kernels-data/bindings/python/kernels_data.pyi @@ -7,7 +7,7 @@ from typing import Optional, final __all__ = [ "Backend", "BackendInfo", - "BuildInfo", + "Provenance", "DigestAlgorithm", "GitHash", "KernelBuilderVersion", @@ -99,7 +99,7 @@ class KernelBuilderVersion: def __repr__(self) -> str: ... @final -class BuildInfo: +class Provenance: """Build provenance: git state of the `kernel-builder` and kernel source.""" @property @@ -320,5 +320,5 @@ class Metadata: @property def digest(self) -> Optional[Digest]: ... @property - def build_info(self) -> Optional[BuildInfo]: ... + def provenance(self) -> Optional[Provenance]: ... def __repr__(self) -> str: ... diff --git a/kernels-data/bindings/python/src/lib.rs b/kernels-data/bindings/python/src/lib.rs index 537cc3d88..26b2335cc 100644 --- a/kernels-data/bindings/python/src/lib.rs +++ b/kernels-data/bindings/python/src/lib.rs @@ -6,7 +6,7 @@ use std::str::FromStr; use kernels_data::config::{Backend, KernelName}; use kernels_data::digest::{Digest, DigestAlgorithm, DigestViolation}; -use kernels_data::metadata::{BackendInfo, BuildInfo, GitHash, KernelBuilderVersion, Metadata}; +use kernels_data::metadata::{BackendInfo, GitHash, KernelBuilderVersion, Metadata, Provenance}; use kernels_data::version::Version; use pyo3::Bound as PyBound; use pyo3::exceptions::{PyException, PyOSError, PyRuntimeError, PyValueError}; @@ -261,15 +261,15 @@ impl PyKernelBuilderVersion { } } -#[pyclass(name = "BuildInfo", frozen)] +#[pyclass(name = "Provenance", frozen)] #[derive(Clone, Debug)] -struct PyBuildInfo { +struct PyProvenance { kernel_builder: Option, kernel: Option, } -impl From for PyBuildInfo { - fn from(b: BuildInfo) -> Self { +impl From for PyProvenance { + fn from(b: Provenance) -> Self { Self { kernel_builder: b.kernel_builder.map(Into::into), kernel: b.kernel.map(Into::into), @@ -278,7 +278,7 @@ impl From for PyBuildInfo { } #[pymethods] -impl PyBuildInfo { +impl PyProvenance { #[getter] fn kernel_builder(&self) -> Option { self.kernel_builder.clone() @@ -301,7 +301,7 @@ impl PyBuildInfo { fn __repr__(&self) -> String { format!( - "BuildInfo(kernel_builder={}, kernel={})", + "Provenance(kernel_builder={}, kernel={})", self.kernel_builder .as_ref() .map_or("None".to_string(), |kb| kb.__repr__()), @@ -325,7 +325,7 @@ struct PyMetadata { python_depends: Vec, backend: PyBackendInfo, digest: Option, - build_info: Option, + provenance: Option, } impl From for PyMetadata { @@ -340,7 +340,7 @@ impl From for PyMetadata { python_depends: m.python_depends, backend: m.backend.into(), digest: m.digest.map(Into::into), - build_info: m.build_info.map(Into::into), + provenance: m.provenance.map(Into::into), } } } @@ -420,13 +420,13 @@ impl PyMetadata { } #[getter] - fn build_info(&self) -> Option { - self.build_info.clone() + fn provenance(&self) -> Option { + self.provenance.clone() } fn __repr__(&self) -> String { format!( - "Metadata(id={}, name={:?}, version={:?}, license={:?}, upstream={:?}, source={:?}, python_depends={:?}, backend={}, digest={}, build_info={})", + "Metadata(id={}, name={:?}, version={:?}, license={:?}, upstream={:?}, source={:?}, python_depends={:?}, backend={}, digest={}, provenance={})", self.id, self.name, self.version, @@ -438,7 +438,7 @@ impl PyMetadata { self.digest .as_ref() .map_or("None".to_string(), |sd| sd.__repr__()), - self.build_info + self.provenance .as_ref() .map_or("None".to_string(), |bi| bi.__repr__()) ) @@ -664,7 +664,7 @@ impl PyDigest { fn kernels_data_py(m: &PyBound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; - m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/kernels-data/src/metadata.rs b/kernels-data/src/metadata.rs index b76f12d9a..1709468e6 100644 --- a/kernels-data/src/metadata.rs +++ b/kernels-data/src/metadata.rs @@ -32,16 +32,18 @@ pub struct KernelBuilderVersion { pub git: Option, } +/// Provenance of a kernel build: the git state of the `kernel-builder` and of +/// the kernel source it was built from. #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "kebab-case")] -pub struct BuildInfo { +pub struct Provenance { #[serde(skip_serializing_if = "Option::is_none")] pub kernel_builder: Option, #[serde(skip_serializing_if = "Option::is_none")] pub kernel: Option, } -impl BuildInfo { +impl Provenance { /// Whether either the `kernel-builder` or the kernel source was dirty. pub fn is_dirty(&self) -> bool { self.kernel_builder @@ -69,7 +71,7 @@ pub struct Metadata { #[serde(skip_serializing_if = "Option::is_none")] pub digest: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub build_info: Option, + pub provenance: Option, } impl Metadata { @@ -97,7 +99,7 @@ impl Metadata { backend_type: backend, }, digest: None, - build_info: None, + provenance: None, }) } @@ -126,10 +128,10 @@ mod tests { use crate::config::{Backend, Build, Framework, General, KernelName, TorchNoarch, TvmFfi}; - use super::{BuildInfo, GitHash, KernelBuilderVersion, Metadata}; + use super::{GitHash, KernelBuilderVersion, Metadata, Provenance}; - fn sample_build_info(kernel_builder_dirty: bool, kernel_dirty: bool) -> BuildInfo { - BuildInfo { + fn sample_provenance(kernel_builder_dirty: bool, kernel_dirty: bool) -> Provenance { + Provenance { kernel_builder: Some(KernelBuilderVersion { version: "0.1.0".to_string(), git: Some(GitHash { @@ -229,42 +231,42 @@ mod tests { } #[test] - fn build_info_is_dirty_reflects_either_source() { - assert!(!sample_build_info(false, false).is_dirty()); - assert!(sample_build_info(true, false).is_dirty()); - assert!(sample_build_info(false, true).is_dirty()); - assert!(sample_build_info(true, true).is_dirty()); + fn provenance_is_dirty_reflects_either_source() { + assert!(!sample_provenance(false, false).is_dirty()); + assert!(sample_provenance(true, false).is_dirty()); + assert!(sample_provenance(false, true).is_dirty()); + assert!(sample_provenance(true, true).is_dirty()); } #[test] - fn metadata_without_build_info_serializes_without_key() { + fn metadata_without_provenance_serializes_without_key() { let build = torch_noarch_build(); let metadata = Metadata::for_backend(&build, "test-id".to_string(), Backend::Cuda).unwrap(); let json = serde_json::to_string(&metadata).unwrap(); - assert!(!json.contains("build-info")); + assert!(!json.contains("provenance")); let parsed: Metadata = serde_json::from_str(&json).unwrap(); - assert!(parsed.build_info.is_none()); + assert!(parsed.provenance.is_none()); } #[test] - fn metadata_round_trips_build_info_in_kebab_case() { + fn metadata_round_trips_provenance_in_kebab_case() { let build = torch_noarch_build(); let mut metadata = Metadata::for_backend(&build, "test-id".to_string(), Backend::Cuda).unwrap(); - metadata.build_info = Some(sample_build_info(false, true)); + metadata.provenance = Some(sample_provenance(false, true)); let json = serde_json::to_string(&metadata).unwrap(); - assert!(json.contains("\"build-info\"")); + assert!(json.contains("\"provenance\"")); assert!(json.contains("\"kernel-builder\"")); let parsed: Metadata = serde_json::from_str(&json).unwrap(); - let build_info = parsed.build_info.expect("build-info should round-trip"); - assert!(build_info.is_dirty()); - assert_eq!(build_info.kernel.unwrap().sha, "b".repeat(40)); + let provenance = parsed.provenance.expect("provenance should round-trip"); + assert!(provenance.is_dirty()); + assert_eq!(provenance.kernel.unwrap().sha, "b".repeat(40)); - let kernel_builder = build_info.kernel_builder.unwrap(); + let kernel_builder = provenance.kernel_builder.unwrap(); assert_eq!(kernel_builder.version, "0.1.0"); // The embedded `GitHash` is flattened into the `kernel-builder` object. let git = kernel_builder From d40ad4b865cb8738eddcc3450fbd045cafc9c2ae Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Mon, 6 Jul 2026 16:37:31 +0530 Subject: [PATCH 15/21] make provenance args atomic. --- nix-builder/lib/build.nix | 20 +++++------ .../lib/extension/provenance-flags.nix | 10 ++++++ nix-builder/lib/extension/torch/arch.nix | 10 +++--- nix-builder/lib/extension/torch/no-arch.nix | 9 ++--- nix-builder/lib/extension/tvm-ffi/arch.nix | 10 +++--- nix-builder/lib/gen-flake-outputs.nix | 35 ++++++++++--------- 6 files changed, 56 insertions(+), 38 deletions(-) create mode 100644 nix-builder/lib/extension/provenance-flags.nix diff --git a/nix-builder/lib/build.nix b/nix-builder/lib/build.nix index a8d839935..2a5f2fcc0 100644 --- a/nix-builder/lib/build.nix +++ b/nix-builder/lib/build.nix @@ -113,7 +113,7 @@ rec { rev, doGetKernelCheck, stripRPath ? false, - provenanceArgs ? "", + kernelProvenance ? null, }: let inherit (lib) fileset; @@ -152,7 +152,7 @@ rec { doGetKernelCheck pythonDeps backendPythonDeps - provenanceArgs + kernelProvenance ; kernelName = kernelConfig.name; } @@ -168,7 +168,7 @@ rec { rev pythonDeps backendPythonDeps - provenanceArgs + kernelProvenance ; kernelName = kernelConfig.name; @@ -186,7 +186,7 @@ rec { rev pythonDeps backendPythonDeps - provenanceArgs + kernelProvenance ; torchStableAbiVersion = kernelConfig.torchStableAbiVersionForBackend buildConfig.backend; @@ -203,7 +203,7 @@ rec { doGetKernelCheck, bundleOnly, buildSets, - provenanceArgs ? "", + kernelProvenance ? null, }: let kernelConfig = readKernelConfig path; @@ -217,7 +217,7 @@ rec { kernelConfig rev doGetKernelCheck - provenanceArgs + kernelProvenance ; stripRPath = true; }; @@ -233,7 +233,7 @@ rec { rev, doGetKernelCheck, buildSets, - provenanceArgs ? "", + kernelProvenance ? null, }: let extensions = mkDistTorchExtensions { @@ -242,7 +242,7 @@ rec { path rev doGetKernelCheck - provenanceArgs + kernelProvenance ; bundleOnly = true; }; @@ -347,7 +347,7 @@ rec { buildSets, doGetKernelCheck, pythonCheckInputs, - provenanceArgs ? "", + kernelProvenance ? null, }: let kernelConfig = readKernelConfig path; @@ -364,7 +364,7 @@ rec { kernelConfig rev doGetKernelCheck - provenanceArgs + kernelProvenance ; }; testPython = diff --git a/nix-builder/lib/extension/provenance-flags.nix b/nix-builder/lib/extension/provenance-flags.nix new file mode 100644 index 000000000..7ed49cc58 --- /dev/null +++ b/nix-builder/lib/extension/provenance-flags.nix @@ -0,0 +1,10 @@ +# Assemble `kernel-builder create-pyproject` flags from the kernel's atomic git +# provenance (`{ sha; dirty; }`, or `null` when unknown). The Nix sandbox has no +# `.git`, so the kernel's commit SHA and dirty state are passed in explicitly. +{ + lib, + kernelProvenance ? null, +}: +lib.optionalString (kernelProvenance != null) ( + "--kernel-sha ${kernelProvenance.sha}" + lib.optionalString kernelProvenance.dirty " --kernel-dirty" +) diff --git a/nix-builder/lib/extension/torch/arch.nix b/nix-builder/lib/extension/torch/arch.nix index d16df70f2..a22cf9ef4 100644 --- a/nix-builder/lib/extension/torch/arch.nix +++ b/nix-builder/lib/extension/torch/arch.nix @@ -71,9 +71,9 @@ # Revision to bake into the ops name. rev, - # Extra `create-pyproject` flags recording git provenance (commit SHA and - # dirty state) of the kernel source and `kernel-builder`. - provenanceArgs ? "", + # Git provenance (`{ sha; dirty; }`, or `null`) of the kernel source, recorded + # in the build metadata. + kernelProvenance ? null, src, }: @@ -115,6 +115,8 @@ let metalSupport = buildConfig.metal or false; + provenanceFlags = import ../provenance-flags.nix { inherit lib kernelProvenance; }; + in stdenv.mkDerivation (prevAttrs: { @@ -136,7 +138,7 @@ stdenv.mkDerivation (prevAttrs: { mkdir -p $out cp -r --no-preserve=mode ${src}/* $out/ ${pkgs.kernel-builder}/bin/kernel-builder create-pyproject \ - --unique-id ${rev} ${provenanceArgs} $out + --unique-id ${rev} ${provenanceFlags} $out ''; preConfigure = diff --git a/nix-builder/lib/extension/torch/no-arch.nix b/nix-builder/lib/extension/torch/no-arch.nix index c3264734a..203d727b2 100644 --- a/nix-builder/lib/extension/torch/no-arch.nix +++ b/nix-builder/lib/extension/torch/no-arch.nix @@ -42,9 +42,9 @@ backendPythonDeps, - # Extra `create-pyproject` flags recording git provenance (commit SHA and - # dirty state) of the kernel source and `kernel-builder`. - provenanceArgs ? "", + # Git provenance (`{ sha; dirty; }`, or `null`) of the kernel source, recorded + # in the build metadata. + kernelProvenance ? null, }: # Extra validation - the environment should correspind to the build config. @@ -72,6 +72,7 @@ let ++ [ torch ]; moduleName = builtins.replaceStrings [ "-" ] [ "_" ] kernelName; metalSupport = buildConfig.metal or false; + provenanceFlags = import ../provenance-flags.nix { inherit lib kernelProvenance; }; in stdenv.mkDerivation (prevAttrs: { @@ -83,7 +84,7 @@ stdenv.mkDerivation (prevAttrs: { mkdir -p $out cp -r --no-preserve=mode ${src}/* $out/ ${pkgs.kernel-builder}/bin/kernel-builder create-pyproject \ - --unique-id ${rev} ${provenanceArgs} $out + --unique-id ${rev} ${provenanceFlags} $out ''; framework = "torch"; diff --git a/nix-builder/lib/extension/tvm-ffi/arch.nix b/nix-builder/lib/extension/tvm-ffi/arch.nix index 6c32a238e..2ab7293de 100644 --- a/nix-builder/lib/extension/tvm-ffi/arch.nix +++ b/nix-builder/lib/extension/tvm-ffi/arch.nix @@ -68,9 +68,9 @@ # Revision to bake into the ops name. rev, - # Extra `create-pyproject` flags recording git provenance (commit SHA and - # dirty state) of the kernel source and `kernel-builder`. - provenanceArgs ? "", + # Git provenance (`{ sha; dirty; }`, or `null`) of the kernel source, recorded + # in the build metadata. + kernelProvenance ? null, src, }: @@ -115,6 +115,8 @@ let metalSupport = buildConfig.metal or false; + provenanceFlags = import ../provenance-flags.nix { inherit lib kernelProvenance; }; + in stdenv.mkDerivation (prevAttrs: { @@ -133,7 +135,7 @@ stdenv.mkDerivation (prevAttrs: { # Generate build files. postPatch = '' kernel-builder create-pyproject \ - --unique-id ${rev} ${provenanceArgs} . + --unique-id ${rev} ${provenanceFlags} . ''; preConfigure = diff --git a/nix-builder/lib/gen-flake-outputs.nix b/nix-builder/lib/gen-flake-outputs.nix index 3ba1a2f2e..58f963513 100644 --- a/nix-builder/lib/gen-flake-outputs.nix +++ b/nix-builder/lib/gen-flake-outputs.nix @@ -36,11 +36,12 @@ let revUnderscored = builtins.replaceStrings [ "-" ] [ "_" ] flakeRev; - # Extra `kernel-builder create-pyproject` flags that record the git provenance - # (commit SHA + dirty state) of the kernel source in the build metadata. The - # Nix sandbox has no `.git`, so this information has to be passed in - # explicitly. (`kernel-builder`'s own provenance is burned into its binary.) - provenanceArgs = + # Git provenance (commit SHA + dirty state) of the kernel source, recorded in + # the build metadata. The Nix sandbox has no `.git`, so it has to be passed in + # explicitly; the value is atomic data (`{ sha; dirty; }`, or `null` when + # unknown), not pre-formatted `create-pyproject` flags. (`kernel-builder`'s own + # provenance is burned into its binary.) + kernelProvenance = let kernelSha = if self == null then @@ -51,12 +52,14 @@ let lib.removeSuffix "-dirty" self.dirtyRev else null; - kernelDirty = self != null && !(self ? rev); - parts = - lib.optional (kernelSha != null) "--kernel-sha ${kernelSha}" - ++ lib.optional (kernelSha != null && kernelDirty) "--kernel-dirty"; in - lib.concatStringsSep " " parts; + if kernelSha == null then + null + else + { + sha = kernelSha; + dirty = self != null && !(self ? rev); + }; applicableBuildSets = build.applicableBuildSets { inherit path buildSets; }; @@ -175,7 +178,7 @@ in packages = let bundle = build.mkExtensionBundle { - inherit path doGetKernelCheck provenanceArgs; + inherit path doGetKernelCheck kernelProvenance; buildSets = applicableBuildSets; rev = revUnderscored; }; @@ -184,7 +187,7 @@ in path doGetKernelCheck pythonCheckInputs - provenanceArgs + kernelProvenance ; buildSets = applicableBuildSets; rev = revUnderscored; @@ -202,7 +205,7 @@ in builtins.map (backend: { name = backend; value = build.mkExtensionBundle { - inherit path doGetKernelCheck provenanceArgs; + inherit path doGetKernelCheck kernelProvenance; buildSets = builtins.filter ( set: buildConfigBackend set.buildConfig == backend ) applicableBuildSets; @@ -268,7 +271,7 @@ in ++ (headOrEmpty (setsWithFramework "xpu")); in build.mkExtensionBundle { - inherit path doGetKernelCheck provenanceArgs; + inherit path doGetKernelCheck kernelProvenance; buildSets = onePerFramework; rev = revUnderscored; }; @@ -281,7 +284,7 @@ in builtins.map (backend: { name = backend; value = build.mkExtensionBundle { - inherit path doGetKernelCheck provenanceArgs; + inherit path doGetKernelCheck kernelProvenance; # It is too costly to build all variants in CI, so we just build one per framework. buildSets = headOrEmpty ( builtins.filter (set: set.buildConfig.framework == backend) buildSetsSorted @@ -304,7 +307,7 @@ in }; redistributable = build.mkDistTorchExtensions { - inherit path doGetKernelCheck provenanceArgs; + inherit path doGetKernelCheck kernelProvenance; bundleOnly = false; rev = revUnderscored; buildSets = applicableBuildSets; From f5ab62935f2abe04d0381a5eb7822536668dc5b8 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Mon, 6 Jul 2026 16:59:52 +0530 Subject: [PATCH 16/21] pay attention to defaults. --- kernel-builder/src/pyproject/mod.rs | 2 +- kernels-data/bindings/python/kernels_data.pyi | 4 ++-- kernels-data/bindings/python/src/lib.rs | 15 +++++---------- kernels-data/src/metadata.rs | 18 +++++++++--------- nix-builder/lib/build.nix | 12 ++++++++---- nix-builder/lib/extension/provenance-flags.nix | 2 +- nix-builder/lib/extension/torch/arch.nix | 2 +- nix-builder/lib/extension/torch/no-arch.nix | 2 +- nix-builder/lib/extension/tvm-ffi/arch.nix | 2 +- 9 files changed, 29 insertions(+), 30 deletions(-) diff --git a/kernel-builder/src/pyproject/mod.rs b/kernel-builder/src/pyproject/mod.rs index 6d7b83243..1c7d5376c 100644 --- a/kernel-builder/src/pyproject/mod.rs +++ b/kernel-builder/src/pyproject/mod.rs @@ -68,7 +68,7 @@ pub fn create_pyproject( }) .or_else(|| ops_identifier::git_hash(&kernel_dir)); let provenance = Provenance { - kernel_builder: Some(common::kernel_builder_version()), + kernel_builder: common::kernel_builder_version(), kernel, }; diff --git a/kernels-data/bindings/python/kernels_data.pyi b/kernels-data/bindings/python/kernels_data.pyi index f8c6ff989..20e29619f 100644 --- a/kernels-data/bindings/python/kernels_data.pyi +++ b/kernels-data/bindings/python/kernels_data.pyi @@ -103,8 +103,8 @@ class Provenance: """Build provenance: git state of the `kernel-builder` and kernel source.""" @property - def kernel_builder(self) -> Optional[KernelBuilderVersion]: - """Provenance of the `kernel-builder` that produced the build.""" + def kernel_builder(self) -> KernelBuilderVersion: + """The `kernel-builder` that produced the build (always known).""" ... @property diff --git a/kernels-data/bindings/python/src/lib.rs b/kernels-data/bindings/python/src/lib.rs index 26b2335cc..9aa29f06d 100644 --- a/kernels-data/bindings/python/src/lib.rs +++ b/kernels-data/bindings/python/src/lib.rs @@ -264,14 +264,14 @@ impl PyKernelBuilderVersion { #[pyclass(name = "Provenance", frozen)] #[derive(Clone, Debug)] struct PyProvenance { - kernel_builder: Option, + kernel_builder: PyKernelBuilderVersion, kernel: Option, } impl From for PyProvenance { fn from(b: Provenance) -> Self { Self { - kernel_builder: b.kernel_builder.map(Into::into), + kernel_builder: b.kernel_builder.into(), kernel: b.kernel.map(Into::into), } } @@ -280,7 +280,7 @@ impl From for PyProvenance { #[pymethods] impl PyProvenance { #[getter] - fn kernel_builder(&self) -> Option { + fn kernel_builder(&self) -> PyKernelBuilderVersion { self.kernel_builder.clone() } @@ -292,19 +292,14 @@ impl PyProvenance { /// Whether either the `kernel-builder` or the kernel source was dirty. #[getter] fn dirty(&self) -> bool { - self.kernel_builder - .as_ref() - .and_then(|kb| kb.git.as_ref()) - .is_some_and(|g| g.dirty) + self.kernel_builder.git.as_ref().is_some_and(|g| g.dirty) || self.kernel.as_ref().is_some_and(|k| k.dirty) } fn __repr__(&self) -> String { format!( "Provenance(kernel_builder={}, kernel={})", - self.kernel_builder - .as_ref() - .map_or("None".to_string(), |kb| kb.__repr__()), + self.kernel_builder.__repr__(), self.kernel .as_ref() .map_or("None".to_string(), |k| k.__repr__()) diff --git a/kernels-data/src/metadata.rs b/kernels-data/src/metadata.rs index 1709468e6..d895f7fbb 100644 --- a/kernels-data/src/metadata.rs +++ b/kernels-data/src/metadata.rs @@ -37,8 +37,11 @@ pub struct KernelBuilderVersion { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "kebab-case")] pub struct Provenance { - #[serde(skip_serializing_if = "Option::is_none")] - pub kernel_builder: Option, + /// The `kernel-builder` that produced the build. Always known: it is baked + /// into the `kernel-builder` binary at compile time. + pub kernel_builder: KernelBuilderVersion, + /// Git provenance of the kernel source. `None` when the kernel source was + /// not built from a git repository (so its revision is unknown). #[serde(skip_serializing_if = "Option::is_none")] pub kernel: Option, } @@ -46,10 +49,7 @@ pub struct Provenance { impl Provenance { /// Whether either the `kernel-builder` or the kernel source was dirty. pub fn is_dirty(&self) -> bool { - self.kernel_builder - .as_ref() - .and_then(|kb| kb.git.as_ref()) - .is_some_and(|g| g.dirty) + self.kernel_builder.git.as_ref().is_some_and(|g| g.dirty) || self.kernel.as_ref().is_some_and(|k| k.dirty) } } @@ -132,13 +132,13 @@ mod tests { fn sample_provenance(kernel_builder_dirty: bool, kernel_dirty: bool) -> Provenance { Provenance { - kernel_builder: Some(KernelBuilderVersion { + kernel_builder: KernelBuilderVersion { version: "0.1.0".to_string(), git: Some(GitHash { sha: "a".repeat(40), dirty: kernel_builder_dirty, }), - }), + }, kernel: Some(GitHash { sha: "b".repeat(40), dirty: kernel_dirty, @@ -266,7 +266,7 @@ mod tests { assert!(provenance.is_dirty()); assert_eq!(provenance.kernel.unwrap().sha, "b".repeat(40)); - let kernel_builder = provenance.kernel_builder.unwrap(); + let kernel_builder = provenance.kernel_builder; assert_eq!(kernel_builder.version, "0.1.0"); // The embedded `GitHash` is flattened into the `kernel-builder` object. let git = kernel_builder diff --git a/nix-builder/lib/build.nix b/nix-builder/lib/build.nix index 2a5f2fcc0..f890149d4 100644 --- a/nix-builder/lib/build.nix +++ b/nix-builder/lib/build.nix @@ -113,7 +113,7 @@ rec { rev, doGetKernelCheck, stripRPath ? false, - kernelProvenance ? null, + kernelProvenance, }: let inherit (lib) fileset; @@ -203,7 +203,7 @@ rec { doGetKernelCheck, bundleOnly, buildSets, - kernelProvenance ? null, + kernelProvenance, }: let kernelConfig = readKernelConfig path; @@ -233,7 +233,7 @@ rec { rev, doGetKernelCheck, buildSets, - kernelProvenance ? null, + kernelProvenance, }: let extensions = mkDistTorchExtensions { @@ -297,6 +297,8 @@ rec { rev doGetKernelCheck ; + # Dev/test shells do not record build provenance. + kernelProvenance = null; }; in { @@ -347,7 +349,7 @@ rec { buildSets, doGetKernelCheck, pythonCheckInputs, - kernelProvenance ? null, + kernelProvenance, }: let kernelConfig = readKernelConfig path; @@ -422,6 +424,8 @@ rec { rev doGetKernelCheck ; + # Dev/test shells do not record build provenance. + kernelProvenance = null; }; python = ( pkgs.python3.withPackages ( diff --git a/nix-builder/lib/extension/provenance-flags.nix b/nix-builder/lib/extension/provenance-flags.nix index 7ed49cc58..bfcd1e4db 100644 --- a/nix-builder/lib/extension/provenance-flags.nix +++ b/nix-builder/lib/extension/provenance-flags.nix @@ -3,7 +3,7 @@ # `.git`, so the kernel's commit SHA and dirty state are passed in explicitly. { lib, - kernelProvenance ? null, + kernelProvenance, }: lib.optionalString (kernelProvenance != null) ( "--kernel-sha ${kernelProvenance.sha}" + lib.optionalString kernelProvenance.dirty " --kernel-dirty" diff --git a/nix-builder/lib/extension/torch/arch.nix b/nix-builder/lib/extension/torch/arch.nix index a22cf9ef4..0fa51eaf2 100644 --- a/nix-builder/lib/extension/torch/arch.nix +++ b/nix-builder/lib/extension/torch/arch.nix @@ -73,7 +73,7 @@ # Git provenance (`{ sha; dirty; }`, or `null`) of the kernel source, recorded # in the build metadata. - kernelProvenance ? null, + kernelProvenance, src, }: diff --git a/nix-builder/lib/extension/torch/no-arch.nix b/nix-builder/lib/extension/torch/no-arch.nix index 203d727b2..e0f2b132d 100644 --- a/nix-builder/lib/extension/torch/no-arch.nix +++ b/nix-builder/lib/extension/torch/no-arch.nix @@ -44,7 +44,7 @@ # Git provenance (`{ sha; dirty; }`, or `null`) of the kernel source, recorded # in the build metadata. - kernelProvenance ? null, + kernelProvenance, }: # Extra validation - the environment should correspind to the build config. diff --git a/nix-builder/lib/extension/tvm-ffi/arch.nix b/nix-builder/lib/extension/tvm-ffi/arch.nix index 2ab7293de..ef1e1d2d9 100644 --- a/nix-builder/lib/extension/tvm-ffi/arch.nix +++ b/nix-builder/lib/extension/tvm-ffi/arch.nix @@ -70,7 +70,7 @@ # Git provenance (`{ sha; dirty; }`, or `null`) of the kernel source, recorded # in the build metadata. - kernelProvenance ? null, + kernelProvenance, src, }: From cdf77108460e2f0ad5ac9039afb6749999eab787 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Mon, 6 Jul 2026 17:10:36 +0530 Subject: [PATCH 17/21] put abstractions in the right place. --- flake.nix | 18 ++++--------- nix-builder/lib/build-sets.nix | 9 +++---- nix-builder/lib/flake-provenance.nix | 23 ++++++++++++++++ nix-builder/lib/gen-flake-outputs.nix | 29 ++++----------------- nix-builder/lib/mk-build-set.nix | 13 +++++---- nix-builder/pkgs/kernel-builder/default.nix | 25 +++++++++--------- 6 files changed, 56 insertions(+), 61 deletions(-) create mode 100644 nix-builder/lib/flake-provenance.nix diff --git a/flake.nix b/flake.nix index e2cdd8429..cf44c68e5 100644 --- a/flake.nix +++ b/flake.nix @@ -27,21 +27,13 @@ rust-overlay, }: let - # Git provenance of `kernel-builder` itself, burned into the binary at - # build time and recorded in the build metadata of the kernels it builds. - # `self` here is the `kernel-builder` flake (as resolved in the consuming - # kernel's lock). - builderRev = self.rev or self.dirtyRev or null; - builderDirty = !(self ? rev); - inherit (import ./nix-builder/lib/build-sets.nix { - inherit - nixpkgs - rust-overlay - builderRev - builderDirty - ; + inherit nixpkgs rust-overlay; + # The `kernel-builder` flake itself, so its git provenance can be + # burned into the binary at build time and recorded in the build + # metadata of the kernels it builds. + builderSelf = self; }) mkBuildSets partitionBuildSetsBySystem diff --git a/nix-builder/lib/build-sets.nix b/nix-builder/lib/build-sets.nix index cbf6c36f4..90f2c33bd 100644 --- a/nix-builder/lib/build-sets.nix +++ b/nix-builder/lib/build-sets.nix @@ -2,9 +2,9 @@ nixpkgs, rust-overlay, - # Git provenance of `kernel-builder` itself, burned into the binary. - builderRev ? null, - builderDirty ? false, + # The `kernel-builder` flake itself, so its git provenance can be burned into + # the binary. + builderSelf ? null, }: let @@ -32,8 +32,7 @@ let inherit nixpkgs rust-overlay - builderRev - builderDirty + builderSelf ; }; diff --git a/nix-builder/lib/flake-provenance.nix b/nix-builder/lib/flake-provenance.nix new file mode 100644 index 000000000..d2bb58f58 --- /dev/null +++ b/nix-builder/lib/flake-provenance.nix @@ -0,0 +1,23 @@ +# Extract atomic git provenance (`{ sha; dirty; }`, or `null`) from a flake's +# `self`. The result is `null` when the flake was built from a non-git source +# (no revision is available, e.g. a local `path:`). +{ lib }: +flakeSelf: +let + sha = + if flakeSelf == null then + null + else if flakeSelf ? rev then + flakeSelf.rev + else if flakeSelf ? dirtyRev then + lib.removeSuffix "-dirty" flakeSelf.dirtyRev + else + null; +in +if sha == null then + null +else + { + inherit sha; + dirty = !(flakeSelf ? rev); + } diff --git a/nix-builder/lib/gen-flake-outputs.nix b/nix-builder/lib/gen-flake-outputs.nix index 58f963513..99fba216e 100644 --- a/nix-builder/lib/gen-flake-outputs.nix +++ b/nix-builder/lib/gen-flake-outputs.nix @@ -36,30 +36,11 @@ let revUnderscored = builtins.replaceStrings [ "-" ] [ "_" ] flakeRev; - # Git provenance (commit SHA + dirty state) of the kernel source, recorded in - # the build metadata. The Nix sandbox has no `.git`, so it has to be passed in - # explicitly; the value is atomic data (`{ sha; dirty; }`, or `null` when - # unknown), not pre-formatted `create-pyproject` flags. (`kernel-builder`'s own - # provenance is burned into its binary.) - kernelProvenance = - let - kernelSha = - if self == null then - null - else if self ? rev then - self.rev - else if self ? dirtyRev then - lib.removeSuffix "-dirty" self.dirtyRev - else - null; - in - if kernelSha == null then - null - else - { - sha = kernelSha; - dirty = self != null && !(self ? rev); - }; + # Git provenance (`{ sha; dirty; }`, or `null`) of the kernel source, recorded + # in the build metadata. The Nix sandbox has no `.git`, so it is derived from + # the kernel flake's `self` here and passed in explicitly. (`kernel-builder`'s + # own provenance is burned into its binary.) + kernelProvenance = import ./flake-provenance.nix { inherit lib; } self; applicableBuildSets = build.applicableBuildSets { inherit path buildSets; }; diff --git a/nix-builder/lib/mk-build-set.nix b/nix-builder/lib/mk-build-set.nix index 465e9ceb6..3436005a5 100644 --- a/nix-builder/lib/mk-build-set.nix +++ b/nix-builder/lib/mk-build-set.nix @@ -2,10 +2,9 @@ nixpkgs, rust-overlay, - # Git provenance of `kernel-builder` itself, burned into the binary so that - # the kernels it builds record which `kernel-builder` produced them. - builderRev ? null, - builderDirty ? false, + # The `kernel-builder` flake itself, so its git provenance can be burned into + # the binary that the kernels it builds record. + builderSelf ? null, }: let @@ -13,10 +12,10 @@ let overlay = import ../overlay.nix; - # Inject the `kernel-builder` git provenance into the package. It cannot be - # detected from `.git` in the build sandbox, so it is passed through here. + # Inject the `kernel-builder` flake so the package can burn its git provenance + # into the binary. It cannot be detected from `.git` in the build sandbox. builderProvenanceOverlay = _final: prev: { - kernel-builder = prev.kernel-builder.override { inherit builderRev builderDirty; }; + kernel-builder = prev.kernel-builder.override { inherit builderSelf; }; }; flattenVersion = version: lib.replaceStrings [ "." ] [ "_" ] (lib.versions.pad 2 version); diff --git a/nix-builder/pkgs/kernel-builder/default.nix b/nix-builder/pkgs/kernel-builder/default.nix index a40c8b5be..8527fd641 100644 --- a/nix-builder/pkgs/kernel-builder/default.nix +++ b/nix-builder/pkgs/kernel-builder/default.nix @@ -6,12 +6,11 @@ libgit2, openssl, - # Git provenance of `kernel-builder` itself. Burned into the binary at build - # time (the build sandbox has no `.git`, so `build.rs` cannot detect it) and - # later recorded in the build metadata of the kernels it builds. `null`/`false` - # when `kernel-builder` is built from a non-git source (e.g. a local `path:`). - builderRev ? null, - builderDirty ? false, + # The `kernel-builder` flake itself (or `null` for a non-git source). Its git + # provenance is burned into the binary at build time and later recorded in the + # build metadata of the kernels it builds. The build sandbox has no `.git`, so + # `build.rs` cannot detect it; it is derived from the flake here instead. + builderSelf ? null, }: let @@ -21,18 +20,20 @@ let "-p" "hf-kernel-builder" ]; + + builderProvenance = import ../../lib/flake-provenance.nix { inherit lib; } builderSelf; in rustPlatform.buildRustPackage { inherit version; pname = "kernel-builder"; - # The build sandbox has no `.git`, so the `built` crate cannot detect the - # `kernel-builder` git provenance. Supply it through `built`'s override - # environment variables (`hf_kernel_builder` is the package name with hyphens - # replaced by underscores), which `build.rs` bakes into the binary. + # Supply the git provenance through `built`'s override environment variables + # (`hf_kernel_builder` is the package name with hyphens replaced by + # underscores), which `build.rs` bakes into the binary. "BUILT_OVERRIDE_hf_kernel_builder_GIT_COMMIT_HASH" = - if builderRev == null then "BUILT_OVERRIDE_NONE" else lib.removeSuffix "-dirty" builderRev; - "BUILT_OVERRIDE_hf_kernel_builder_GIT_DIRTY" = if builderDirty then "true" else "false"; + if builderProvenance == null then "BUILT_OVERRIDE_NONE" else builderProvenance.sha; + "BUILT_OVERRIDE_hf_kernel_builder_GIT_DIRTY" = + if builderProvenance != null && builderProvenance.dirty then "true" else "false"; src = let From ebfedf0ffbf7b12da51cc336f90a26650090fdca Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Mon, 6 Jul 2026 17:13:57 +0530 Subject: [PATCH 18/21] document create-pyproject ping pong --- docs/source/kernel-requirements.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/source/kernel-requirements.md b/docs/source/kernel-requirements.md index dc14e4a26..cc54f7160 100644 --- a/docs/source/kernel-requirements.md +++ b/docs/source/kernel-requirements.md @@ -100,6 +100,13 @@ metadata. Currently the following top-level keys are supported: > builds are not flagged as dirty. Local `create-pyproject` runs only > consider changes to tracked files. + > **Note:** The kernel `sha`/`dirty` are captured at the moment + > `create-pyproject` runs, so they describe the source tree as it was *then*. + > Running `create-pyproject` and committing afterwards is bad practice: the + > recorded provenance keeps pointing at the pre-commit state (a stale `sha`, + > and `dirty: true` if the tree was dirty) even though the committed source + > differs. Generate the metadata from the final, committed source instead. + Example `metadata.json`: ```json From 26a5229c702623c4f93f339b92e3f2691e065efb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= Date: Mon, 6 Jul 2026 17:26:58 +0000 Subject: [PATCH 19/21] Simplify flake-provenance.nix --- nix-builder/lib/flake-provenance.nix | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/nix-builder/lib/flake-provenance.nix b/nix-builder/lib/flake-provenance.nix index d2bb58f58..92f0f73ea 100644 --- a/nix-builder/lib/flake-provenance.nix +++ b/nix-builder/lib/flake-provenance.nix @@ -3,21 +3,18 @@ # (no revision is available, e.g. a local `path:`). { lib }: flakeSelf: -let - sha = - if flakeSelf == null then - null - else if flakeSelf ? rev then - flakeSelf.rev - else if flakeSelf ? dirtyRev then - lib.removeSuffix "-dirty" flakeSelf.dirtyRev - else - null; -in -if sha == null then + +if flakeSelf == null then null -else +else if flakeSelf ? rev then + { + sha = flakeSelf.rev; + dirty = false; + } +else if flakeSelf ? dirtyRev then { - inherit sha; - dirty = !(flakeSelf ? rev); + sha = lib.removeSuffix "-dirty" flakeSelf.dirtyRev; + dirty = true; } +else + null From 3463a54e155249fbc76a7da46619b06446878abd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= Date: Mon, 6 Jul 2026 17:34:51 +0000 Subject: [PATCH 20/21] Do not set bogus provenance fields when the builder provenance is null --- nix-builder/pkgs/kernel-builder/default.nix | 142 ++++++++++---------- 1 file changed, 72 insertions(+), 70 deletions(-) diff --git a/nix-builder/pkgs/kernel-builder/default.nix b/nix-builder/pkgs/kernel-builder/default.nix index 8527fd641..c7bb7bbc9 100644 --- a/nix-builder/pkgs/kernel-builder/default.nix +++ b/nix-builder/pkgs/kernel-builder/default.nix @@ -23,84 +23,86 @@ let builderProvenance = import ../../lib/flake-provenance.nix { inherit lib; } builderSelf; in -rustPlatform.buildRustPackage { - inherit version; - pname = "kernel-builder"; - +rustPlatform.buildRustPackage ( # Supply the git provenance through `built`'s override environment variables # (`hf_kernel_builder` is the package name with hyphens replaced by - # underscores), which `build.rs` bakes into the binary. - "BUILT_OVERRIDE_hf_kernel_builder_GIT_COMMIT_HASH" = - if builderProvenance == null then "BUILT_OVERRIDE_NONE" else builderProvenance.sha; - "BUILT_OVERRIDE_hf_kernel_builder_GIT_DIRTY" = - if builderProvenance != null && builderProvenance.dirty then "true" else "false"; + # underscores), which `build.rs` bakes into the binary. When there is no provenance + # information (e.g. non-git source), do not set the variables. + lib.optionalAttrs (builderProvenance != null) { + "BUILT_OVERRIDE_hf_kernel_builder_GIT_COMMIT_HASH" = builderProvenance.sha; + "BUILT_OVERRIDE_hf_kernel_builder_GIT_DIRTY" = if builderProvenance.dirty then "true" else "false"; + } + // { + inherit version; + pname = "kernel-builder"; + + src = + let + sourceFiles = + file: + file.name == "Cargo.toml" + || file.name == "Cargo.lock" + || file.name == "flake.nix" + || file.name == "manylinux-policy.json" + || file.name == "pyproject.toml" + || file.name == "pyproject_universal.toml" + || file.name == "python_dependencies.json" + || file.name == "shim_function_versions.txt" + || file.name == "stable_abi.toml" + || file.name == ".gitattributes" + || file.name == ".gitignore" + || (builtins.any file.hasExt [ + "cmake" + "cpp" + "cu" + "h" + "in" + "md" + "metal" + "mm" + "py" + "rs" + "toml" + ]); + in + import ../crate-dirs.nix { + inherit lib sourceFiles; + }; - src = - let - sourceFiles = - file: - file.name == "Cargo.toml" - || file.name == "Cargo.lock" - || file.name == "flake.nix" - || file.name == "manylinux-policy.json" - || file.name == "pyproject.toml" - || file.name == "pyproject_universal.toml" - || file.name == "python_dependencies.json" - || file.name == "shim_function_versions.txt" - || file.name == "stable_abi.toml" - || file.name == ".gitattributes" - || file.name == ".gitignore" - || (builtins.any file.hasExt [ - "cmake" - "cpp" - "cu" - "h" - "in" - "md" - "metal" - "mm" - "py" - "rs" - "toml" - ]); - in - import ../crate-dirs.nix { - inherit lib sourceFiles; + cargoLock = { + lockFile = ../../../Cargo.lock; }; - cargoLock = { - lockFile = ../../../Cargo.lock; - }; + cargoBuildFlags = cargoFlags; + cargoTestFlags = cargoFlags; - cargoBuildFlags = cargoFlags; - cargoTestFlags = cargoFlags; + # e2e tests look for binary at target/debug/ which doesn't exist in nix + doCheck = false; - # e2e tests look for binary at target/debug/ which doesn't exist in nix - doCheck = false; + nativeBuildInputs = [ + installShellFiles + pkg-config + ]; - nativeBuildInputs = [ - installShellFiles - pkg-config - ]; + buildInputs = [ + libgit2 + openssl.dev + ]; - buildInputs = [ - libgit2 - openssl.dev - ]; + postInstall = '' + for shell in bash fish zsh; do + $out/bin/kernel-builder completions $shell > kernel-builder.$shell + done + installShellCompletion kernel-builder.{bash,fish,zsh} + ''; - postInstall = '' - for shell in bash fish zsh; do - $out/bin/kernel-builder completions $shell > kernel-builder.$shell - done - installShellCompletion kernel-builder.{bash,fish,zsh} - ''; + setupHooks = [ + ./check-kernel-abi-hook.sh + ./check-kernel-build-hook.sh + ]; - setupHooks = [ - ./check-kernel-abi-hook.sh - ./check-kernel-build-hook.sh - ]; - - meta = { - description = "Create cmake build infrastructure from build.toml files"; - }; -} + meta = { + description = "Create cmake build infrastructure from build.toml files"; + }; + } +) From 91b7290ec3e1f560f6cc960cb309b7a4bdb6b8c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= Date: Mon, 6 Jul 2026 18:01:02 +0000 Subject: [PATCH 21/21] Avoid overlay and pass through provenance in a more focused way --- flake.nix | 13 ++++++++----- nix-builder/lib/build-sets.nix | 10 ++++------ nix-builder/lib/mk-build-set.nix | 15 ++++----------- nix-builder/overlay.nix | 9 ++++++++- nix-builder/pkgs/kernel-builder/default.nix | 12 +++++------- 5 files changed, 29 insertions(+), 30 deletions(-) diff --git a/flake.nix b/flake.nix index cf44c68e5..cc3993a75 100644 --- a/flake.nix +++ b/flake.nix @@ -27,13 +27,16 @@ rust-overlay, }: let + # Git provenance (`{ sha, dirty }` or `null`) of the `kernel-builder` + # flake itself, so it can be burned into the binary at build time and + # recorded in the build metadata of the kernels it builds. + builderProvenance = import ./nix-builder/lib/flake-provenance.nix { + inherit (nixpkgs) lib; + } self; + inherit (import ./nix-builder/lib/build-sets.nix { - inherit nixpkgs rust-overlay; - # The `kernel-builder` flake itself, so its git provenance can be - # burned into the binary at build time and recorded in the build - # metadata of the kernels it builds. - builderSelf = self; + inherit nixpkgs rust-overlay builderProvenance; }) mkBuildSets partitionBuildSetsBySystem diff --git a/nix-builder/lib/build-sets.nix b/nix-builder/lib/build-sets.nix index 90f2c33bd..76cada998 100644 --- a/nix-builder/lib/build-sets.nix +++ b/nix-builder/lib/build-sets.nix @@ -2,16 +2,14 @@ nixpkgs, rust-overlay, - # The `kernel-builder` flake itself, so its git provenance can be burned into - # the binary. - builderSelf ? null, + # Git provenance (`{ sha, dirty }` or `null`) of the `kernel-builder` flake, + # so it can be burned into the binary. + builderProvenance ? null, }: let inherit (nixpkgs) lib; - overlay = import ../nix/overlay.nix; - inherit (import ./torch-version-utils.nix { inherit lib; }) backend flattenSystems @@ -32,7 +30,7 @@ let inherit nixpkgs rust-overlay - builderSelf + builderProvenance ; }; diff --git a/nix-builder/lib/mk-build-set.nix b/nix-builder/lib/mk-build-set.nix index 3436005a5..b12308250 100644 --- a/nix-builder/lib/mk-build-set.nix +++ b/nix-builder/lib/mk-build-set.nix @@ -2,21 +2,15 @@ nixpkgs, rust-overlay, - # The `kernel-builder` flake itself, so its git provenance can be burned into - # the binary that the kernels it builds record. - builderSelf ? null, + # Git provenance (`{ sha, dirty }` or `null`) of the `kernel-builder` flake, + # so it can be burned into the binary that the kernels it builds record. + builderProvenance ? null, }: let inherit (nixpkgs) lib; - overlay = import ../overlay.nix; - - # Inject the `kernel-builder` flake so the package can burn its git provenance - # into the binary. It cannot be detected from `.git` in the build sandbox. - builderProvenanceOverlay = _final: prev: { - kernel-builder = prev.kernel-builder.override { inherit builderSelf; }; - }; + overlay = import ../overlay.nix { inherit builderProvenance; }; flattenVersion = version: lib.replaceStrings [ "." ] [ "_" ] (lib.versions.pad 2 version); @@ -137,7 +131,6 @@ let inherit config system; overlays = [ overlay - builderProvenanceOverlay rust-overlay.overlays.default ] ++ backendOverlay diff --git a/nix-builder/overlay.nix b/nix-builder/overlay.nix index 9fab8d225..b7328dbe6 100644 --- a/nix-builder/overlay.nix +++ b/nix-builder/overlay.nix @@ -1,6 +1,13 @@ +{ + # Git provenance (`{ sha, dirty }` or `null`) of the `kernel-builder` flake, + # burned into the binary. It cannot be detected from `.git` in the build + # sandbox. + builderProvenance ? null, +}: + final: prev: { - kernel-builder = final.callPackage ./pkgs/kernel-builder { }; + kernel-builder = final.callPackage ./pkgs/kernel-builder { inherit builderProvenance; }; cmakeNvccThreadsHook = final.callPackage ./pkgs/cmake-nvcc-threads-hook { }; diff --git a/nix-builder/pkgs/kernel-builder/default.nix b/nix-builder/pkgs/kernel-builder/default.nix index c7bb7bbc9..31d02bcae 100644 --- a/nix-builder/pkgs/kernel-builder/default.nix +++ b/nix-builder/pkgs/kernel-builder/default.nix @@ -6,11 +6,11 @@ libgit2, openssl, - # The `kernel-builder` flake itself (or `null` for a non-git source). Its git - # provenance is burned into the binary at build time and later recorded in the - # build metadata of the kernels it builds. The build sandbox has no `.git`, so - # `build.rs` cannot detect it; it is derived from the flake here instead. - builderSelf ? null, + # Git provenance (`{ sha, dirty }`, or `null` for a non-git source) of the + # `kernel-builder` flake. It is burned into the binary at build time and + # later recorded in the build metadata of the kernels it builds. The build + # sandbox has no `.git`, so `build.rs` cannot detect it. + builderProvenance ? null, }: let @@ -20,8 +20,6 @@ let "-p" "hf-kernel-builder" ]; - - builderProvenance = import ../../lib/flake-provenance.nix { inherit lib; } builderSelf; in rustPlatform.buildRustPackage ( # Supply the git provenance through `built`'s override environment variables