diff --git a/docs/source/builder-cli.md b/docs/source/builder-cli.md index 3fd7d7644..532bc8ba1 100644 --- a/docs/source/builder-cli.md +++ b/docs/source/builder-cli.md @@ -83,7 +83,7 @@ Initialize a new kernel project from template Default value: `Apache-2.0` * `--name ` — Name of the kernel repo (e.g. `drbh/my-kernel`) -* `--backends ` — Backends to enable (`all`, `cpu`, `cuda`, `metal`, `neuron`, `rocm`, `xpu`) +* `--backends ` — Backends to enable (`all`, `cpu`, `cuda`, `metal`, `neuron`, `rocm`, `tpu`, `xpu`) * `--overwrite` — Overwrite existing scaffold files (preserves other files) diff --git a/docs/source/builder/build-variants.md b/docs/source/builder/build-variants.md index bdc9033fb..5c7011d76 100644 --- a/docs/source/builder/build-variants.md +++ b/docs/source/builder/build-variants.md @@ -62,6 +62,10 @@ available. This list will be updated as new PyTorch versions are released. - `torch213-cxx11-rocm71-x86_64-linux` - `torch213-cxx11-rocm72-x86_64-linux` +## TPU x86_64-linux + +- `torch-tpu` + ## XPU x86_64-linux - `torch211-cxx11-xpu20253-x86_64-linux` @@ -77,4 +81,5 @@ one or more of the following variants: - `torch-cuda` - `torch-metal` - `torch-rocm` +- `torch-tpu` - `torch-xpu` diff --git a/docs/source/kernel-requirements.md b/docs/source/kernel-requirements.md index eb4324819..2c4dd34f9 100644 --- a/docs/source/kernel-requirements.md +++ b/docs/source/kernel-requirements.md @@ -149,7 +149,7 @@ The `backend` specifies a dictionary of the following form: ``` The backend `type` must be one of `cann`, `cpu`, `cuda`, `metal`, `neuron`, -`rocm`, or `xpu`. For CUDA and ROCm, the supported architectures must +`rocm`, `tpu`, or `xpu`. For CUDA and ROCm, the supported architectures must be specified in the `archs` field. ### Python dependencies diff --git a/examples/kernels/relu-tpu/CARD.md b/examples/kernels/relu-tpu/CARD.md new file mode 100644 index 000000000..92ecc33da --- /dev/null +++ b/examples/kernels/relu-tpu/CARD.md @@ -0,0 +1,61 @@ +--- +library_name: kernels +{% if license %}license: {{ license }} +{% endif %}--- + +This is the repository card of {{ repo_id }} that has been pushed on the Hub. It was built to be used with the [`kernels` library](https://github.com/huggingface/kernels). This card was automatically generated. + +## How to use +{% if functions %} + +```python +# make sure `kernels` is installed: `pip install -U kernels` +from kernels import get_kernel + +kernel_module = get_kernel("{{ repo_id }}", version={{ version }}) +{{ functions[0] }} = kernel_module.{{ functions[0] }} + +{{ functions[0] }}(...) +``` +{% else %} + +Usage example not available. +{% endif %} + +## Available functions +{% if functions %} +{% for func in functions %} +- `{{ func }}` +{% endfor %} +{% else %} + +Function list not available. +{% endif %} +{% if layers %} + +## Available layers +{% for layer in layers %} +- `{{ layer }}` +{% endfor %} +{% endif %} + +## Benchmarks +{% if has_benchmark %} + +Benchmarking script is available for this kernel. Run `kernels benchmark {{ repo_id }} --version {{ version }}`. +{% else %} + +No benchmark available yet. +{% endif %} +{% if upstream %} + +## Upstream + +The original source code for this kernel comes from {{ upstream }}. +{% endif %} +{% if source %} + +## Source + +The kernel-builder formatted source for this kernel is available at {{ source }}. +{% endif %} diff --git a/examples/kernels/relu-tpu/build.toml b/examples/kernels/relu-tpu/build.toml new file mode 100644 index 000000000..c930a6a61 --- /dev/null +++ b/examples/kernels/relu-tpu/build.toml @@ -0,0 +1,16 @@ +[general] +name = "relu-tpu" +version = 1 +edition = 5 +license = "Apache-2.0" +backends = ["tpu"] + +[general.hub] +repo-id = "kernels-test/relu-tpu" + +[general.tpu] +python-depends = ["torch_tpu", "jax"] + +[torch-noarch] + +[kernel] diff --git a/examples/kernels/relu-tpu/flake.nix b/examples/kernels/relu-tpu/flake.nix new file mode 100644 index 000000000..a30872561 --- /dev/null +++ b/examples/kernels/relu-tpu/flake.nix @@ -0,0 +1,17 @@ +{ + description = "Flake for TPU ReLU kernel"; + + inputs = { + kernel-builder.url = "path:../../.."; + }; + + outputs = + { + self, + kernel-builder, + }: + kernel-builder.lib.genKernelFlakeOutputs { + inherit self; + path = ./.; + }; +} diff --git a/examples/kernels/relu-tpu/torch-ext/relu_tpu/__init__.py b/examples/kernels/relu-tpu/torch-ext/relu_tpu/__init__.py new file mode 100644 index 000000000..677bca5ff --- /dev/null +++ b/examples/kernels/relu-tpu/torch-ext/relu_tpu/__init__.py @@ -0,0 +1,14 @@ +import jax +import jax.numpy as jnp +from torch_tpu._internal.pallas import jax_op + + +def _jax_relu(x: jax.Array) -> jax.Array: + return jnp.maximum(x, 0) + + +relu = jax_op("relu_tpu::relu", _jax_relu) + +from . import layers # noqa: E402 + +__all__ = ["layers", "relu"] diff --git a/examples/kernels/relu-tpu/torch-ext/relu_tpu/layers/__init__.py b/examples/kernels/relu-tpu/torch-ext/relu_tpu/layers/__init__.py new file mode 100644 index 000000000..7800a2ef7 --- /dev/null +++ b/examples/kernels/relu-tpu/torch-ext/relu_tpu/layers/__init__.py @@ -0,0 +1,9 @@ +import torch +import torch.nn as nn + +from .. import relu + + +class ReLU(nn.Module): + def forward(self, x: torch.Tensor) -> torch.Tensor: + return relu(x) diff --git a/kernel-builder/src/init.rs b/kernel-builder/src/init.rs index 8ce37f48a..f12b4edd6 100644 --- a/kernel-builder/src/init.rs +++ b/kernel-builder/src/init.rs @@ -44,7 +44,7 @@ pub struct InitArgs { #[arg(long, value_name = "OWNER/REPO")] pub name: Option, - /// Backends to enable (`all`, `cpu`, `cuda`, `metal`, `neuron`, `rocm`, `xpu`). + /// Backends to enable (`all`, `cpu`, `cuda`, `metal`, `neuron`, `rocm`, `tpu`, `xpu`). #[arg(long, num_args = 1.., default_values_t = default_init_backends())] pub backends: Vec, diff --git a/kernel-builder/src/pyproject/templates/torch/noarch/_ops.py b/kernel-builder/src/pyproject/templates/torch/noarch/_ops.py index 75fd8e086..e8d30b5ff 100644 --- a/kernel-builder/src/pyproject/templates/torch/noarch/_ops.py +++ b/kernel-builder/src/pyproject/templates/torch/noarch/_ops.py @@ -4,7 +4,13 @@ def get_backend() -> str: """Detect the backend by inspecting torch.""" import torch - if hasattr(torch, "neuron"): + if hasattr(torch.backends, "tpu"): + # torch_tpu sets torch.backends.tpu when it is imported (via + # torch's device-backend autoload), regardless of whether TPU + # hardware is present — analogous to torch.version.cuda being + # set on CUDA builds without a GPU. + return "tpu" + elif hasattr(torch, "neuron"): # Needs to be sorted before specific Torch builds, since Neuron # extension can be loaded into e.g. CUDA Torch builds. return "neuron" diff --git a/kernels-data/bindings/python/kernels_data.pyi b/kernels-data/bindings/python/kernels_data.pyi index 20e29619f..43b7af6a6 100644 --- a/kernels-data/bindings/python/kernels_data.pyi +++ b/kernels-data/bindings/python/kernels_data.pyi @@ -32,6 +32,7 @@ class Backend(Enum): Metal = "Metal" Neuron = "Neuron" ROCm = "ROCm" + TPU = "TPU" XPU = "XPU" @staticmethod @@ -40,7 +41,7 @@ class Backend(Enum): Args: s: One of `"cann"`, `"cpu"`, `"cuda"`, `"metal"`, - `"neuron"`, `"rocm"`, `"xpu"`. + `"neuron"`, `"rocm"`, `"tpu"`, `"xpu"`. Raises: ValueError: If the backend name is unknown. diff --git a/kernels-data/bindings/python/src/lib.rs b/kernels-data/bindings/python/src/lib.rs index 9aa29f06d..3cfcae33e 100644 --- a/kernels-data/bindings/python/src/lib.rs +++ b/kernels-data/bindings/python/src/lib.rs @@ -88,6 +88,8 @@ enum PyBackend { Neuron, #[pyo3(name = "ROCm")] Rocm, + #[pyo3(name = "TPU")] + Tpu, #[pyo3(name = "XPU")] Xpu, } @@ -101,6 +103,7 @@ impl From for PyBackend { Backend::Metal => PyBackend::Metal, Backend::Neuron => PyBackend::Neuron, Backend::Rocm => PyBackend::Rocm, + Backend::Tpu => PyBackend::Tpu, Backend::Xpu => PyBackend::Xpu, } } @@ -115,6 +118,7 @@ impl From for Backend { PyBackend::Metal => Backend::Metal, PyBackend::Neuron => Backend::Neuron, PyBackend::Rocm => Backend::Rocm, + PyBackend::Tpu => Backend::Tpu, PyBackend::Xpu => Backend::Xpu, } } @@ -123,7 +127,7 @@ impl From for Backend { #[pymethods] impl PyBackend { /// Parse a backend name (`"cann"`, `"cpu"`, `"cuda"`, `"metal"`, - /// `"neuron"`, `"rocm"`, `"xpu"`). + /// `"neuron"`, `"rocm"`, `"tpu"`, `"xpu"`). #[staticmethod] #[pyo3(name = "from_str")] fn py_from_str(s: &str) -> PyResult { @@ -144,6 +148,7 @@ impl PyBackend { PyBackend::Metal => "Metal", PyBackend::Neuron => "Neuron", PyBackend::Rocm => "ROCm", + PyBackend::Tpu => "TPU", PyBackend::Xpu => "XPU", }; format!("Backend.{variant}") diff --git a/kernels-data/bindings/python/tests/test_kernels_data.py b/kernels-data/bindings/python/tests/test_kernels_data.py index 29bb52fe5..87f9d1119 100644 --- a/kernels-data/bindings/python/tests/test_kernels_data.py +++ b/kernels-data/bindings/python/tests/test_kernels_data.py @@ -68,7 +68,7 @@ def test_backend_hash(): def test_backend_unknown(): with pytest.raises(ValueError): - Backend.from_str("tpu") + Backend.from_str("dsp") def test_backend_all_variants_and_casing(): @@ -80,8 +80,11 @@ def test_backend_all_variants_and_casing(): assert repr(Backend.ROCm) == "Backend.ROCm" assert repr(Backend.XPU) == "Backend.XPU" assert repr(Backend.CANN) == "Backend.CANN" + assert str(Backend.TPU) == "tpu" + assert repr(Backend.TPU) == "Backend.TPU" assert Backend.from_str("cann") == Backend.CANN assert Backend.from_str("ROCM") == Backend.ROCm + assert Backend.from_str("TPU") == Backend.TPU assert Backend.from_str("metal") == Backend.Metal diff --git a/kernels-data/src/config/mod.rs b/kernels-data/src/config/mod.rs index bc7855f71..ed111d70d 100644 --- a/kernels-data/src/config/mod.rs +++ b/kernels-data/src/config/mod.rs @@ -113,6 +113,7 @@ pub struct General { pub cuda: Option, pub neuron: Option, + pub tpu: Option, pub xpu: Option, } @@ -144,6 +145,10 @@ impl General { .cuda .as_ref() .and_then(|cuda| cuda.python_depends.as_ref()), + Backend::Tpu => self + .tpu + .as_ref() + .and_then(|tpu| tpu.python_depends.as_ref()), Backend::Xpu => self .xpu .as_ref() @@ -191,6 +196,10 @@ pub struct NeuronGeneral { pub python_depends: Option>, } +pub struct TpuGeneral { + pub python_depends: Option>, +} + pub struct Hub { pub repo_id: Option, pub branch: Option, @@ -379,11 +388,12 @@ pub enum Backend { Metal, Neuron, Rocm, + Tpu, Xpu, } impl Backend { - pub const fn all() -> [Backend; 7] { + pub const fn all() -> [Backend; 8] { [ Backend::Cann, Backend::Cpu, @@ -391,6 +401,7 @@ impl Backend { Backend::Metal, Backend::Neuron, Backend::Rocm, + Backend::Tpu, Backend::Xpu, ] } @@ -403,6 +414,7 @@ impl Backend { Backend::Metal => "metal", Backend::Neuron => "neuron", Backend::Rocm => "rocm", + Backend::Tpu => "tpu", Backend::Xpu => "xpu", } } @@ -417,6 +429,7 @@ impl Display for Backend { Backend::Metal => write!(f, "metal"), Backend::Neuron => write!(f, "neuron"), Backend::Rocm => write!(f, "rocm"), + Backend::Tpu => write!(f, "tpu"), Backend::Xpu => write!(f, "xpu"), } } @@ -433,6 +446,7 @@ impl FromStr for Backend { "metal" => Ok(Backend::Metal), "neuron" => Ok(Backend::Neuron), "rocm" => Ok(Backend::Rocm), + "tpu" => Ok(Backend::Tpu), "xpu" => Ok(Backend::Xpu), _ => Err(format!("Unknown backend: {s}")), } @@ -444,3 +458,51 @@ pub enum ConfigError { #[error("Cannot migrate configuration: {reason:?}")] Migration { reason: String }, } + +#[cfg(test)] +mod tests { + use super::*; + + fn general_with_tpu_deps() -> General { + General { + name: KernelName::new("test-kernel").unwrap(), + version: 1, + license: "apache-2.0".to_string(), + upstream: None, + source: None, + backends: vec![Backend::Tpu], + hub: None, + python_depends: None, + cuda: None, + neuron: None, + tpu: Some(TpuGeneral { + python_depends: Some(vec!["torch_tpu".to_string()]), + }), + xpu: None, + } + } + + #[test] + fn backend_python_depends_resolves_tpu() { + let general = general_with_tpu_deps(); + let deps = general + .backend_python_depends(Backend::Tpu) + .map(|dep| dep.map(|(name, _)| name.to_string())) + .collect::>>() + .unwrap(); + + assert_eq!(deps, vec!["torch_tpu".to_string()]); + } + + #[test] + fn backend_python_depends_empty_for_backend_without_deps() { + let general = general_with_tpu_deps(); + + assert!( + general + .backend_python_depends(Backend::Cpu) + .next() + .is_none() + ); + } +} diff --git a/kernels-data/src/config/v2.rs b/kernels-data/src/config/v2.rs index eed543ba6..f79204a8b 100644 --- a/kernels-data/src/config/v2.rs +++ b/kernels-data/src/config/v2.rs @@ -179,6 +179,7 @@ impl General { hub: general.hub.map(Into::into), neuron: None, python_depends: None, + tpu: None, xpu: None, } } diff --git a/kernels-data/src/config/v3.rs b/kernels-data/src/config/v3.rs index 590d16e6a..766a0cf9f 100644 --- a/kernels-data/src/config/v3.rs +++ b/kernels-data/src/config/v3.rs @@ -50,6 +50,8 @@ pub struct General { pub python_depends: Option>, + pub tpu: Option, + pub xpu: Option, } @@ -67,6 +69,12 @@ pub struct NeuronGeneral { pub python_depends: Option>, } +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub struct TpuGeneral { + pub python_depends: Option>, +} + #[derive(Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields, rename_all = "kebab-case")] pub struct XpuGeneral { @@ -155,6 +163,7 @@ pub enum Backend { Metal, Neuron, Rocm, + Tpu, Xpu, } @@ -205,6 +214,7 @@ impl TryFrom for super::General { hub: general.hub.map(Into::into), neuron: general.neuron.map(Into::into), python_depends: general.python_depends, + tpu: general.tpu.map(Into::into), xpu: general.xpu.map(Into::into), }) } @@ -228,6 +238,14 @@ impl From for super::NeuronGeneral { } } +impl From for super::TpuGeneral { + fn from(tpu: TpuGeneral) -> Self { + Self { + python_depends: tpu.python_depends, + } + } +} + impl From for super::XpuGeneral { fn from(xpu: XpuGeneral) -> Self { Self { @@ -279,6 +297,7 @@ impl From for super::Backend { Backend::Metal => super::Backend::Metal, Backend::Neuron => super::Backend::Neuron, Backend::Rocm => super::Backend::Rocm, + Backend::Tpu => super::Backend::Tpu, Backend::Xpu => super::Backend::Xpu, } } diff --git a/kernels-data/src/config/v4.rs b/kernels-data/src/config/v4.rs index a88db69c9..ecc6c0125 100644 --- a/kernels-data/src/config/v4.rs +++ b/kernels-data/src/config/v4.rs @@ -49,6 +49,8 @@ pub struct General { pub python_depends: Option>, + pub tpu: Option, + pub xpu: Option, } @@ -66,6 +68,12 @@ pub struct NeuronGeneral { pub python_depends: Option>, } +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub struct TpuGeneral { + pub python_depends: Option>, +} + #[derive(Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields, rename_all = "kebab-case")] pub struct XpuGeneral { @@ -171,6 +179,7 @@ pub enum Backend { Metal, Neuron, Rocm, + Tpu, Xpu, } @@ -203,6 +212,7 @@ impl From for super::General { hub: general.hub.map(Into::into), neuron: general.neuron.map(Into::into), python_depends: general.python_depends, + tpu: general.tpu.map(Into::into), xpu: general.xpu.map(Into::into), } } @@ -238,6 +248,14 @@ impl From for super::NeuronGeneral { } } +impl From for super::TpuGeneral { + fn from(tpu: TpuGeneral) -> Self { + Self { + python_depends: tpu.python_depends, + } + } +} + impl From for super::XpuGeneral { fn from(xpu: XpuGeneral) -> Self { Self { @@ -300,6 +318,7 @@ impl From for super::Backend { Backend::Metal => super::Backend::Metal, Backend::Neuron => super::Backend::Neuron, Backend::Rocm => super::Backend::Rocm, + Backend::Tpu => super::Backend::Tpu, Backend::Xpu => super::Backend::Xpu, } } diff --git a/kernels-data/src/config/v5.rs b/kernels-data/src/config/v5.rs index 0154eeef6..3e8a99960 100644 --- a/kernels-data/src/config/v5.rs +++ b/kernels-data/src/config/v5.rs @@ -62,6 +62,8 @@ pub struct General { pub python_depends: Option>, + pub tpu: Option, + pub xpu: Option, } @@ -79,6 +81,12 @@ pub struct NeuronGeneral { pub python_depends: Option>, } +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub struct TpuGeneral { + pub python_depends: Option>, +} + #[derive(Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields, rename_all = "kebab-case")] pub struct XpuGeneral { @@ -185,6 +193,7 @@ pub enum Backend { Metal, Neuron, Rocm, + Tpu, Xpu, } @@ -217,6 +226,7 @@ impl From for super::General { hub: general.hub.map(Into::into), neuron: general.neuron.map(Into::into), python_depends: general.python_depends, + tpu: general.tpu.map(Into::into), xpu: general.xpu.map(Into::into), } } @@ -252,6 +262,14 @@ impl From for super::NeuronGeneral { } } +impl From for super::TpuGeneral { + fn from(tpu: TpuGeneral) -> Self { + Self { + python_depends: tpu.python_depends, + } + } +} + impl From for super::XpuGeneral { fn from(xpu: XpuGeneral) -> Self { Self { @@ -313,6 +331,7 @@ impl From for super::Backend { Backend::Metal => super::Backend::Metal, Backend::Neuron => super::Backend::Neuron, Backend::Rocm => super::Backend::Rocm, + Backend::Tpu => super::Backend::Tpu, Backend::Xpu => super::Backend::Xpu, } } @@ -420,6 +439,7 @@ impl From for General { hub: general.hub.map(Into::into), neuron: general.neuron.map(Into::into), python_depends: general.python_depends, + tpu: general.tpu.map(Into::into), xpu: general.xpu.map(Into::into), } } @@ -455,6 +475,14 @@ impl From for NeuronGeneral { } } +impl From for TpuGeneral { + fn from(tpu: super::TpuGeneral) -> Self { + Self { + python_depends: tpu.python_depends, + } + } +} + impl From for XpuGeneral { fn from(xpu: super::XpuGeneral) -> Self { Self { @@ -516,6 +544,7 @@ impl From for Backend { super::Backend::Metal => Backend::Metal, super::Backend::Neuron => Backend::Neuron, super::Backend::Rocm => Backend::Rocm, + super::Backend::Tpu => Backend::Tpu, super::Backend::Xpu => Backend::Xpu, } } diff --git a/kernels-data/src/metadata.rs b/kernels-data/src/metadata.rs index d895f7fbb..3da04dc46 100644 --- a/kernels-data/src/metadata.rs +++ b/kernels-data/src/metadata.rs @@ -159,6 +159,7 @@ mod tests { python_depends: None, cuda: None, neuron: None, + tpu: None, xpu: None, }, kernels: HashMap::new(), @@ -214,6 +215,7 @@ mod tests { python_depends: None, cuda: None, neuron: None, + tpu: None, xpu: None, }, kernels: HashMap::new(), diff --git a/kernels-data/src/python_dependencies.json b/kernels-data/src/python_dependencies.json index 266904e82..8de880f5b 100644 --- a/kernels-data/src/python_dependencies.json +++ b/kernels-data/src/python_dependencies.json @@ -34,6 +34,20 @@ "nix": [], "python": [{ "pkg": "onednn-devel" }] } + }, + "tpu": { + "torch_tpu": { + "nix": ["torch_tpu"], + "python": [{ "pkg": "torch_tpu", "import": "torch_tpu" }] + }, + "jax": { + "nix": ["jax"], + "python": [{ "pkg": "jax", "import": "jax" }] + }, + "libtpu": { + "nix": ["libtpu"], + "python": [{ "pkg": "libtpu" }] + } } } } diff --git a/kernels/pytest.ini b/kernels/pytest.ini index a94211637..22164adcc 100644 --- a/kernels/pytest.ini +++ b/kernels/pytest.ini @@ -5,4 +5,5 @@ markers = darwin_only: marks tests that should only run on macOS xpu_only: marks tests that should only run on hosts with Intel XPUs npu_only: marks tests that should only run on Ascend NPUs + tpu_only: marks tests that should only run on hosts with TPU devices token: enable tests that require a write token diff --git a/kernels/src/kernels/backends.py b/kernels/src/kernels/backends.py index 5dff532a1..c5b829209 100644 --- a/kernels/src/kernels/backends.py +++ b/kernels/src/kernels/backends.py @@ -149,6 +149,24 @@ def parse(s: str) -> "ROCm": return ROCm(version=Version(f"{m.group(1)}.{m.group(2)}")) +@strict +@dataclass(unsafe_hash=True) +class TPU: + @property + def name(self) -> str: + return "tpu" + + @property + def variant_str(self) -> str: + return "tpu" + + @staticmethod + def parse(s: str) -> "TPU": + if s != "tpu": + raise ValueError(f"Invalid TPU variant string: {s!r}") + return TPU() + + @dataclass(unsafe_hash=True) class XPU: _VARIANT_REGEX: ClassVar[re.Pattern] = re.compile(r"xpu(\d+)(\d+)") @@ -179,6 +197,8 @@ def parse_backend(s: str) -> Backend: return Metal.parse(s) elif s == "neuron": return Neuron.parse(s) + elif s == "tpu": + return TPU.parse(s) elif s.startswith("cu"): return CUDA.parse(s) elif s.startswith("rocm"): @@ -195,7 +215,14 @@ def _backend() -> Backend: if has_torch: import torch - if hasattr(torch, "neuron"): + if hasattr(torch.backends, "tpu"): + # torch_tpu sets torch.backends.tpu when it is imported (via + # torch's device-backend autoload), regardless of whether TPU + # hardware is present — analogous to torch.version.cuda being + # set on CUDA builds without a GPU. The hardware-gated + # torch.tpu namespace only appears on hosts with TPU devices. + return TPU() + elif hasattr(torch, "neuron"): # Needs to be sorted before specific Torch builds, since Neuron # extension can be loaded into e.g. CUDA Torch builds. return Neuron() diff --git a/kernels/src/kernels/layer/kernelize.py b/kernels/src/kernels/layer/kernelize.py index 5294a5f5d..60d221d6a 100644 --- a/kernels/src/kernels/layer/kernelize.py +++ b/kernels/src/kernels/layer/kernelize.py @@ -192,7 +192,7 @@ def kernelize( `Mode.TRAINING | Mode.TORCH_COMPILE` kernelizes the model for training with `torch.compile`. device (`Union[str, torch.device]`, *optional*): - The device type to load kernels for. Supported device types are: "cuda", "mps", "npu", "rocm", "xpu". + The device type to load kernels for. Supported device types are: "cuda", "mps", "npu", "rocm", "tpu", "xpu". The device type will be inferred from the model parameters when not provided. use_fallback (`bool`, *optional*, defaults to `True`): Whether to use the original forward method of modules when no compatible kernel could be found. @@ -276,7 +276,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: def _validate_device_type(device_type: str) -> None: """Validate that the device type is supported.""" - supported_devices = {"cpu", "cuda", "mps", "neuron", "npu", "rocm", "xpu"} + supported_devices = {"cpu", "cuda", "mps", "neuron", "npu", "rocm", "tpu", "xpu"} if device_type not in supported_devices: raise ValueError( f"Unsupported device type '{device_type}'. Supported device types are: {', '.join(sorted(supported_devices))}" diff --git a/kernels/src/kernels/layer/repos.py b/kernels/src/kernels/layer/repos.py index fd02707c2..c53c833bb 100644 --- a/kernels/src/kernels/layer/repos.py +++ b/kernels/src/kernels/layer/repos.py @@ -37,6 +37,8 @@ def create_repo(device: Device) -> "DeviceRepos": return _NPURepos() elif device.type == "neuron": return _NeuronRepos() + elif device.type == "tpu": + return _TPURepos() else: raise ValueError(f"Unknown device type: {device.type}") @@ -114,6 +116,26 @@ def insert(self, device: Device, repos: dict[Mode, RepositoryProtocol]): self._repos = repos +class _TPURepos(DeviceRepos): + _repos: dict[Mode, RepositoryProtocol] + + def __init__(self): + super().__init__() + self._repos = {} + + @property + def repos( + self, + ) -> dict[Mode, RepositoryProtocol] | None: + return self._repos + + def insert(self, device: Device, repos: dict[Mode, RepositoryProtocol]): + if device.type != "tpu": + raise ValueError(f"Device type must be 'tpu', got {device.type}") + + self._repos = repos + + class _NPURepos(DeviceRepos): _repos: dict[Mode, RepositoryProtocol] diff --git a/kernels/src/kernels/python_depends.json b/kernels/src/kernels/python_depends.json index 266904e82..8de880f5b 100644 --- a/kernels/src/kernels/python_depends.json +++ b/kernels/src/kernels/python_depends.json @@ -34,6 +34,20 @@ "nix": [], "python": [{ "pkg": "onednn-devel" }] } + }, + "tpu": { + "torch_tpu": { + "nix": ["torch_tpu"], + "python": [{ "pkg": "torch_tpu", "import": "torch_tpu" }] + }, + "jax": { + "nix": ["jax"], + "python": [{ "pkg": "jax", "import": "jax" }] + }, + "libtpu": { + "nix": ["libtpu"], + "python": [{ "pkg": "libtpu" }] + } } } } diff --git a/kernels/tests/conftest.py b/kernels/tests/conftest.py index a9cb3c01c..513261ffd 100644 --- a/kernels/tests/conftest.py +++ b/kernels/tests/conftest.py @@ -34,6 +34,8 @@ has_npu = torch is not None and _get_torch_privateuse_backend_name() == "npu" +has_tpu = torch is not None and hasattr(torch, "tpu") and torch.tpu.device_count() > 0 + has_jax = importlib.util.find_spec("jax") is not None and importlib.util.find_spec("jax_tvm_ffi") is not None @@ -53,6 +55,8 @@ def device(): return "xpu" elif has_npu: return "npu" + elif has_tpu: + return "tpu" return "cpu" @@ -74,5 +78,7 @@ def pytest_runtest_setup(item): pytest.skip("skipping XPU-only test on host without XPU") if "npu_only" in item.keywords and not has_npu: pytest.skip("skipping NPU-only test on host without NPU") + if "tpu_only" in item.keywords and not has_tpu: + pytest.skip("skipping TPU-only test on host without TPU") if "token" in item.keywords and not item.config.getoption("--token"): pytest.skip("need --token option to run this test") diff --git a/kernels/tests/test_basic.py b/kernels/tests/test_basic.py index 1460f9454..5c9ce8dbd 100644 --- a/kernels/tests/test_basic.py +++ b/kernels/tests/test_basic.py @@ -229,6 +229,13 @@ def test_neuron(): torch.testing.assert_close(relu.relu(x), x.relu()) +@pytest.mark.tpu_only +def test_tpu(): + relu = get_kernel("kernels-test/relu-tpu", version=1) + x = torch.randn((16, 16), device="tpu") + torch.testing.assert_close(relu.relu(x), x.relu()) + + def test_trust_remote_code_blocks_untrusted_org(): """Kernels from untrusted orgs should be rejected by default.""" with pytest.raises(ValueError, match=r"not from a trusted publisher"): diff --git a/kernels/tests/test_layer.py b/kernels/tests/test_layer.py index e67abfc28..790e64458 100644 --- a/kernels/tests/test_layer.py +++ b/kernels/tests/test_layer.py @@ -255,6 +255,47 @@ def forward(self, x): assert smol.relu.n_calls == 1 +@pytest.mark.tpu_only +def test_hub_forward_tpu(): + torch.manual_seed(0) + + mapping = {"ReLU": {"tpu": LayerRepository(repo_id="kernels-test/relu-tpu", version=1, layer_name="ReLU")}} + + relu = ReLU() + X = torch.randn((16, 16), device="tpu") + Y = relu(X) + + with use_kernel_mapping(mapping): + relu_with_kernel = kernelize(ReLUWithKernel(), device="tpu", mode=Mode.INFERENCE) + Y_kernel = relu_with_kernel(X) + + torch.testing.assert_close(Y_kernel, Y) + + assert relu.n_calls == 1 + assert relu_with_kernel.n_calls == 0 + + # Check that the device type can be determined automatically. + class SMOL(nn.Module): + def __init__(self): + super().__init__() + self.linear = nn.Linear(16, 16) + self.relu = ReLUWithKernel() + + def forward(self, x): + return self.relu(self.linear(x)) + + smol = SMOL().to("tpu") + + Y = smol(X) + + with use_kernel_mapping(mapping): + smol = kernelize(smol, mode=Mode.INFERENCE) + Y_kernel = smol(X) + + torch.testing.assert_close(Y, Y_kernel) + assert smol.relu.n_calls == 1 + + @pytest.mark.rocm_only def test_hub_forward_rocm(): torch.manual_seed(0) diff --git a/kernels/tests/test_variants.py b/kernels/tests/test_variants.py index 008e2c8a2..1707c720e 100644 --- a/kernels/tests/test_variants.py +++ b/kernels/tests/test_variants.py @@ -74,6 +74,7 @@ "torch-metal", "torch-neuron", "torch-rocm", + "torch-tpu", "torch-xpu", "torch-npu", "torch-universal", diff --git a/nix-builder/README.md b/nix-builder/README.md index 7b31003ba..e59b7119e 100644 --- a/nix-builder/README.md +++ b/nix-builder/README.md @@ -54,8 +54,9 @@ The compiled kernel will then be available in the local `build/` directory. | Metal | ✓ | ✓ | ✗ | 2 | | Huawei NPU | ✓ | ✗ | ✗ | 3 | | Neuron | ✓ | x | x | 3 | +| TPU | ✓ | ✗ | ✗ | 3 | -**Warning:** Neuron support is experimental and currently requires pre-release packages. +**Warning:** Neuron and TPU support are experimental and packages are not public yet. # 📚 Documentation diff --git a/nix-builder/build-variants.json b/nix-builder/build-variants.json index 7fa073eba..da668552c 100644 --- a/nix-builder/build-variants.json +++ b/nix-builder/build-variants.json @@ -54,6 +54,9 @@ "torch213-cxx11-rocm71-x86_64-linux", "torch213-cxx11-rocm72-x86_64-linux" ], + "tpu": [ + "torch-tpu" + ], "xpu": [ "torch211-cxx11-xpu20253-x86_64-linux", "torch212-cxx11-xpu20253-x86_64-linux", diff --git a/nix-builder/lib/build-variants.nix b/nix-builder/lib/build-variants.nix index f7dfb7776..8948911f1 100644 --- a/nix-builder/lib/build-variants.nix +++ b/nix-builder/lib/build-variants.nix @@ -26,7 +26,10 @@ rec { throw "No compute framework set in Torch version"; in version: - if version.system == "aarch64-darwin" then + # TPU builds are noarch: one variant, independent of Torch version and system. + if backend version == "tpu" then + "torch-tpu" + else if version.system == "aarch64-darwin" then "torch${flattenVersion version.torchVersion}-${computeString version}-${version.system}" else "torch${flattenVersion version.torchVersion}-cxx11-${computeString version}-${version.system}"; diff --git a/nix-builder/lib/gen-flake-outputs.nix b/nix-builder/lib/gen-flake-outputs.nix index 99fba216e..9e553cb3d 100644 --- a/nix-builder/lib/gen-flake-outputs.nix +++ b/nix-builder/lib/gen-flake-outputs.nix @@ -58,6 +58,8 @@ let "xpu" else if buildConfig.metal or false then "metal" + else if buildConfig.tpu or false then + "tpu" else throw "Cannot determine framework for build set"; @@ -249,6 +251,7 @@ in ++ (headOrEmpty (setsWithFramework "cuda")) ++ (headOrEmpty (setsWithFramework "metal")) ++ (headOrEmpty (setsWithFramework "rocm")) + ++ (headOrEmpty (setsWithFramework "tpu")) ++ (headOrEmpty (setsWithFramework "xpu")); in build.mkExtensionBundle { diff --git a/nix-builder/lib/kernel-config.nix b/nix-builder/lib/kernel-config.nix index 0b274135f..4770404a6 100644 --- a/nix-builder/lib/kernel-config.nix +++ b/nix-builder/lib/kernel-config.nix @@ -48,6 +48,7 @@ in cuda = false; metal = false; rocm = false; + tpu = false; xpu = false; }; in @@ -63,6 +64,7 @@ in cuda = false; metal = false; rocm = false; + tpu = false; xpu = false; }; in diff --git a/nix-builder/lib/mk-build-set.nix b/nix-builder/lib/mk-build-set.nix index 920d51cc1..2b0355c63 100644 --- a/nix-builder/lib/mk-build-set.nix +++ b/nix-builder/lib/mk-build-set.nix @@ -82,6 +82,14 @@ let rocmSupport = true; }; + tpu = { + # torch_tpu is Apache-2.0, but libtpu's wheel METADATA declares + # its license as "Google Cloud Platform Terms of Service" + # (unfree), so the tpu buildSet needs allowUnfree just like the + # cuda/rocm/xpu sets. See pkgs/python-modules/libtpu/default.nix. + allowUnfree = true; + }; + xpu = { allowUnfree = true; xpuSupport = true; @@ -102,6 +110,7 @@ buildConfig@{ ptxasVersion ? cudaVersion, metal ? false, rocmVersion ? null, + tpu ? false, xpuVersion ? null, torchVersion, system, @@ -117,6 +126,8 @@ let [ (overlayForCudaVersion cudaVersion ptxasVersion) ] else if buildConfig.backend == "rocm" then [ (overlayForRocmVersion rocmVersion) ] + else if buildConfig.backend == "tpu" then + [ ] else if buildConfig.backend == "metal" then [ ] else if buildConfig.backend == "xpu" then diff --git a/nix-builder/lib/torch-version-utils.nix b/nix-builder/lib/torch-version-utils.nix index 5b43482cb..b7009de5f 100644 --- a/nix-builder/lib/torch-version-utils.nix +++ b/nix-builder/lib/torch-version-utils.nix @@ -4,6 +4,7 @@ let isCuda = version: version ? cudaVersion; isMetal = version: version.metal or false; isRocm = version: version ? rocmVersion; + isTpu = version: version.tpu or false; isXpu = version: version ? xpuVersion; in @@ -25,6 +26,8 @@ rec { "metal" else if isRocm version then "rocm" + else if isTpu version then + "tpu" else if isXpu version then "xpu" else diff --git a/nix-builder/overlay.nix b/nix-builder/overlay.nix index f94806512..34357e892 100644 --- a/nix-builder/overlay.nix +++ b/nix-builder/overlay.nix @@ -165,6 +165,10 @@ final: prev: jax-tvm-ffi = python-self.callPackage ./pkgs/python-modules/jax-tvm-ffi { }; + libtpu = python-self.callPackage ./pkgs/python-modules/libtpu { }; + + torch_tpu = python-self.callPackage ./pkgs/python-modules/torch_tpu { }; + jupyter-server = python-super.jupyter-server.overrideAttrs ( _: prevAttrs: { # Gets stuck sometimes, already tested in nixpkgs. diff --git a/nix-builder/pkgs/python-modules/libtpu/default.nix b/nix-builder/pkgs/python-modules/libtpu/default.nix new file mode 100644 index 000000000..53cc24f01 --- /dev/null +++ b/nix-builder/pkgs/python-modules/libtpu/default.nix @@ -0,0 +1,66 @@ +{ + lib, + buildPythonPackage, + fetchurl, + python, +}: + +# Fetched directly from Google's Artifact Registry, which requires an +# OAuth2 bearer token. The token is passed at fetch time through the +# GCLOUD_ACCESS_TOKEN environment variable and turned into a netrc +# entry, so it never appears in the URL or the .drv: +# +# export GCLOUD_ACCESS_TOKEN=$(gcloud auth print-access-token) +# +# (With a multi-user Nix daemon the variable must be visible to the +# daemon, not the client shell.) +# +# libtpu ships the `libtpu/libtpu.so` runtime that both torch_tpu and +# jaxlib dlopen. Pinned to 0.0.x to match the jax 0.10.x / torch_tpu +# 0.1.x stack (torch_tpu requires libtpu>=0.0.40; jax's "tpu" extra +# pins libtpu==0.0.43.*), so bump it together with jaxlib/torch_tpu. +# +# NOTE on license: libtpu's wheel METADATA declares its license as +# "Google Cloud Platform Terms of Service" — i.e. unfree, NOT +# Apache-2.0 (unlike torch_tpu). The tpu buildSet therefore sets +# allowUnfree = true (see lib/mk-build-set.nix). + +let + # The wheel ships per-CPython-ABI builds (cp311..cp314); pick the tag + # matching the python this package set is built for. The hash below is + # for cp313 (the nixpkgs default python); re-run + # scripts/helpers/get_torch_tpu_hash.sh when either moves. + abi = "cp${lib.versions.major python.version}${lib.versions.minor python.version}"; +in +buildPythonPackage rec { + pname = "libtpu"; + version = "0.0.43"; + format = "wheel"; + + src = fetchurl { + url = "https://us-python.pkg.dev/ml-oss-artifacts-transient/torch-tpu-virtual-registry/libtpu/libtpu-${version}-${abi}-${abi}-manylinux_2_31_x86_64.whl"; + hash = "sha256-X5LVmwJuRcNkMG+m7kKgcaOSKnWjeuw4HtCU+lfWamY="; # cp313 + netrcImpureEnvVars = [ "GCLOUD_ACCESS_TOKEN" ]; + netrcPhase = '' + if [ -z "''${GCLOUD_ACCESS_TOKEN:-}" ]; then + echo "GCLOUD_ACCESS_TOKEN is not set; cannot fetch libtpu." >&2 + echo "Run: export GCLOUD_ACCESS_TOKEN=\$(gcloud auth print-access-token)" >&2 + exit 1 + fi + printf 'machine us-python.pkg.dev\nlogin oauth2accesstoken\npassword %s\n' \ + "$GCLOUD_ACCESS_TOKEN" > netrc + ''; + }; + + dependencies = [ ]; + + pythonImportsCheck = [ "libtpu" ]; + doInstallCheck = false; # requires actual /dev/accel + + meta = with lib; { + description = "TPU runtime shared library, dlopened by torch_tpu and jaxlib"; + homepage = "https://cloud.google.com/tpu"; + license = licenses.unfree; # "Google Cloud Platform Terms of Service" + platforms = [ "x86_64-linux" ]; + }; +} diff --git a/nix-builder/pkgs/python-modules/torch_tpu/default.nix b/nix-builder/pkgs/python-modules/torch_tpu/default.nix new file mode 100644 index 000000000..caa24ec62 --- /dev/null +++ b/nix-builder/pkgs/python-modules/torch_tpu/default.nix @@ -0,0 +1,93 @@ +{ + lib, + autoPatchelfHook, + buildPythonPackage, + fetchurl, + python, + stdenv, + + # jax is not a hard dependency of the torch_tpu wheel, but TPU kernels + # import torch_tpu._internal.pallas (jax_op), which needs jax at import + # time, so it is listed here. jaxlib is not a torch_tpu dependency; it + # is pulled in transitively by jax (nixpkgs' jax declares it). + # libtpu is pinned in-tree via pkgs/python-modules/libtpu. + jax, + libtpu, + + # The remaining wheel Requires-Dist entries (numpy and absl-py come in + # transitively through jax). + frozendict, + immutabledict, + portpicker, + tensorboard, + torch, +}: + +# Fetched directly from Google's Artifact Registry, which requires an +# OAuth2 bearer token. The token is passed at fetch time through the +# GCLOUD_ACCESS_TOKEN environment variable and turned into a netrc +# entry, so it never appears in the URL or the .drv: +# +# export GCLOUD_ACCESS_TOKEN=$(gcloud auth print-access-token) +# +# (With a multi-user Nix daemon the variable must be visible to the +# daemon, not the client shell.) The dev build is dated; bump the date +# suffix in `version` and re-run +# scripts/helpers/get_torch_tpu_hash.sh when refreshing. + +let + # The wheel ships per-CPython-ABI builds (cp311..cp314); pick the tag + # matching the python this package set is built for. The hash below is + # for cp313 (the nixpkgs default python); re-run + # scripts/helpers/get_torch_tpu_hash.sh when either moves. + abi = "cp${lib.versions.major python.version}${lib.versions.minor python.version}"; +in +buildPythonPackage rec { + pname = "torch_tpu"; + version = "0.1.1.dev20260707090224"; + format = "wheel"; + + src = fetchurl { + url = "https://us-python.pkg.dev/ml-oss-artifacts-transient/torch-tpu-virtual-registry/torch-tpu/torch_tpu-${version}-${abi}-${abi}-manylinux_2_31_x86_64.whl"; + hash = "sha256-eCjwoX0UKd/L/IccxXn88p0GdyQjFdPl2Er1luZz4H0="; # cp313 + netrcImpureEnvVars = [ "GCLOUD_ACCESS_TOKEN" ]; + netrcPhase = '' + if [ -z "''${GCLOUD_ACCESS_TOKEN:-}" ]; then + echo "GCLOUD_ACCESS_TOKEN is not set; cannot fetch torch_tpu." >&2 + echo "Run: export GCLOUD_ACCESS_TOKEN=\$(gcloud auth print-access-token)" >&2 + exit 1 + fi + printf 'machine us-python.pkg.dev\nlogin oauth2accesstoken\npassword %s\n' \ + "$GCLOUD_ACCESS_TOKEN" > netrc + ''; + }; + + dependencies = [ + libtpu + jax + frozendict + immutabledict + portpicker + tensorboard + torch + ]; + + # The bundled extensions link against libtorch_python.so / libc10.so / + # libtorch_cpu.so, which live in torch's site-packages, not on the + # default search path. + nativeBuildInputs = [ autoPatchelfHook ]; + buildInputs = [ stdenv.cc.cc.lib ]; + preFixup = '' + addAutoPatchelfSearchPath ${torch}/${python.sitePackages}/torch/lib + ''; + + pythonImportsCheck = [ "torch_tpu" ]; + doInstallCheck = false; # requires actual /dev/accel + + meta = with lib; { + description = "Torch TPU backend (PrivateUse1 name: \"tpu\")"; + homepage = "https://github.com/google-pytorch/torch_tpu"; + license = licenses.asl20; + platforms = [ "x86_64-linux" ]; + }; +} diff --git a/nix-builder/scripts/gen_variants_markdown.py b/nix-builder/scripts/gen_variants_markdown.py index b097cebcc..f528144ce 100755 --- a/nix-builder/scripts/gen_variants_markdown.py +++ b/nix-builder/scripts/gen_variants_markdown.py @@ -8,6 +8,7 @@ "cuda": "CUDA", "metal": "Metal", "rocm": "ROCm", + "tpu": "TPU", "xpu": "XPU", } diff --git a/nix-builder/scripts/helpers/get_torch_tpu_hash.sh b/nix-builder/scripts/helpers/get_torch_tpu_hash.sh new file mode 100755 index 000000000..ee3f96865 --- /dev/null +++ b/nix-builder/scripts/helpers/get_torch_tpu_hash.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Print the wheel URL and SRI hash for a torch-tpu-virtual-registry +# package, so a developer can paste the result straight into +# pkgs/python-modules/{torch_tpu,libtpu}/default.nix. +# +# The registry's simple index publishes each wheel's sha256 as a URL +# fragment, so no download is needed to compute the hash. +# +# Usage: +# bash scripts/helpers/get_torch_tpu_hash.sh torch_tpu 0.1.1.dev20260707090224 +# bash scripts/helpers/get_torch_tpu_hash.sh libtpu 0.0.43 cp313 +# +# The third argument is the CPython ABI tag (default cp313, matching the +# nixpkgs default python used by the builder). +# +# Requires `gcloud auth login` once, plus curl and python3. +set -euo pipefail +pkg="${1:?usage: $0 [abi], e.g. $0 torch_tpu 0.1.1.dev20260707090224}" +ver="${2:?usage: $0 [abi], e.g. $0 torch_tpu 0.1.1.dev20260707090224}" +abi="${3:-cp313}" +token="$(gcloud auth print-access-token)" +index="https://us-python.pkg.dev/ml-oss-artifacts-transient/torch-tpu-virtual-registry/simple/${pkg//_/-}/" +index_html="$(curl -sSf -u "oauth2accesstoken:${token}" "${index}")" +href="$(grep -o 'href="[^"]*"' <<<"${index_html}" \ + | grep -F "/${pkg}-${ver}-${abi}-" \ + | sed 's/^href="//; s/"$//' \ + | head -1 || true)" +if [ -z "${href}" ]; then + echo "No ${abi} wheel for ${pkg}==${ver} in ${index}" >&2 + exit 1 +fi +echo "URL: ${href%%#*}" +echo "Hash: $(python3 -c \ + "import base64,sys; print('sha256-' + base64.b64encode(bytes.fromhex(sys.argv[1])).decode())" \ + "${href##*#sha256=}")" diff --git a/nix-builder/versions.nix b/nix-builder/versions.nix index 1618d26e1..3eb871387 100644 --- a/nix-builder/versions.nix +++ b/nix-builder/versions.nix @@ -58,6 +58,12 @@ systems = [ "x86_64-linux" ]; bundleBuild = true; } + { + torchVersion = "2.11"; + tpu = true; + systems = [ "x86_64-linux" ]; + bundleBuild = true; + } { torchVersion = "2.11"; xpuVersion = "2025.3.2";