From 50b6a7d69d894c87e85bc70d16de8f0dff079221 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 13:19:18 -0700 Subject: [PATCH 01/41] feat(py): add warmup() to RustToolchain for shared build layer --- dsls/harmont-py/harmont/rust.py | 11 +++++++ dsls/harmont-py/tests/test_rust.py | 48 ++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/dsls/harmont-py/harmont/rust.py b/dsls/harmont-py/harmont/rust.py index b7189522..5cf9a145 100644 --- a/dsls/harmont-py/harmont/rust.py +++ b/dsls/harmont-py/harmont/rust.py @@ -73,6 +73,13 @@ def clippy(self, **kw: Any) -> Step: def fmt(self, **kw: Any) -> Step: return self._emit("cargo fmt --check", ":rust: fmt", **kw) + def warmup(self, **kw: Any) -> Step: + return self._emit( + "cargo build --workspace --tests --locked", + ":rust: warmup", + **kw, + ) + def doc(self, **kw: Any) -> Step: return self._emit("cargo doc --no-deps", ":rust: doc", **kw) @@ -140,6 +147,10 @@ def fmt(self, **kw: Any) -> Step: action_kw = {k: kw.pop(k) for k in list(kw) if k in _ACTION_KWARGS} return self(**kw).fmt(**action_kw) + def warmup(self, **kw: Any) -> Step: + action_kw = {k: kw.pop(k) for k in list(kw) if k in _ACTION_KWARGS} + return self(**kw).warmup(**action_kw) + def doc(self, **kw: Any) -> Step: action_kw = {k: kw.pop(k) for k in list(kw) if k in _ACTION_KWARGS} return self(**kw).doc(**action_kw) diff --git a/dsls/harmont-py/tests/test_rust.py b/dsls/harmont-py/tests/test_rust.py index 99e8334d..60cb6023 100644 --- a/dsls/harmont-py/tests/test_rust.py +++ b/dsls/harmont-py/tests/test_rust.py @@ -174,3 +174,51 @@ def test_rust_bare_form_accepts_path_kwarg(): def test_rust_bare_form_forwards_action_kwargs(): s = hm.rust.build(path="cli", label=":rust: custom") assert s.label == ":rust: custom" + + +def test_rust_warmup_returns_step(): + rust = hm.rust(path="cli") + w = rust.warmup() + assert w.cmd is not None + assert "cargo build" in w.cmd + assert "--workspace" in w.cmd + assert "--tests" in w.cmd + assert "--locked" in w.cmd + + +def test_rust_warmup_chains_from_installed(): + rust = hm.rust(path="cli") + w = rust.warmup() + assert w.parent is rust.installed + + +def test_rust_warmup_default_label(): + rust = hm.rust(path=".") + assert rust.warmup().label == ":rust: warmup" + + +def test_rust_warmup_label_override(): + rust = hm.rust(path=".") + assert rust.warmup(label=":rust: pre-build").label == ":rust: pre-build" + + +def test_rust_warmup_in_pipeline(): + """warmup step appears in pipeline IR.""" + rust = hm.rust(path="cli") + w = rust.warmup() + t = w.sh( + ". $HOME/.cargo/env && cd cli && cargo test --workspace --locked", label=":rust: test" + ) + p = hm.pipeline(t, rust.fmt(), default_image="ubuntu:24.04") + cmds = _cmds(p) + assert any("cargo build --workspace --tests --locked" in c for c in cmds) + assert any("cargo test --workspace --locked" in c for c in cmds) + assert any("cargo fmt" in c for c in cmds) + assert len([c for c in cmds if "sh.rustup.rs" in c]) == 1 + assert len([c for c in cmds if "apt-get install" in c]) == 1 + + +def test_rust_bare_form_warmup(): + p = hm.pipeline(hm.rust.warmup()) + cmds = _cmds(p) + assert any("cargo build --workspace --tests --locked" in c for c in cmds) From b80b54ed29dfeec485e60aae4327798703e895e5 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 13:20:08 -0700 Subject: [PATCH 02/41] feat(ts): add warmup() to RustToolchain for shared build layer --- dsls/harmont-ts/src/toolchains/rust.ts | 8 +++++++ dsls/harmont-ts/tests/toolchains/rust.test.ts | 23 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/dsls/harmont-ts/src/toolchains/rust.ts b/dsls/harmont-ts/src/toolchains/rust.ts index 2ee5b596..56f6e972 100644 --- a/dsls/harmont-ts/src/toolchains/rust.ts +++ b/dsls/harmont-ts/src/toolchains/rust.ts @@ -66,6 +66,14 @@ export class RustToolchain { doc(opts?: ActionOptions): Step { return this._cargo("cargo doc --no-deps", ":rust: doc", opts); } + + warmup(opts?: ActionOptions): Step { + return this._cargo( + "cargo build --workspace --tests --locked", + ":rust: warmup", + opts, + ); + } } export function rust(opts?: RustOptions): RustToolchain { diff --git a/dsls/harmont-ts/tests/toolchains/rust.test.ts b/dsls/harmont-ts/tests/toolchains/rust.test.ts index 6f3546b5..6f929feb 100644 --- a/dsls/harmont-ts/tests/toolchains/rust.test.ts +++ b/dsls/harmont-ts/tests/toolchains/rust.test.ts @@ -83,6 +83,29 @@ describe("rust actions", () => { expect(r.fmt()._label).toBe(":rust: fmt"); expect(r.doc()._label).toBe(":rust: doc"); }); + + it("warmup runs cargo build --workspace --tests --locked", () => { + const r = rust(); + expect(r.warmup()._cmd).toContain( + "cargo build --workspace --tests --locked", + ); + }); + + it("warmup chains from install", () => { + const r = rust(); + expect(r.warmup()._parent).toBe(r.install()); + }); + + it("warmup default label", () => { + const r = rust(); + expect(r.warmup()._label).toBe(":rust: warmup"); + }); + + it("warmup accepts options", () => { + const r = rust(); + const w = r.warmup({ label: ":rust: pre-build" }); + expect(w._label).toBe(":rust: pre-build"); + }); }); describe("rust install chain", () => { From 2674b23828d8b9b7a392f7a0be49df06a3579463 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 13:21:26 -0700 Subject: [PATCH 03/41] feat(ts): add aptBase() factory for cross-toolchain apt sharing --- dsls/harmont-ts/src/index.ts | 1 + dsls/harmont-ts/src/toolchains/shared.ts | 11 +++++ .../tests/toolchains/shared.test.ts | 46 +++++++++++++++++++ 3 files changed, 58 insertions(+) create mode 100644 dsls/harmont-ts/tests/toolchains/shared.test.ts diff --git a/dsls/harmont-ts/src/index.ts b/dsls/harmont-ts/src/index.ts index 26987f16..65bf0c1b 100644 --- a/dsls/harmont-ts/src/index.ts +++ b/dsls/harmont-ts/src/index.ts @@ -21,4 +21,5 @@ export { } from "./triggers.js"; export { pipeline, type PipelineIR, type PipelineOptions } from "./pipeline.js"; export { target, clearTargetCache } from "./target.js"; +export { aptBase } from "./toolchains/shared.js"; export { renderEnvelope, type PipelineDefinition } from "./envelope.js"; diff --git a/dsls/harmont-ts/src/toolchains/shared.ts b/dsls/harmont-ts/src/toolchains/shared.ts index f7a3da38..ca0922d2 100644 --- a/dsls/harmont-ts/src/toolchains/shared.ts +++ b/dsls/harmont-ts/src/toolchains/shared.ts @@ -12,6 +12,17 @@ export function nodeInstallCmd(version: string): string { return `curl -fsSL https://deb.nodesource.com/setup_${major}.x | bash - && apt-get install -y nodejs`; } +export function aptBase(opts: { + packages: readonly string[]; + image?: string; + label?: string; +}): Step { + return scratch({ image: opts.image }).sh(aptInstallCmd(opts.packages), { + label: opts.label ?? ":apt: base", + cache: ttl(APT_TTL_SECONDS), + }); +} + export function makeInstallChain(opts: { aptPackages: readonly string[]; installCmd: string; diff --git a/dsls/harmont-ts/tests/toolchains/shared.test.ts b/dsls/harmont-ts/tests/toolchains/shared.test.ts new file mode 100644 index 00000000..fa9dd936 --- /dev/null +++ b/dsls/harmont-ts/tests/toolchains/shared.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { aptBase } from "../../src/toolchains/shared.js"; +import { rust } from "../../src/toolchains/rust.js"; +import { uv } from "../../src/toolchains/py/uv.js"; +import { pipeline } from "../../src/pipeline.js"; + +describe("aptBase", () => { + it("creates a step with apt-get install", () => { + const base = aptBase({ packages: ["curl", "ca-certificates"] }); + expect(base._cmd).toContain( + "apt-get update && apt-get install -y curl ca-certificates", + ); + }); + + it("default label is :apt: base", () => { + const base = aptBase({ packages: ["curl"] }); + expect(base._label).toBe(":apt: base"); + }); + + it("accepts custom label", () => { + const base = aptBase({ packages: ["curl"], label: ":lock: deps" }); + expect(base._label).toBe(":lock: deps"); + }); + + it("shared across rust and python toolchains", () => { + const base = aptBase({ + packages: [ + "curl", + "ca-certificates", + "build-essential", + "pkg-config", + "libssl-dev", + "python3", + "python3-venv", + ], + }); + const r = rust({ base }); + const p = uv({ path: "dsls/harmont-py", base }); + const ir = pipeline(r.build(), p.test(), { defaultImage: "ubuntu:24.04" }); + const cmds = ir.graph.nodes.map( + (n: { step: { cmd: string } }) => n.step.cmd, + ); + const aptSteps = cmds.filter((c: string) => c.includes("apt-get install")); + expect(aptSteps).toHaveLength(1); + }); +}); From 53d7e75885dbd53224a5b4fafb34c20117ae6526 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 13:21:44 -0700 Subject: [PATCH 04/41] feat(py): add apt_base() factory for cross-toolchain apt sharing --- dsls/harmont-py/harmont/__init__.py | 2 + dsls/harmont-py/harmont/_toolchain.py | 15 ++++++ .../tests/test_toolchain_compose.py | 52 +++++++++++++++++++ 3 files changed, 69 insertions(+) diff --git a/dsls/harmont-py/harmont/__init__.py b/dsls/harmont-py/harmont/__init__.py index 829b8706..8952b83d 100644 --- a/dsls/harmont-py/harmont/__init__.py +++ b/dsls/harmont-py/harmont/__init__.py @@ -34,6 +34,7 @@ from ._envelope import dump_registry_json from ._step import Step, scratch, wait from ._target import clear_target_cache, target # noqa: F401 clear_target_cache used by tests +from ._toolchain import apt_base from ._typing import BaseImage, Dep, Target from .cache import ( CacheCompose, @@ -140,6 +141,7 @@ def sh( "Pipeline", "Step", "Target", + "apt_base", "cmake", "compose", "composer", diff --git a/dsls/harmont-py/harmont/_toolchain.py b/dsls/harmont-py/harmont/_toolchain.py index fb67b6af..5bd3c305 100644 --- a/dsls/harmont-py/harmont/_toolchain.py +++ b/dsls/harmont-py/harmont/_toolchain.py @@ -77,3 +77,18 @@ def make_install_chain( label=f":{lang_tag}: {install_tag}", cache=install_cache, ) + + +def apt_base( + *, + packages: tuple[str, ...], + image: str | None = None, + label: str = ":apt: base", +) -> Step: + """Create a standalone apt-base step sharable across toolchains via ``base=``.""" + return scratch().sh( + apt_install_cmd(packages), + label=label, + image=image, + cache=CacheTTL(duration=APT_TTL), + ) diff --git a/dsls/harmont-py/tests/test_toolchain_compose.py b/dsls/harmont-py/tests/test_toolchain_compose.py index 14eefa7a..bdafca47 100644 --- a/dsls/harmont-py/tests/test_toolchain_compose.py +++ b/dsls/harmont-py/tests/test_toolchain_compose.py @@ -82,3 +82,55 @@ def test_mixed_pipeline_compiles(): ) assert p["version"] == "0" assert len(p["graph"]["nodes"]) > 0 + + +def _step_by_substring(p: dict, needle: str) -> dict: + for n in p["graph"]["nodes"]: + if needle in (n["step"].get("cmd") or ""): + return n["step"] + msg = f"no command step containing {needle!r}" + raise AssertionError(msg) + + +def test_apt_base_shared_across_toolchains(): + """Single apt-base feeds both rust and python toolchains.""" + base = hm.apt_base( + packages=( + "curl", + "ca-certificates", + "build-essential", + "pkg-config", + "libssl-dev", + "python3", + "python3-venv", + ), + ) + rust = hm.rust(path=".", base=base) + py = hm.py.uv(path="dsls/harmont-py", base=base) + p = hm.pipeline( + rust.build(), + py.test(), + default_image="ubuntu:24.04", + ) + cmds = _cmds(p) + assert len([c for c in cmds if "apt-get install" in c]) == 1 + assert any("sh.rustup.rs" in c for c in cmds) + assert any("uv" in c for c in cmds) + + +def test_apt_base_default_label(): + base = hm.apt_base(packages=("curl",)) + assert base.label == ":apt: base" + + +def test_apt_base_custom_image(): + base = hm.apt_base(packages=("curl",), image="debian:bookworm") + rust = hm.rust(path=".", base=base) + p = hm.pipeline(rust.build(), default_image="ubuntu:24.04") + apt_step = _step_by_substring(p, "apt-get install") + assert apt_step.get("image") == "debian:bookworm" + + +def test_apt_base_custom_label(): + base = hm.apt_base(packages=("curl",), label=":lock: deps") + assert base.label == ":lock: deps" From ae1f30ec23b302c07b2dc6a678da90c1e488c8b6 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 13:22:39 -0700 Subject: [PATCH 05/41] perf: dogfood pipeline uses shared apt-base + warmup for faster CI - Single apt-get call shared across Rust and Python toolchains - warmup() pre-compiles workspace; test/clippy reuse artifacts - fmt stays off installed for parallel execution --- .harmont/ci.py | 34 ++++++++++++++++++++++++++-------- .harmont/ci.ts | 36 ++++++++++++++++++++++++++++++------ 2 files changed, 56 insertions(+), 14 deletions(-) diff --git a/.harmont/ci.py b/.harmont/ci.py index a1afc663..97741608 100644 --- a/.harmont/ci.py +++ b/.harmont/ci.py @@ -5,15 +5,30 @@ from harmont.py.uv import UvProject from harmont.rust import RustToolchain +ALL_APT = ( + "curl", + "ca-certificates", + "build-essential", + "pkg-config", + "libssl-dev", + "python3", + "python3-venv", +) + + +@hm.target() +def shared_base() -> hm.Step: + return hm.apt_base(packages=ALL_APT) + @hm.target() -def rust_project() -> RustToolchain: - return hm.rust(path=".") +def rust_project(shared_base: hm.Target[hm.Step]) -> RustToolchain: + return hm.rust(path=".", base=shared_base) @hm.target() -def py_project() -> UvProject: - return hm.py.uv(path="dsls/harmont-py") +def py_project(shared_base: hm.Target[hm.Step]) -> UvProject: + return hm.py.uv(path="dsls/harmont-py", base=shared_base) @hm.pipeline( @@ -29,13 +44,16 @@ def ci( rust_project: hm.Target[RustToolchain], py_project: hm.Target[UvProject], ) -> tuple[hm.Step, ...]: + warm = rust_project.warmup() return ( - rust_project.build(), - rust_project.installed.sh( - ". $HOME/.cargo/env && cd . && cargo test --lib", + warm.sh( + ". $HOME/.cargo/env && cd . && cargo test --workspace --locked --no-fail-fast", label=":rust: test", ), - rust_project.clippy(), + warm.sh( + ". $HOME/.cargo/env && cd . && cargo clippy --workspace --tests --locked -- -D warnings", + label=":rust: clippy", + ), rust_project.fmt(), py_project.lint(), py_project.fmt(), diff --git a/.harmont/ci.ts b/.harmont/ci.ts index 308083e5..0f7ab100 100644 --- a/.harmont/ci.ts +++ b/.harmont/ci.ts @@ -1,17 +1,41 @@ -import { pipeline, push, pullRequest, type PipelineDefinition } from "harmont"; +import { + pipeline, + push, + pullRequest, + aptBase, + type PipelineDefinition, +} from "harmont"; import { rust, py } from "harmont/toolchains"; -const rustProject = rust({ path: "." }); -const pyProject = py.uv({ path: "dsls/harmont-py" }); +const ALL_APT = [ + "curl", + "ca-certificates", + "build-essential", + "pkg-config", + "libssl-dev", + "python3", + "python3-venv", +] as const; + +const base = aptBase({ packages: ALL_APT }); +const rustProject = rust({ path: ".", base }); +const pyProject = py.uv({ path: "dsls/harmont-py", base }); + +const warm = rustProject.warmup(); const pipelines: PipelineDefinition[] = [ { slug: "ci", triggers: [push({ branch: "main" }), pullRequest({ branches: ["main"] })], pipeline: pipeline( - rustProject.build(), - rustProject.install().sh(`. $HOME/.cargo/env && cd . && cargo test --lib`, { label: ":rust: test" }), - rustProject.clippy(), + warm.sh( + `. $HOME/.cargo/env && cd . && cargo test --workspace --locked --no-fail-fast`, + { label: ":rust: test" }, + ), + warm.sh( + `. $HOME/.cargo/env && cd . && cargo clippy --workspace --tests --locked -- -D warnings`, + { label: ":rust: clippy" }, + ), rustProject.fmt(), pyProject.lint(), pyProject.fmt(), From a120cd2ff6e20f20de527367091488c424920584 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 13:27:58 -0700 Subject: [PATCH 06/41] fix: use --lib for cargo test in dogfood (no python3 in Rust container) --- .harmont/ci.py | 2 +- .harmont/ci.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.harmont/ci.py b/.harmont/ci.py index 97741608..3bd79d64 100644 --- a/.harmont/ci.py +++ b/.harmont/ci.py @@ -47,7 +47,7 @@ def ci( warm = rust_project.warmup() return ( warm.sh( - ". $HOME/.cargo/env && cd . && cargo test --workspace --locked --no-fail-fast", + ". $HOME/.cargo/env && cd . && cargo test --workspace --lib --locked --no-fail-fast", label=":rust: test", ), warm.sh( diff --git a/.harmont/ci.ts b/.harmont/ci.ts index 0f7ab100..ae65d56f 100644 --- a/.harmont/ci.ts +++ b/.harmont/ci.ts @@ -29,7 +29,7 @@ const pipelines: PipelineDefinition[] = [ triggers: [push({ branch: "main" }), pullRequest({ branches: ["main"] })], pipeline: pipeline( warm.sh( - `. $HOME/.cargo/env && cd . && cargo test --workspace --locked --no-fail-fast`, + `. $HOME/.cargo/env && cd . && cargo test --workspace --lib --locked --no-fail-fast`, { label: ":rust: test" }, ), warm.sh( From ff98d340f524caed33a41fd5a5e506ab5ca88c86 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 13:32:54 -0700 Subject: [PATCH 07/41] fix: install harmont-py in Rust container for full test suite Add python3-pip to shared apt-base and pip install harmont-py before cargo test so integration tests can import harmont. --- .harmont/ci.py | 4 +++- .harmont/ci.ts | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.harmont/ci.py b/.harmont/ci.py index 3bd79d64..2b41d281 100644 --- a/.harmont/ci.py +++ b/.harmont/ci.py @@ -12,6 +12,7 @@ "pkg-config", "libssl-dev", "python3", + "python3-pip", "python3-venv", ) @@ -47,7 +48,8 @@ def ci( warm = rust_project.warmup() return ( warm.sh( - ". $HOME/.cargo/env && cd . && cargo test --workspace --lib --locked --no-fail-fast", + "python3 -m pip install --break-system-packages dsls/harmont-py" + " && . $HOME/.cargo/env && cd . && cargo test --workspace --locked --no-fail-fast", label=":rust: test", ), warm.sh( diff --git a/.harmont/ci.ts b/.harmont/ci.ts index ae65d56f..1a8ad289 100644 --- a/.harmont/ci.ts +++ b/.harmont/ci.ts @@ -14,6 +14,7 @@ const ALL_APT = [ "pkg-config", "libssl-dev", "python3", + "python3-pip", "python3-venv", ] as const; @@ -29,7 +30,7 @@ const pipelines: PipelineDefinition[] = [ triggers: [push({ branch: "main" }), pullRequest({ branches: ["main"] })], pipeline: pipeline( warm.sh( - `. $HOME/.cargo/env && cd . && cargo test --workspace --lib --locked --no-fail-fast`, + `python3 -m pip install --break-system-packages dsls/harmont-py && . $HOME/.cargo/env && cd . && cargo test --workspace --locked --no-fail-fast`, { label: ":rust: test" }, ), warm.sh( From b8b67bab7217c41733c4d4422ad7f14fa3615237 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 13:36:57 -0700 Subject: [PATCH 08/41] fix: revert to --lib for cargo test (integration tests need Docker socket) --- .harmont/ci.py | 4 +--- .harmont/ci.ts | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.harmont/ci.py b/.harmont/ci.py index 2b41d281..3bd79d64 100644 --- a/.harmont/ci.py +++ b/.harmont/ci.py @@ -12,7 +12,6 @@ "pkg-config", "libssl-dev", "python3", - "python3-pip", "python3-venv", ) @@ -48,8 +47,7 @@ def ci( warm = rust_project.warmup() return ( warm.sh( - "python3 -m pip install --break-system-packages dsls/harmont-py" - " && . $HOME/.cargo/env && cd . && cargo test --workspace --locked --no-fail-fast", + ". $HOME/.cargo/env && cd . && cargo test --workspace --lib --locked --no-fail-fast", label=":rust: test", ), warm.sh( diff --git a/.harmont/ci.ts b/.harmont/ci.ts index 1a8ad289..ae65d56f 100644 --- a/.harmont/ci.ts +++ b/.harmont/ci.ts @@ -14,7 +14,6 @@ const ALL_APT = [ "pkg-config", "libssl-dev", "python3", - "python3-pip", "python3-venv", ] as const; @@ -30,7 +29,7 @@ const pipelines: PipelineDefinition[] = [ triggers: [push({ branch: "main" }), pullRequest({ branches: ["main"] })], pipeline: pipeline( warm.sh( - `python3 -m pip install --break-system-packages dsls/harmont-py && . $HOME/.cargo/env && cd . && cargo test --workspace --locked --no-fail-fast`, + `. $HOME/.cargo/env && cd . && cargo test --workspace --lib --locked --no-fail-fast`, { label: ":rust: test" }, ), warm.sh( From b45782b1645779f592d0d676e66d2c15ae3a1bfa Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 14:08:53 -0700 Subject: [PATCH 09/41] perf: cache warmup layer on Cargo.lock (skip rebuild when deps unchanged) --- .harmont/ci.py | 3 ++- .harmont/ci.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.harmont/ci.py b/.harmont/ci.py index 3bd79d64..dbbd6d70 100644 --- a/.harmont/ci.py +++ b/.harmont/ci.py @@ -2,6 +2,7 @@ from __future__ import annotations import harmont as hm +from harmont.cache import CacheOnChange from harmont.py.uv import UvProject from harmont.rust import RustToolchain @@ -44,7 +45,7 @@ def ci( rust_project: hm.Target[RustToolchain], py_project: hm.Target[UvProject], ) -> tuple[hm.Step, ...]: - warm = rust_project.warmup() + warm = rust_project.warmup(cache=CacheOnChange(paths=("Cargo.lock",))) return ( warm.sh( ". $HOME/.cargo/env && cd . && cargo test --workspace --lib --locked --no-fail-fast", diff --git a/.harmont/ci.ts b/.harmont/ci.ts index ae65d56f..f332acba 100644 --- a/.harmont/ci.ts +++ b/.harmont/ci.ts @@ -3,6 +3,7 @@ import { push, pullRequest, aptBase, + onChange, type PipelineDefinition, } from "harmont"; import { rust, py } from "harmont/toolchains"; @@ -21,7 +22,7 @@ const base = aptBase({ packages: ALL_APT }); const rustProject = rust({ path: ".", base }); const pyProject = py.uv({ path: "dsls/harmont-py", base }); -const warm = rustProject.warmup(); +const warm = rustProject.warmup({ cache: onChange("Cargo.lock") }); const pipelines: PipelineDefinition[] = [ { From 15ccaff0f0919dfdcf098bbd7fe6038ec1d0ff4b Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 14:08:56 -0700 Subject: [PATCH 10/41] perf: persist harmont Docker cache across GHA runs via actions/cache --- .github/workflows/ci.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de93bb10..8c7c68dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -112,11 +112,32 @@ jobs: sudo /usr/bin/python3 -m pip install --break-system-packages dsls/harmont-py /usr/bin/python3 -c "import harmont; print('harmont', harmont.__file__)" + - name: Restore harmont Docker cache + id: docker-cache + uses: actions/cache@v4 + with: + path: /tmp/harmont-docker-cache.tar + key: harmont-docker-${{ hashFiles('Cargo.lock') }} + restore-keys: | + harmont-docker- + + - name: Load cached Docker images + if: steps.docker-cache.outputs.cache-hit == 'true' + run: docker load -i /tmp/harmont-docker-cache.tar || true + - name: hm run ci env: HM_NONINTERACTIVE: '1' run: ./target/debug/hm run ci + - name: Save harmont Docker images + if: always() + run: | + images=$(docker images --format '{{.Repository}}:{{.Tag}}' | grep '^harmont-local/' || true) + if [ -n "$images" ]; then + docker save $images -o /tmp/harmont-docker-cache.tar + fi + integration: name: docker-gated integration test runs-on: ubuntu-latest From ac4b24010f3a254d50d1cf060895e838ea95b457 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 14:13:37 -0700 Subject: [PATCH 11/41] ci: trigger cache-warm run From 4f30d29cba78ee6d207a68a14b2a112b1764ae50 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 15:38:09 -0700 Subject: [PATCH 12/41] feat(py): add RustProject high-level abstraction with rust.project()/rust.toolchain() --- dsls/harmont-py/harmont/rust.py | 131 ++++++++++++++++++++++++-------- 1 file changed, 99 insertions(+), 32 deletions(-) diff --git a/dsls/harmont-py/harmont/rust.py b/dsls/harmont-py/harmont/rust.py index 5cf9a145..f4167bf0 100644 --- a/dsls/harmont-py/harmont/rust.py +++ b/dsls/harmont-py/harmont/rust.py @@ -1,8 +1,9 @@ """Rust toolchain abstraction (HAR-15). -Public surface lives on the module-level singleton :data:`rust`. Call it -to construct a :class:`RustToolchain`, or use the bare-form action -methods (``rust.build()``, ``rust.test()``, etc.) for a one-shot leaf. +Public surface lives on the module-level singleton :data:`rust`: + + hm.rust.toolchain(...) -> RustToolchain (install-only) + hm.rust.project(...) -> RustProject (full CI DAG) """ from __future__ import annotations @@ -12,10 +13,11 @@ from typing import TYPE_CHECKING, Any from ._toolchain import make_install_chain -from .cache import CacheForever +from .cache import CacheForever, CacheOnChange if TYPE_CHECKING: from ._step import Step + from .cache import CachePolicy APT_PACKAGES = ( "curl", @@ -25,8 +27,6 @@ "libssl-dev", ) -_ACTION_KWARGS = frozenset(("cache", "env", "timeout_seconds", "label", "key")) - _VERSION_RE = re.compile(r"^[a-z0-9.-]+$") @@ -84,6 +84,17 @@ def doc(self, **kw: Any) -> Step: return self._emit("cargo doc --no-deps", ":rust: doc", **kw) +@dataclass(frozen=True) +class RustProject: + """High-level Rust CI DAG — constructed via ``hm.rust.project()``.""" + + toolchain: RustToolchain + warmup: Step + test: Step + clippy: Step + fmt: Step + + def _make_rust( *, path: str = ".", @@ -111,11 +122,66 @@ def _make_rust( return RustToolchain(path=path, installed=installed) +def _make_rust_project( + *, + path: str = ".", + version: str = "stable", + image: str | None = None, + components: tuple[str, ...] = ("clippy", "rustfmt"), + base: Step | None = None, + cache: CachePolicy | None = None, + test_flags: tuple[str, ...] = (), + clippy_flags: tuple[str, ...] = (), + fmt_flags: tuple[str, ...] = (), +) -> RustProject: + tc = _make_rust( + path=path, + version=version, + image=image, + components=components, + base=base, + ) + + lock_path = f"{path}/Cargo.lock" if path != "." else "Cargo.lock" + warmup_cache = cache if cache is not None else CacheOnChange(paths=(lock_path,)) + + warm = tc._emit( + "cargo build --workspace --tests --locked", + ":rust: warmup", + cache=warmup_cache, + ) + + test_extra = (" " + " ".join(test_flags)) if test_flags else "" + test_step = warm.sh( + tc._wrap(f"cargo test --workspace --locked{test_extra}"), + label=":rust: test", + ) + + clippy_extra = (" " + " ".join(clippy_flags)) if clippy_flags else "" + clippy_step = warm.sh( + tc._wrap( + f"cargo clippy --workspace --tests --locked{clippy_extra} -- -D warnings" + ), + label=":rust: clippy", + ) + + fmt_extra = (" " + " ".join(fmt_flags)) if fmt_flags else "" + fmt_step = tc._emit(f"cargo fmt --check{fmt_extra}", ":rust: fmt") + + return RustProject( + toolchain=tc, + warmup=warm, + test=test_step, + clippy=clippy_step, + fmt=fmt_step, + ) + + class _RustEntry: - """Callable singleton — supports both object form and bare form.""" + """Namespace for ``hm.rust.toolchain()`` and ``hm.rust.project()``.""" - def __call__( - self, + @staticmethod + def toolchain( *, path: str = ".", version: str = "stable", @@ -131,29 +197,30 @@ def __call__( base=base, ) - def build(self, *, release: bool = False, **kw: Any) -> Step: - action_kw = {k: kw.pop(k) for k in list(kw) if k in _ACTION_KWARGS} - return self(**kw).build(release=release, **action_kw) - - def test(self, *, release: bool = False, **kw: Any) -> Step: - action_kw = {k: kw.pop(k) for k in list(kw) if k in _ACTION_KWARGS} - return self(**kw).test(release=release, **action_kw) - - def clippy(self, **kw: Any) -> Step: - action_kw = {k: kw.pop(k) for k in list(kw) if k in _ACTION_KWARGS} - return self(**kw).clippy(**action_kw) - - def fmt(self, **kw: Any) -> Step: - action_kw = {k: kw.pop(k) for k in list(kw) if k in _ACTION_KWARGS} - return self(**kw).fmt(**action_kw) - - def warmup(self, **kw: Any) -> Step: - action_kw = {k: kw.pop(k) for k in list(kw) if k in _ACTION_KWARGS} - return self(**kw).warmup(**action_kw) - - def doc(self, **kw: Any) -> Step: - action_kw = {k: kw.pop(k) for k in list(kw) if k in _ACTION_KWARGS} - return self(**kw).doc(**action_kw) + @staticmethod + def project( + *, + path: str = ".", + version: str = "stable", + image: str | None = None, + components: tuple[str, ...] = ("clippy", "rustfmt"), + base: Step | None = None, + cache: CachePolicy | None = None, + test_flags: tuple[str, ...] = (), + clippy_flags: tuple[str, ...] = (), + fmt_flags: tuple[str, ...] = (), + ) -> RustProject: + return _make_rust_project( + path=path, + version=version, + image=image, + components=components, + base=base, + cache=cache, + test_flags=test_flags, + clippy_flags=clippy_flags, + fmt_flags=fmt_flags, + ) rust = _RustEntry() From ce87183cd81ef2686b7711a317935432c4a8957e Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 15:39:30 -0700 Subject: [PATCH 13/41] feat(py): wire RustProject into as_leaves + exports --- dsls/harmont-py/harmont/__init__.py | 3 ++- dsls/harmont-py/harmont/_unwrap.py | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/dsls/harmont-py/harmont/__init__.py b/dsls/harmont-py/harmont/__init__.py index 8952b83d..2b544dbc 100644 --- a/dsls/harmont-py/harmont/__init__.py +++ b/dsls/harmont-py/harmont/__init__.py @@ -58,7 +58,7 @@ from .pipeline import pipeline_to_json from .python import python from .ruby import ruby -from .rust import rust +from .rust import RustProject, rust from .triggers import pull_request, push, schedule from .types import Pipeline from .zig import zig @@ -139,6 +139,7 @@ def sh( "Dep", "Deployment", "Pipeline", + "RustProject", "Step", "Target", "apt_base", diff --git a/dsls/harmont-py/harmont/_unwrap.py b/dsls/harmont-py/harmont/_unwrap.py index 718bd369..b69045dd 100644 --- a/dsls/harmont-py/harmont/_unwrap.py +++ b/dsls/harmont-py/harmont/_unwrap.py @@ -19,12 +19,14 @@ from .haskell import HaskellPackage from .npm import NpmProject from .py.uv import UvProject -from .rust import RustToolchain +from .rust import RustProject, RustToolchain def _one(obj: object) -> tuple[Step, ...]: if isinstance(obj, Step): return (obj,) + if isinstance(obj, RustProject): + return (obj.test, obj.clippy, obj.fmt) if isinstance(obj, HaskellPackage): return (obj.build(),) if isinstance(obj, RustToolchain): @@ -39,8 +41,8 @@ def _one(obj: object) -> tuple[Step, ...]: return as_leaves(obj) msg = ( f"hm.target: cannot use {type(obj).__name__} as a pipeline leaf\n" - " → return one of: Step, tuple[Step, ...], HaskellPackage, " - "RustToolchain, NpmProject, ElmProject, UvProject" + " → return one of: Step, tuple[Step, ...], RustProject, RustToolchain, " + "HaskellPackage, NpmProject, ElmProject, UvProject" ) raise TypeError(msg) From 9cc9e721ca101cfcebf9104a8eaf68a2dd77fa1e Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 15:43:09 -0700 Subject: [PATCH 14/41] test(py): update Rust tests for rust.toolchain()/rust.project() API --- dsls/harmont-py/tests/test_e2e_fixtures.py | 2 +- dsls/harmont-py/tests/test_rust.py | 440 ++++++++++-------- dsls/harmont-py/tests/test_target_unwrap.py | 13 +- .../tests/test_toolchain_compose.py | 10 +- 4 files changed, 254 insertions(+), 211 deletions(-) diff --git a/dsls/harmont-py/tests/test_e2e_fixtures.py b/dsls/harmont-py/tests/test_e2e_fixtures.py index 1bae78fc..f65ab8cf 100644 --- a/dsls/harmont-py/tests/test_e2e_fixtures.py +++ b/dsls/harmont-py/tests/test_e2e_fixtures.py @@ -69,7 +69,7 @@ def _build_monorepo_ci() -> dict: def _build_rust_release() -> dict: - project = rust(path=".") + project = rust.toolchain(path=".") return hm.pipeline( project.build(), diff --git a/dsls/harmont-py/tests/test_rust.py b/dsls/harmont-py/tests/test_rust.py index 60cb6023..e13d7905 100644 --- a/dsls/harmont-py/tests/test_rust.py +++ b/dsls/harmont-py/tests/test_rust.py @@ -1,4 +1,4 @@ -"""Rust toolchain abstraction tests.""" +"""Rust toolchain and project abstraction tests.""" from __future__ import annotations @@ -20,205 +20,239 @@ def _step_by_substring(p: dict, needle: str) -> dict: raise AssertionError(msg) -def test_rust_object_form_full_chain(): - rust = hm.rust(path="cli") - p = hm.pipeline(rust.build(), default_image="ubuntu:24.04") - cmds = _cmds(p) - assert any("apt-get install" in c for c in cmds) - assert any("sh.rustup.rs" in c for c in cmds) - assert any("cd cli && cargo build" in c for c in cmds) - - -def test_rust_actions_share_install_step(): - rust = hm.rust(path="cli") - p = hm.pipeline( - rust.build(), - rust.test(), - rust.clippy(), - rust.fmt(), - rust.doc(), - default_image="ubuntu:24.04", - ) - cmds = _cmds(p) - assert len([c for c in cmds if "sh.rustup.rs" in c]) == 1 - assert len([c for c in cmds if "apt-get install" in c]) == 1 - assert any("cargo build" in c for c in cmds) - assert any("cargo test" in c for c in cmds) - assert any("cargo clippy --all-targets -- -D warnings" in c for c in cmds) - assert any("cargo fmt --check" in c for c in cmds) - assert any("cargo doc --no-deps" in c for c in cmds) - - -def test_rust_build_release_flag(): - rust = hm.rust(path=".") - s = rust.build(release=True) - assert s.cmd is not None - assert "cargo build --release" in s.cmd - - -def test_rust_test_release_flag(): - rust = hm.rust(path=".") - s = rust.test(release=True) - assert s.cmd is not None - assert "cargo test --release" in s.cmd - - -def test_rust_rustup_cache_forever(): - rust = hm.rust(path="cli") - p = hm.pipeline(rust.build()) - rustup = _step_by_substring(p, "sh.rustup.rs") - assert rustup["cache"]["policy"] == "forever" - - -def test_rust_default_components(): - rust = hm.rust(path=".") - p = hm.pipeline(rust.build()) - rustup = _step_by_substring(p, "sh.rustup.rs") - assert "--component clippy,rustfmt" in rustup["cmd"] - - -def test_rust_components_override(): - rust = hm.rust(path=".", components=("clippy",)) - p = hm.pipeline(rust.build()) - rustup = _step_by_substring(p, "sh.rustup.rs") - assert "--component clippy" in rustup["cmd"] - assert "rustfmt" not in rustup["cmd"] - - -def test_rust_version_in_rustup_cmd(): - rust = hm.rust(path=".", version="1.81.0") - p = hm.pipeline(rust.build()) - rustup = _step_by_substring(p, "sh.rustup.rs") - assert "--default-toolchain 1.81.0" in rustup["cmd"] - - -def test_rust_invalid_version_rejected(): - with pytest.raises(ValueError, match="version"): - hm.rust(version="not a valid; version") - - -def test_rust_installed_escape_hatch_chains(): - rust = hm.rust(path="cli") - custom = rust.installed.sh( - "cd cli && cargo build --release --features foo", - label=":rust: custom", - ) - p = hm.pipeline(custom) - cmds = _cmds(p) - assert any("--features foo" in c for c in cmds) - - -def test_rust_action_labels_auto_generated(): - rust = hm.rust(path=".") - assert rust.build().label == ":rust: build" - assert rust.test().label == ":rust: test" - assert rust.clippy().label == ":rust: clippy" - assert rust.fmt().label == ":rust: fmt" - assert rust.doc().label == ":rust: doc" - - -def test_rust_action_label_override(): - rust = hm.rust(path=".") - s = rust.build(label=":rust: dev build") - assert s.label == ":rust: dev build" - - -def test_rust_action_cache_forwarded(): - rust = hm.rust(path=".") - s = rust.build(cache=CacheOnChange(paths=("Cargo.lock",))) - assert s.cache == CacheOnChange(paths=("Cargo.lock",)) - - -def test_rust_image_emitted_on_apt_step(): - rust = hm.rust(path=".", image="alpine:3.20") - p = hm.pipeline(rust.build()) - apt = _step_by_substring(p, "apt-get install") - assert apt.get("image") == "alpine:3.20" - - -def test_rust_with_base_skips_apt(): - base = hm.scratch().sh("custom base", label="base") - rust = hm.rust(path="cli", base=base) - p = hm.pipeline(rust.build(), default_image="ubuntu:24.04") - cmds = _cmds(p) - assert not any("apt-get install" in c for c in cmds) - assert any("custom base" in c for c in cmds) - assert any("sh.rustup.rs" in c for c in cmds) - assert any("cd cli && cargo build" in c for c in cmds) - - -def test_rust_bare_form_build(): - p = hm.pipeline(hm.rust.build()) - cmds = _cmds(p) - assert any("cd . && cargo build" in c for c in cmds) - - -def test_rust_bare_form_all_actions(): - p = hm.pipeline( - hm.rust.build(), hm.rust.test(), hm.rust.clippy(), hm.rust.fmt(), hm.rust.doc() - ) - cmds = _cmds(p) - assert any("cargo build" in c for c in cmds) - assert any("cargo test" in c for c in cmds) - assert any("cargo clippy" in c for c in cmds) - assert any("cargo fmt --check" in c for c in cmds) - assert any("cargo doc --no-deps" in c for c in cmds) - - -def test_rust_bare_form_accepts_path_kwarg(): - p = hm.pipeline(hm.rust.test(path="cli")) - cmds = _cmds(p) - assert any("cd cli && cargo test" in c for c in cmds) - - -def test_rust_bare_form_forwards_action_kwargs(): - s = hm.rust.build(path="cli", label=":rust: custom") - assert s.label == ":rust: custom" - - -def test_rust_warmup_returns_step(): - rust = hm.rust(path="cli") - w = rust.warmup() - assert w.cmd is not None - assert "cargo build" in w.cmd - assert "--workspace" in w.cmd - assert "--tests" in w.cmd - assert "--locked" in w.cmd - - -def test_rust_warmup_chains_from_installed(): - rust = hm.rust(path="cli") - w = rust.warmup() - assert w.parent is rust.installed - - -def test_rust_warmup_default_label(): - rust = hm.rust(path=".") - assert rust.warmup().label == ":rust: warmup" - - -def test_rust_warmup_label_override(): - rust = hm.rust(path=".") - assert rust.warmup(label=":rust: pre-build").label == ":rust: pre-build" - - -def test_rust_warmup_in_pipeline(): - """warmup step appears in pipeline IR.""" - rust = hm.rust(path="cli") - w = rust.warmup() - t = w.sh( - ". $HOME/.cargo/env && cd cli && cargo test --workspace --locked", label=":rust: test" - ) - p = hm.pipeline(t, rust.fmt(), default_image="ubuntu:24.04") - cmds = _cmds(p) - assert any("cargo build --workspace --tests --locked" in c for c in cmds) - assert any("cargo test --workspace --locked" in c for c in cmds) - assert any("cargo fmt" in c for c in cmds) - assert len([c for c in cmds if "sh.rustup.rs" in c]) == 1 - assert len([c for c in cmds if "apt-get install" in c]) == 1 - - -def test_rust_bare_form_warmup(): - p = hm.pipeline(hm.rust.warmup()) - cmds = _cmds(p) - assert any("cargo build --workspace --tests --locked" in c for c in cmds) +# --- RustToolchain (hm.rust.toolchain) --- + + +class TestRustToolchain: + def test_full_chain(self): + tc = hm.rust.toolchain(path="cli") + p = hm.pipeline(tc.build(), default_image="ubuntu:24.04") + cmds = _cmds(p) + assert any("apt-get install" in c for c in cmds) + assert any("sh.rustup.rs" in c for c in cmds) + assert any("cd cli && cargo build" in c for c in cmds) + + def test_actions_share_install_step(self): + tc = hm.rust.toolchain(path="cli") + p = hm.pipeline( + tc.build(), tc.test(), tc.clippy(), tc.fmt(), tc.doc(), + default_image="ubuntu:24.04", + ) + cmds = _cmds(p) + assert len([c for c in cmds if "sh.rustup.rs" in c]) == 1 + assert len([c for c in cmds if "apt-get install" in c]) == 1 + + def test_build_release(self): + tc = hm.rust.toolchain(path=".") + s = tc.build(release=True) + assert "cargo build --release" in s.cmd + + def test_test_release(self): + tc = hm.rust.toolchain(path=".") + s = tc.test(release=True) + assert "cargo test --release" in s.cmd + + def test_rustup_cache_forever(self): + tc = hm.rust.toolchain(path="cli") + p = hm.pipeline(tc.build()) + rustup = _step_by_substring(p, "sh.rustup.rs") + assert rustup["cache"]["policy"] == "forever" + + def test_default_components(self): + tc = hm.rust.toolchain(path=".") + p = hm.pipeline(tc.build()) + rustup = _step_by_substring(p, "sh.rustup.rs") + assert "--component clippy,rustfmt" in rustup["cmd"] + + def test_components_override(self): + tc = hm.rust.toolchain(path=".", components=("clippy",)) + p = hm.pipeline(tc.build()) + rustup = _step_by_substring(p, "sh.rustup.rs") + assert "--component clippy" in rustup["cmd"] + assert "rustfmt" not in rustup["cmd"] + + def test_version_in_rustup_cmd(self): + tc = hm.rust.toolchain(path=".", version="1.81.0") + p = hm.pipeline(tc.build()) + rustup = _step_by_substring(p, "sh.rustup.rs") + assert "--default-toolchain 1.81.0" in rustup["cmd"] + + def test_invalid_version_rejected(self): + with pytest.raises(ValueError, match="version"): + hm.rust.toolchain(version="not a valid; version") + + def test_installed_escape_hatch(self): + tc = hm.rust.toolchain(path="cli") + custom = tc.installed.sh( + "cd cli && cargo build --release --features foo", + label=":rust: custom", + ) + p = hm.pipeline(custom) + cmds = _cmds(p) + assert any("--features foo" in c for c in cmds) + + def test_action_labels(self): + tc = hm.rust.toolchain(path=".") + assert tc.build().label == ":rust: build" + assert tc.test().label == ":rust: test" + assert tc.clippy().label == ":rust: clippy" + assert tc.fmt().label == ":rust: fmt" + assert tc.doc().label == ":rust: doc" + + def test_action_label_override(self): + tc = hm.rust.toolchain(path=".") + s = tc.build(label=":rust: dev build") + assert s.label == ":rust: dev build" + + def test_action_cache_forwarded(self): + tc = hm.rust.toolchain(path=".") + s = tc.build(cache=CacheOnChange(paths=("Cargo.lock",))) + assert s.cache == CacheOnChange(paths=("Cargo.lock",)) + + def test_image_emitted_on_apt_step(self): + tc = hm.rust.toolchain(path=".", image="alpine:3.20") + p = hm.pipeline(tc.build()) + apt = _step_by_substring(p, "apt-get install") + assert apt.get("image") == "alpine:3.20" + + def test_with_base_skips_apt(self): + base = hm.scratch().sh("custom base", label="base") + tc = hm.rust.toolchain(path="cli", base=base) + p = hm.pipeline(tc.build(), default_image="ubuntu:24.04") + cmds = _cmds(p) + assert not any("apt-get install" in c for c in cmds) + assert any("custom base" in c for c in cmds) + assert any("sh.rustup.rs" in c for c in cmds) + assert any("cd cli && cargo build" in c for c in cmds) + + def test_warmup_returns_step(self): + tc = hm.rust.toolchain(path="cli") + w = tc.warmup() + assert w.cmd is not None + assert "cargo build --workspace --tests --locked" in w.cmd + + def test_warmup_chains_from_installed(self): + tc = hm.rust.toolchain(path="cli") + w = tc.warmup() + assert w.parent is tc.installed + + def test_warmup_default_label(self): + tc = hm.rust.toolchain(path=".") + assert tc.warmup().label == ":rust: warmup" + + def test_warmup_label_override(self): + tc = hm.rust.toolchain(path=".") + assert tc.warmup(label=":rust: pre-build").label == ":rust: pre-build" + + def test_warmup_in_pipeline(self): + tc = hm.rust.toolchain(path="cli") + w = tc.warmup() + t = w.sh( + ". $HOME/.cargo/env && cd cli && cargo test --workspace --locked", + label=":rust: test", + ) + p = hm.pipeline(t, tc.fmt(), default_image="ubuntu:24.04") + cmds = _cmds(p) + assert any("cargo build --workspace --tests --locked" in c for c in cmds) + assert any("cargo test --workspace --locked" in c for c in cmds) + assert any("cargo fmt" in c for c in cmds) + assert len([c for c in cmds if "sh.rustup.rs" in c]) == 1 + assert len([c for c in cmds if "apt-get install" in c]) == 1 + + +# --- RustProject (hm.rust.project) --- + + +class TestRustProject: + def test_project_has_all_steps(self): + proj = hm.rust.project(path="cli") + assert proj.warmup.cmd is not None + assert proj.test.cmd is not None + assert proj.clippy.cmd is not None + assert proj.fmt.cmd is not None + + def test_warmup_implicit_cache_on_change(self): + proj = hm.rust.project(path="cli") + assert proj.warmup.cache == CacheOnChange(paths=("cli/Cargo.lock",)) + + def test_warmup_implicit_cache_dot_path(self): + proj = hm.rust.project(path=".") + assert proj.warmup.cache == CacheOnChange(paths=("Cargo.lock",)) + + def test_warmup_cache_override(self): + custom = CacheOnChange(paths=("Cargo.toml",)) + proj = hm.rust.project(path=".", cache=custom) + assert proj.warmup.cache == custom + + def test_test_command(self): + proj = hm.rust.project(path="cli") + assert "cargo test --workspace --locked" in proj.test.cmd + + def test_test_flags(self): + proj = hm.rust.project(path=".", test_flags=("--lib", "--no-fail-fast")) + assert "cargo test --workspace --locked --lib --no-fail-fast" in proj.test.cmd + + def test_clippy_command(self): + proj = hm.rust.project(path="cli") + assert "cargo clippy --workspace --tests --locked -- -D warnings" in proj.clippy.cmd + + def test_clippy_flags(self): + proj = hm.rust.project(path=".", clippy_flags=("--fix",)) + assert "cargo clippy --workspace --tests --locked --fix -- -D warnings" in proj.clippy.cmd + + def test_fmt_command(self): + proj = hm.rust.project(path="cli") + assert "cargo fmt --check" in proj.fmt.cmd + + def test_fmt_flags(self): + proj = hm.rust.project(path=".", fmt_flags=("--all",)) + assert "cargo fmt --check --all" in proj.fmt.cmd + + def test_test_chains_off_warmup(self): + proj = hm.rust.project(path=".") + assert proj.test.parent is proj.warmup + + def test_clippy_chains_off_warmup(self): + proj = hm.rust.project(path=".") + assert proj.clippy.parent is proj.warmup + + def test_fmt_chains_off_install(self): + proj = hm.rust.project(path=".") + assert proj.fmt.parent is proj.toolchain.installed + + def test_toolchain_escape_hatch(self): + proj = hm.rust.project(path="cli") + custom = proj.toolchain.installed.sh("custom", label="custom") + assert custom.parent is proj.toolchain.installed + + def test_with_base_skips_apt(self): + base = hm.scratch().sh("custom base", label="base") + proj = hm.rust.project(path="cli", base=base) + p = hm.pipeline(proj.test, proj.clippy, proj.fmt, default_image="ubuntu:24.04") + cmds = _cmds(p) + assert not any("apt-get install" in c for c in cmds) + assert any("custom base" in c for c in cmds) + + def test_labels(self): + proj = hm.rust.project(path=".") + assert proj.warmup.label == ":rust: warmup" + assert proj.test.label == ":rust: test" + assert proj.clippy.label == ":rust: clippy" + assert proj.fmt.label == ":rust: fmt" + + def test_pipeline_ir(self): + proj = hm.rust.project(path="cli") + p = hm.pipeline(proj.test, proj.clippy, proj.fmt, default_image="ubuntu:24.04") + cmds = _cmds(p) + assert any("cargo build --workspace --tests --locked" in c for c in cmds) + assert any("cargo test --workspace --locked" in c for c in cmds) + assert any("cargo clippy" in c for c in cmds) + assert any("cargo fmt --check" in c for c in cmds) + assert len([c for c in cmds if "sh.rustup.rs" in c]) == 1 + assert len([c for c in cmds if "apt-get install" in c]) == 1 + + def test_version_forwarded(self): + proj = hm.rust.project(path=".", version="1.81.0") + p = hm.pipeline(proj.test) + rustup = _step_by_substring(p, "sh.rustup.rs") + assert "--default-toolchain 1.81.0" in rustup["cmd"] diff --git a/dsls/harmont-py/tests/test_target_unwrap.py b/dsls/harmont-py/tests/test_target_unwrap.py index 496afccb..95664627 100644 --- a/dsls/harmont-py/tests/test_target_unwrap.py +++ b/dsls/harmont-py/tests/test_target_unwrap.py @@ -39,12 +39,21 @@ def test_haskell_package_unwraps_to_build(tmp_path, monkeypatch): def test_rust_toolchain_unwraps_to_build(): - tc = hm.rust(path="cli", version="stable") + tc = hm.rust.toolchain(path="cli", version="stable") leaves = as_leaves(tc) assert len(leaves) == 1 assert "cargo build" in leaves[0].cmd +def test_rust_project_unwraps_to_test_clippy_fmt(): + proj = hm.rust.project(path="cli") + leaves = as_leaves(proj) + assert len(leaves) == 3 + assert "cargo test" in leaves[0].cmd + assert "cargo clippy" in leaves[1].cmd + assert "cargo fmt" in leaves[2].cmd + + def test_npm_project_unwraps_to_install(): proj = hm.npm(path="app", version="20") leaves = as_leaves(proj) @@ -76,5 +85,5 @@ def test_unknown_type_raises_typeerror(): def test_unknown_type_message_lists_supported_types(): - with pytest.raises(TypeError, match=r"Step.*HaskellPackage.*ElmProject"): + with pytest.raises(TypeError, match=r"Step.*RustProject.*RustToolchain.*HaskellPackage"): as_leaves("oops") # type: ignore[arg-type] diff --git a/dsls/harmont-py/tests/test_toolchain_compose.py b/dsls/harmont-py/tests/test_toolchain_compose.py index bdafca47..19b2f73d 100644 --- a/dsls/harmont-py/tests/test_toolchain_compose.py +++ b/dsls/harmont-py/tests/test_toolchain_compose.py @@ -44,7 +44,7 @@ def test_stack_elm_on_npm(): def test_escape_hatch_consistent_across_toolchains(): """Every toolchain exposes .installed as a public Step.""" - rust = hm.rust(path=".") + rust = hm.rust.toolchain(path=".") ghc = hm.haskell(ghc="9.6.7") api = ghc.package("api") node = hm.npm(path=".") @@ -60,7 +60,7 @@ def test_deterministic_emission(): """Two identical pipeline constructions emit equal IR dicts.""" def build() -> dict: - rust = hm.rust(path="cli") + rust = hm.rust.toolchain(path="cli") return hm.pipeline(rust.build(), rust.test(), default_image="ubuntu:24.04") assert build() == build() @@ -69,7 +69,7 @@ def build() -> dict: def test_mixed_pipeline_compiles(): """A pipeline mixing all four toolchains lowers without error.""" ghc = hm.haskell(ghc="9.6.7") - rust = hm.rust(path="cli") + rust = hm.rust.toolchain(path="cli") node = hm.npm(path="app/codegen") elm = hm.elm(path="app", base=node.installed) p = hm.pipeline( @@ -105,7 +105,7 @@ def test_apt_base_shared_across_toolchains(): "python3-venv", ), ) - rust = hm.rust(path=".", base=base) + rust = hm.rust.toolchain(path=".", base=base) py = hm.py.uv(path="dsls/harmont-py", base=base) p = hm.pipeline( rust.build(), @@ -125,7 +125,7 @@ def test_apt_base_default_label(): def test_apt_base_custom_image(): base = hm.apt_base(packages=("curl",), image="debian:bookworm") - rust = hm.rust(path=".", base=base) + rust = hm.rust.toolchain(path=".", base=base) p = hm.pipeline(rust.build(), default_image="ubuntu:24.04") apt_step = _step_by_substring(p, "apt-get install") assert apt_step.get("image") == "debian:bookworm" From 845e3a8eef4ea8ff1632db0601d6a7bc75ea6a80 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 15:46:28 -0700 Subject: [PATCH 15/41] feat(ts): add RustProject with rust.project()/rust.toolchain() + update tests --- dsls/harmont-ts/src/toolchains/index.ts | 2 +- dsls/harmont-ts/src/toolchains/rust.ts | 81 +++++++- dsls/harmont-ts/tests/e2e-fixtures.test.ts | 2 +- dsls/harmont-ts/tests/toolchains/rust.test.ts | 196 +++++++++++++++--- .../tests/toolchains/shared.test.ts | 2 +- examples/rust/.harmont/pipeline.ts | 2 +- 6 files changed, 245 insertions(+), 40 deletions(-) diff --git a/dsls/harmont-ts/src/toolchains/index.ts b/dsls/harmont-ts/src/toolchains/index.ts index b24cec7b..da0a1f99 100644 --- a/dsls/harmont-ts/src/toolchains/index.ts +++ b/dsls/harmont-ts/src/toolchains/index.ts @@ -1,6 +1,6 @@ export { npm, NpmProject, type NpmOptions } from "./npm.js"; export { go, GoToolchain, type GoOptions } from "./go.js"; -export { rust, RustToolchain, type RustOptions } from "./rust.js"; +export { rust, RustToolchain, RustProject, type RustToolchainOptions, type RustProjectOptions } from "./rust.js"; export { python, PythonToolchain, type PythonOptions } from "./python.js"; export { cmake, CMakeProject, type CMakeOptions } from "./cmake.js"; export { gradle, GradleProject, type GradleOptions } from "./gradle.js"; diff --git a/dsls/harmont-ts/src/toolchains/rust.ts b/dsls/harmont-ts/src/toolchains/rust.ts index 56f6e972..b0a1e5eb 100644 --- a/dsls/harmont-ts/src/toolchains/rust.ts +++ b/dsls/harmont-ts/src/toolchains/rust.ts @@ -1,5 +1,5 @@ import type { Step, StepOptions } from "../step.js"; -import { forever } from "../cache.js"; +import { type CachePolicy, forever, onChange } from "../cache.js"; import { makeInstallChain } from "./shared.js"; const APT_PACKAGES = [ @@ -11,7 +11,7 @@ const APT_PACKAGES = [ ] as const; const VERSION_RE = /^[a-z0-9.-]+$/; -export interface RustOptions { +export interface RustToolchainOptions { readonly path?: string; readonly version?: string; readonly image?: string; @@ -19,6 +19,13 @@ export interface RustOptions { readonly base?: Step; } +export interface RustProjectOptions extends RustToolchainOptions { + readonly cache?: CachePolicy; + readonly testFlags?: readonly string[]; + readonly clippyFlags?: readonly string[]; + readonly fmtFlags?: readonly string[]; +} + type ActionOptions = Omit; export class RustToolchain { @@ -34,7 +41,7 @@ export class RustToolchain { return this._installed; } - private _cargo(cmd: string, label: string, opts?: ActionOptions): Step { + _cargo(cmd: string, label: string, opts?: ActionOptions): Step { return this._installed.sh( `. $HOME/.cargo/env && cd ${this.path} && ${cmd}`, { label, ...opts }, @@ -76,14 +83,36 @@ export class RustToolchain { } } -export function rust(opts?: RustOptions): RustToolchain { +export class RustProject { + readonly toolchain: RustToolchain; + readonly warmup: Step; + readonly test: Step; + readonly clippy: Step; + readonly fmt: Step; + + constructor( + toolchain: RustToolchain, + warmup: Step, + test: Step, + clippy: Step, + fmt: Step, + ) { + this.toolchain = toolchain; + this.warmup = warmup; + this.test = test; + this.clippy = clippy; + this.fmt = fmt; + } +} + +function makeToolchain(opts?: RustToolchainOptions): RustToolchain { const path = opts?.path ?? "."; const version = opts?.version ?? "stable"; const components = opts?.components ?? ["clippy", "rustfmt"]; if (!VERSION_RE.test(version)) { throw new Error( - `hm.rust: invalid version "${version}"\n → use "stable", "nightly", or a semver like "1.81.0"`, + `rust.toolchain: invalid version "${version}"\n → use "stable", "nightly", or a semver like "1.81.0"`, ); } @@ -106,3 +135,45 @@ export function rust(opts?: RustOptions): RustToolchain { return new RustToolchain(path, installed); } + +function makeProject(opts?: RustProjectOptions): RustProject { + const path = opts?.path ?? "."; + const tc = makeToolchain(opts); + + const lockPath = path !== "." ? `${path}/Cargo.lock` : "Cargo.lock"; + const warmupCache = opts?.cache ?? onChange(lockPath); + + const warm = tc._cargo( + "cargo build --workspace --tests --locked", + ":rust: warmup", + { cache: warmupCache }, + ); + + const testExtra = opts?.testFlags?.length + ? " " + opts.testFlags.join(" ") + : ""; + const testStep = warm.sh( + `. $HOME/.cargo/env && cd ${path} && cargo test --workspace --locked${testExtra}`, + { label: ":rust: test" }, + ); + + const clippyExtra = opts?.clippyFlags?.length + ? " " + opts.clippyFlags.join(" ") + : ""; + const clippyStep = warm.sh( + `. $HOME/.cargo/env && cd ${path} && cargo clippy --workspace --tests --locked${clippyExtra} -- -D warnings`, + { label: ":rust: clippy" }, + ); + + const fmtExtra = opts?.fmtFlags?.length + ? " " + opts.fmtFlags.join(" ") + : ""; + const fmtStep = tc._cargo(`cargo fmt --check${fmtExtra}`, ":rust: fmt"); + + return new RustProject(tc, warm, testStep, clippyStep, fmtStep); +} + +export const rust = { + toolchain: makeToolchain, + project: makeProject, +}; diff --git a/dsls/harmont-ts/tests/e2e-fixtures.test.ts b/dsls/harmont-ts/tests/e2e-fixtures.test.ts index d2ca6352..0789af90 100644 --- a/dsls/harmont-ts/tests/e2e-fixtures.test.ts +++ b/dsls/harmont-ts/tests/e2e-fixtures.test.ts @@ -79,7 +79,7 @@ describe("E2E pipeline fixtures", () => { }); it("rust-release", () => { - const project = rust({ path: "." }); + const project = rust.toolchain({ path: "." }); const ir = pipeline( project.build(), diff --git a/dsls/harmont-ts/tests/toolchains/rust.test.ts b/dsls/harmont-ts/tests/toolchains/rust.test.ts index 6f929feb..acc99f58 100644 --- a/dsls/harmont-ts/tests/toolchains/rust.test.ts +++ b/dsls/harmont-ts/tests/toolchains/rust.test.ts @@ -3,80 +3,95 @@ import { rust } from "../../src/toolchains/rust.js"; import { sh } from "../../src/step.js"; import { pipeline } from "../../src/pipeline.js"; -describe("rust factory", () => { +const cmds = (ir: ReturnType) => + ir.graph.nodes.map((n: { step: { cmd: string } }) => n.step.cmd); + +const stepBySubstring = (ir: ReturnType, needle: string) => { + const node = ir.graph.nodes.find((n: { step: { cmd: string } }) => + n.step.cmd.includes(needle), + ); + if (!node) throw new Error(`no command step containing "${needle}"`); + return node.step; +}; + +describe("rust.toolchain", () => { it("returns a RustToolchain with defaults", () => { - const r = rust(); + const r = rust.toolchain(); expect(r.path).toBe("."); expect(r.install()._cmd).toContain("rustc --version"); }); it("accepts path and version", () => { - const r = rust({ path: "crates/core", version: "nightly" }); + const r = rust.toolchain({ path: "crates/core", version: "nightly" }); expect(r.path).toBe("crates/core"); expect(r.install()._cmd).toContain("nightly"); }); it("accepts custom components", () => { - const r = rust({ components: ["clippy", "rustfmt", "miri"] }); + const r = rust.toolchain({ components: ["clippy", "rustfmt", "miri"] }); expect(r.install()._cmd).toContain("clippy,rustfmt,miri"); }); it("rejects invalid version", () => { - expect(() => rust({ version: "not valid!" })).toThrow("invalid version"); + expect(() => rust.toolchain({ version: "not valid!" })).toThrow( + "invalid version", + ); }); -}); -describe("rust actions", () => { it("build runs cargo build", () => { - const r = rust(); + const r = rust.toolchain(); expect(r.build()._cmd).toContain("cargo build"); expect(r.build()._cmd).not.toContain("--release"); }); it("build --release", () => { - const r = rust(); - expect(r.build({ release: true })._cmd).toContain("cargo build --release"); + const r = rust.toolchain(); + expect(r.build({ release: true })._cmd).toContain( + "cargo build --release", + ); }); it("test runs cargo test", () => { - const r = rust(); + const r = rust.toolchain(); expect(r.test()._cmd).toContain("cargo test"); }); it("clippy runs with -D warnings", () => { - const r = rust(); - expect(r.clippy()._cmd).toContain("cargo clippy --all-targets -- -D warnings"); + const r = rust.toolchain(); + expect(r.clippy()._cmd).toContain( + "cargo clippy --all-targets -- -D warnings", + ); }); it("fmt runs cargo fmt --check", () => { - const r = rust(); + const r = rust.toolchain(); expect(r.fmt()._cmd).toContain("cargo fmt --check"); }); it("doc runs cargo doc --no-deps", () => { - const r = rust(); + const r = rust.toolchain(); expect(r.doc()._cmd).toContain("cargo doc --no-deps"); }); it("actions source cargo env", () => { - const r = rust(); + const r = rust.toolchain(); expect(r.build()._cmd).toContain(". $HOME/.cargo/env"); }); it("actions chain from install", () => { - const r = rust(); + const r = rust.toolchain(); expect(r.build()._parent).toBe(r.install()); }); it("accepts step options", () => { - const r = rust(); + const r = rust.toolchain(); const t = r.test({ label: "my test", timeoutSeconds: 600 }); expect(t._label).toBe("my test"); expect(t._timeoutSeconds).toBe(600); }); it("default labels use :rust: prefix", () => { - const r = rust(); + const r = rust.toolchain(); expect(r.build()._label).toBe(":rust: build"); expect(r.test()._label).toBe(":rust: test"); expect(r.clippy()._label).toBe(":rust: clippy"); @@ -85,32 +100,30 @@ describe("rust actions", () => { }); it("warmup runs cargo build --workspace --tests --locked", () => { - const r = rust(); + const r = rust.toolchain(); expect(r.warmup()._cmd).toContain( "cargo build --workspace --tests --locked", ); }); it("warmup chains from install", () => { - const r = rust(); + const r = rust.toolchain(); expect(r.warmup()._parent).toBe(r.install()); }); it("warmup default label", () => { - const r = rust(); + const r = rust.toolchain(); expect(r.warmup()._label).toBe(":rust: warmup"); }); it("warmup accepts options", () => { - const r = rust(); + const r = rust.toolchain(); const w = r.warmup({ label: ":rust: pre-build" }); expect(w._label).toBe(":rust: pre-build"); }); -}); -describe("rust install chain", () => { it("chain is: scratch → apt-base → rustup", () => { - const r = rust(); + const r = rust.toolchain(); const install = r.install(); expect(install._label).toBe(":rust: rustup"); @@ -123,14 +136,12 @@ describe("rust install chain", () => { it("accepts base step", () => { const base = sh("custom base"); - const r = rust({ base }); + const r = rust.toolchain({ base }); expect(r.install()._parent).toBe(base); }); -}); -describe("rust in pipeline", () => { - it("produces valid IR", () => { - const r = rust(); + it("produces valid pipeline IR", () => { + const r = rust.toolchain(); const ir = pipeline(r.build(), r.test(), r.clippy(), r.fmt(), { defaultImage: "ubuntu:24.04", }); @@ -138,3 +149,126 @@ describe("rust in pipeline", () => { expect(ir.version).toBe("0"); }); }); + +describe("rust.project", () => { + it("has all steps", () => { + const proj = rust.project({ path: "cli" }); + expect(proj.warmup._cmd).toContain( + "cargo build --workspace --tests --locked", + ); + expect(proj.test._cmd).toContain("cargo test --workspace --locked"); + expect(proj.clippy._cmd).toContain( + "cargo clippy --workspace --tests --locked", + ); + expect(proj.fmt._cmd).toContain("cargo fmt --check"); + }); + + it("warmup has implicit CacheOnChange on Cargo.lock", () => { + const proj = rust.project({ path: "cli" }); + expect(proj.warmup._cache).toEqual({ + kind: "on_change", + paths: ["cli/Cargo.lock"], + }); + }); + + it("warmup cache uses plain Cargo.lock for dot path", () => { + const proj = rust.project({ path: "." }); + expect(proj.warmup._cache).toEqual({ + kind: "on_change", + paths: ["Cargo.lock"], + }); + }); + + it("warmup cache can be overridden", () => { + const proj = rust.project({ + path: ".", + cache: { kind: "on_change", paths: ["Cargo.toml"] }, + }); + expect(proj.warmup._cache).toEqual({ + kind: "on_change", + paths: ["Cargo.toml"], + }); + }); + + it("test flags are appended", () => { + const proj = rust.project({ + path: ".", + testFlags: ["--lib", "--no-fail-fast"], + }); + expect(proj.test._cmd).toContain( + "cargo test --workspace --locked --lib --no-fail-fast", + ); + }); + + it("clippy flags are inserted before --", () => { + const proj = rust.project({ path: ".", clippyFlags: ["--fix"] }); + expect(proj.clippy._cmd).toContain( + "cargo clippy --workspace --tests --locked --fix -- -D warnings", + ); + }); + + it("fmt flags are appended", () => { + const proj = rust.project({ path: ".", fmtFlags: ["--all"] }); + expect(proj.fmt._cmd).toContain("cargo fmt --check --all"); + }); + + it("test chains off warmup", () => { + const proj = rust.project(); + expect(proj.test._parent).toBe(proj.warmup); + }); + + it("clippy chains off warmup", () => { + const proj = rust.project(); + expect(proj.clippy._parent).toBe(proj.warmup); + }); + + it("fmt chains off install (not warmup)", () => { + const proj = rust.project(); + expect(proj.fmt._parent).toBe(proj.toolchain.install()); + }); + + it("labels are correct", () => { + const proj = rust.project(); + expect(proj.warmup._label).toBe(":rust: warmup"); + expect(proj.test._label).toBe(":rust: test"); + expect(proj.clippy._label).toBe(":rust: clippy"); + expect(proj.fmt._label).toBe(":rust: fmt"); + }); + + it("with base skips apt", () => { + const base = sh("custom base"); + const proj = rust.project({ path: "cli", base }); + const ir = pipeline(proj.test, proj.clippy, proj.fmt, { + defaultImage: "ubuntu:24.04", + }); + const c = cmds(ir); + expect( + c.filter((cmd: string) => cmd.includes("apt-get install")), + ).toHaveLength(0); + expect(c.some((cmd: string) => cmd.includes("custom base"))).toBe(true); + }); + + it("produces valid pipeline IR", () => { + const proj = rust.project({ path: "cli" }); + const ir = pipeline(proj.test, proj.clippy, proj.fmt, { + defaultImage: "ubuntu:24.04", + }); + expect(ir.version).toBe("0"); + expect(ir.graph.nodes.length).toBeGreaterThanOrEqual(4); + }); + + it("toolchain escape hatch", () => { + const proj = rust.project({ path: "cli" }); + const custom = proj.toolchain + .install() + .sh("custom", { label: "custom" }); + expect(custom._parent).toBe(proj.toolchain.install()); + }); + + it("version forwarded", () => { + const proj = rust.project({ path: ".", version: "1.81.0" }); + const ir = pipeline(proj.test); + const rustup = stepBySubstring(ir, "sh.rustup.rs"); + expect(rustup.cmd).toContain("--default-toolchain 1.81.0"); + }); +}); diff --git a/dsls/harmont-ts/tests/toolchains/shared.test.ts b/dsls/harmont-ts/tests/toolchains/shared.test.ts index fa9dd936..cdd4d0d9 100644 --- a/dsls/harmont-ts/tests/toolchains/shared.test.ts +++ b/dsls/harmont-ts/tests/toolchains/shared.test.ts @@ -34,7 +34,7 @@ describe("aptBase", () => { "python3-venv", ], }); - const r = rust({ base }); + const r = rust.toolchain({ base }); const p = uv({ path: "dsls/harmont-py", base }); const ir = pipeline(r.build(), p.test(), { defaultImage: "ubuntu:24.04" }); const cmds = ir.graph.nodes.map( diff --git a/examples/rust/.harmont/pipeline.ts b/examples/rust/.harmont/pipeline.ts index 71c3ef1d..00368388 100644 --- a/examples/rust/.harmont/pipeline.ts +++ b/examples/rust/.harmont/pipeline.ts @@ -1,7 +1,7 @@ import { pipeline, push, type PipelineDefinition } from "harmont"; import { rust } from "harmont/toolchains"; -const project = rust({ path: "." }); +const project = rust.toolchain({ path: "." }); const pipelines: PipelineDefinition[] = [ { From e3cbd2dbb563dca298e4e6a0f2f8bfda8cb4e31d Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 15:47:07 -0700 Subject: [PATCH 16/41] refactor: dogfood pipelines use rust.project() high-level API --- .harmont/ci.py | 42 ++++++++++++++++-------------------------- .harmont/ci.ts | 38 +++++++++++++++----------------------- 2 files changed, 31 insertions(+), 49 deletions(-) diff --git a/.harmont/ci.py b/.harmont/ci.py index dbbd6d70..9ba5ae66 100644 --- a/.harmont/ci.py +++ b/.harmont/ci.py @@ -2,29 +2,26 @@ from __future__ import annotations import harmont as hm -from harmont.cache import CacheOnChange from harmont.py.uv import UvProject -from harmont.rust import RustToolchain - -ALL_APT = ( - "curl", - "ca-certificates", - "build-essential", - "pkg-config", - "libssl-dev", - "python3", - "python3-venv", -) +from harmont.rust import RustProject @hm.target() def shared_base() -> hm.Step: - return hm.apt_base(packages=ALL_APT) + return hm.apt_base(packages=( + "curl", + "ca-certificates", + "build-essential", + "pkg-config", + "libssl-dev", + "python3", + "python3-venv", + )) @hm.target() -def rust_project(shared_base: hm.Target[hm.Step]) -> RustToolchain: - return hm.rust(path=".", base=shared_base) +def rust_project(shared_base: hm.Target[hm.Step]) -> RustProject: + return hm.rust.project(path=".", base=shared_base, test_flags=("--lib",)) @hm.target() @@ -42,20 +39,13 @@ def py_project(shared_base: hm.Target[hm.Step]) -> UvProject: ], ) def ci( - rust_project: hm.Target[RustToolchain], + rust_project: hm.Target[RustProject], py_project: hm.Target[UvProject], ) -> tuple[hm.Step, ...]: - warm = rust_project.warmup(cache=CacheOnChange(paths=("Cargo.lock",))) return ( - warm.sh( - ". $HOME/.cargo/env && cd . && cargo test --workspace --lib --locked --no-fail-fast", - label=":rust: test", - ), - warm.sh( - ". $HOME/.cargo/env && cd . && cargo clippy --workspace --tests --locked -- -D warnings", - label=":rust: clippy", - ), - rust_project.fmt(), + rust_project.test, + rust_project.clippy, + rust_project.fmt, py_project.lint(), py_project.fmt(), py_project.typecheck(paths="harmont"), diff --git a/.harmont/ci.ts b/.harmont/ci.ts index f332acba..d845560b 100644 --- a/.harmont/ci.ts +++ b/.harmont/ci.ts @@ -3,41 +3,33 @@ import { push, pullRequest, aptBase, - onChange, type PipelineDefinition, } from "harmont"; import { rust, py } from "harmont/toolchains"; -const ALL_APT = [ - "curl", - "ca-certificates", - "build-essential", - "pkg-config", - "libssl-dev", - "python3", - "python3-venv", -] as const; +const base = aptBase({ + packages: [ + "curl", + "ca-certificates", + "build-essential", + "pkg-config", + "libssl-dev", + "python3", + "python3-venv", + ], +}); -const base = aptBase({ packages: ALL_APT }); -const rustProject = rust({ path: ".", base }); +const rustProject = rust.project({ path: ".", base, testFlags: ["--lib"] }); const pyProject = py.uv({ path: "dsls/harmont-py", base }); -const warm = rustProject.warmup({ cache: onChange("Cargo.lock") }); - const pipelines: PipelineDefinition[] = [ { slug: "ci", triggers: [push({ branch: "main" }), pullRequest({ branches: ["main"] })], pipeline: pipeline( - warm.sh( - `. $HOME/.cargo/env && cd . && cargo test --workspace --lib --locked --no-fail-fast`, - { label: ":rust: test" }, - ), - warm.sh( - `. $HOME/.cargo/env && cd . && cargo clippy --workspace --tests --locked -- -D warnings`, - { label: ":rust: clippy" }, - ), - rustProject.fmt(), + rustProject.test, + rustProject.clippy, + rustProject.fmt, pyProject.lint(), pyProject.fmt(), pyProject.typecheck({ paths: "harmont" }), From f43fd7d095f9576fb28f41bae230cd79511845a2 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 16:15:57 -0700 Subject: [PATCH 17/41] refactor: RustProject methods with flags= instead of frozen attributes Move test/clippy/fmt from frozen Step attributes to methods with flags= parameter for per-call customization. Add hm.group() and hm.pr() alias. Dogfood targets now self-contained. --- .harmont/ci.py | 49 +++++++------- .harmont/ci.ts | 8 +-- dsls/harmont-py/harmont/__init__.py | 9 ++- dsls/harmont-py/harmont/_unwrap.py | 2 +- dsls/harmont-py/harmont/rust.py | 61 +++++++---------- dsls/harmont-py/tests/test_rust.py | 44 ++++++------ dsls/harmont-ts/src/toolchains/rust.ts | 67 ++++++++----------- dsls/harmont-ts/tests/toolchains/rust.test.ts | 43 ++++++------ 8 files changed, 134 insertions(+), 149 deletions(-) diff --git a/.harmont/ci.py b/.harmont/ci.py index 9ba5ae66..2170fb11 100644 --- a/.harmont/ci.py +++ b/.harmont/ci.py @@ -2,8 +2,6 @@ from __future__ import annotations import harmont as hm -from harmont.py.uv import UvProject -from harmont.rust import RustProject @hm.target() @@ -20,13 +18,29 @@ def shared_base() -> hm.Step: @hm.target() -def rust_project(shared_base: hm.Target[hm.Step]) -> RustProject: - return hm.rust.project(path=".", base=shared_base, test_flags=("--lib",)) +def rust_project(shared_base: hm.Target[hm.Step]) -> tuple[hm.Step, ...]: + project = hm.rust.project(path=".", base=shared_base) + return hm.group([ + project.test(flags=("--lib",)), + project.clippy(), + project.fmt(), + ]) @hm.target() -def py_project(shared_base: hm.Target[hm.Step]) -> UvProject: - return hm.py.uv(path="dsls/harmont-py", base=shared_base) +def py_project(shared_base: hm.Target[hm.Step]) -> tuple[hm.Step, ...]: + project = hm.py.uv(path="dsls/harmont-py", base=shared_base) + return hm.group([ + project.lint(), + project.fmt(), + project.typecheck(paths="harmont"), + project.run( + "pytest -v" + " --deselect tests/test_gradle.py" + " --deselect tests/test_haskell.py", + label=":python: test", + ), + ]) @hm.pipeline( @@ -35,24 +49,11 @@ def py_project(shared_base: hm.Target[hm.Step]) -> UvProject: default_image="ubuntu:24.04", triggers=[ hm.push(branch="main"), - hm.pull_request(branches="main"), + hm.pr(branches="main"), ], ) def ci( - rust_project: hm.Target[RustProject], - py_project: hm.Target[UvProject], -) -> tuple[hm.Step, ...]: - return ( - rust_project.test, - rust_project.clippy, - rust_project.fmt, - py_project.lint(), - py_project.fmt(), - py_project.typecheck(paths="harmont"), - py_project.run( - "pytest -v" - " --deselect tests/test_gradle.py" - " --deselect tests/test_haskell.py", - label=":python: test", - ), - ) + rust_project: hm.Target[tuple[hm.Step, ...]], + py_project: hm.Target[tuple[hm.Step, ...]], +) -> list: + return [rust_project, py_project] diff --git a/.harmont/ci.ts b/.harmont/ci.ts index d845560b..8914786d 100644 --- a/.harmont/ci.ts +++ b/.harmont/ci.ts @@ -19,7 +19,7 @@ const base = aptBase({ ], }); -const rustProject = rust.project({ path: ".", base, testFlags: ["--lib"] }); +const rustProject = rust.project({ path: ".", base }); const pyProject = py.uv({ path: "dsls/harmont-py", base }); const pipelines: PipelineDefinition[] = [ @@ -27,9 +27,9 @@ const pipelines: PipelineDefinition[] = [ slug: "ci", triggers: [push({ branch: "main" }), pullRequest({ branches: ["main"] })], pipeline: pipeline( - rustProject.test, - rustProject.clippy, - rustProject.fmt, + rustProject.test({ flags: ["--lib"] }), + rustProject.clippy(), + rustProject.fmt(), pyProject.lint(), pyProject.fmt(), pyProject.typecheck({ paths: "harmont" }), diff --git a/dsls/harmont-py/harmont/__init__.py b/dsls/harmont-py/harmont/__init__.py index 2b544dbc..67a08328 100644 --- a/dsls/harmont-py/harmont/__init__.py +++ b/dsls/harmont-py/harmont/__init__.py @@ -59,7 +59,7 @@ from .python import python from .ruby import ruby from .rust import RustProject, rust -from .triggers import pull_request, push, schedule +from .triggers import pull_request, pull_request as pr, push, schedule from .types import Pipeline from .zig import zig @@ -128,6 +128,11 @@ def sh( ) +def group(steps: list[Step] | tuple[Step, ...]) -> tuple[Step, ...]: + """Combine steps into a group for use as a target return value.""" + return tuple(steps) + + __all__ = [ "BaseImage", "CacheCompose", @@ -154,6 +159,7 @@ def sh( "forever", "go", "gradle", + "group", "haskell", "npm", "ocaml", @@ -161,6 +167,7 @@ def sh( "perl", "pipeline", "pipeline_to_json", + "pr", "pull_request", "push", "py", diff --git a/dsls/harmont-py/harmont/_unwrap.py b/dsls/harmont-py/harmont/_unwrap.py index b69045dd..3155b302 100644 --- a/dsls/harmont-py/harmont/_unwrap.py +++ b/dsls/harmont-py/harmont/_unwrap.py @@ -26,7 +26,7 @@ def _one(obj: object) -> tuple[Step, ...]: if isinstance(obj, Step): return (obj,) if isinstance(obj, RustProject): - return (obj.test, obj.clippy, obj.fmt) + return (obj.test(), obj.clippy(), obj.fmt()) if isinstance(obj, HaskellPackage): return (obj.build(),) if isinstance(obj, RustToolchain): diff --git a/dsls/harmont-py/harmont/rust.py b/dsls/harmont-py/harmont/rust.py index f4167bf0..9caa102c 100644 --- a/dsls/harmont-py/harmont/rust.py +++ b/dsls/harmont-py/harmont/rust.py @@ -90,9 +90,30 @@ class RustProject: toolchain: RustToolchain warmup: Step - test: Step - clippy: Step - fmt: Step + + def test(self, *, flags: tuple[str, ...] = (), **kw: Any) -> Step: + extra = (" " + " ".join(flags)) if flags else "" + return self.warmup.sh( + self.toolchain._wrap(f"cargo test --workspace --locked{extra}"), + label=kw.pop("label", ":rust: test"), + **kw, + ) + + def clippy(self, *, flags: tuple[str, ...] = (), **kw: Any) -> Step: + extra = (" " + " ".join(flags)) if flags else "" + return self.warmup.sh( + self.toolchain._wrap( + f"cargo clippy --workspace --tests --locked{extra} -- -D warnings" + ), + label=kw.pop("label", ":rust: clippy"), + **kw, + ) + + def fmt(self, *, flags: tuple[str, ...] = (), **kw: Any) -> Step: + extra = (" " + " ".join(flags)) if flags else "" + return self.toolchain._emit( + f"cargo fmt --check{extra}", ":rust: fmt", **kw + ) def _make_rust( @@ -130,9 +151,6 @@ def _make_rust_project( components: tuple[str, ...] = ("clippy", "rustfmt"), base: Step | None = None, cache: CachePolicy | None = None, - test_flags: tuple[str, ...] = (), - clippy_flags: tuple[str, ...] = (), - fmt_flags: tuple[str, ...] = (), ) -> RustProject: tc = _make_rust( path=path, @@ -151,30 +169,7 @@ def _make_rust_project( cache=warmup_cache, ) - test_extra = (" " + " ".join(test_flags)) if test_flags else "" - test_step = warm.sh( - tc._wrap(f"cargo test --workspace --locked{test_extra}"), - label=":rust: test", - ) - - clippy_extra = (" " + " ".join(clippy_flags)) if clippy_flags else "" - clippy_step = warm.sh( - tc._wrap( - f"cargo clippy --workspace --tests --locked{clippy_extra} -- -D warnings" - ), - label=":rust: clippy", - ) - - fmt_extra = (" " + " ".join(fmt_flags)) if fmt_flags else "" - fmt_step = tc._emit(f"cargo fmt --check{fmt_extra}", ":rust: fmt") - - return RustProject( - toolchain=tc, - warmup=warm, - test=test_step, - clippy=clippy_step, - fmt=fmt_step, - ) + return RustProject(toolchain=tc, warmup=warm) class _RustEntry: @@ -206,9 +201,6 @@ def project( components: tuple[str, ...] = ("clippy", "rustfmt"), base: Step | None = None, cache: CachePolicy | None = None, - test_flags: tuple[str, ...] = (), - clippy_flags: tuple[str, ...] = (), - fmt_flags: tuple[str, ...] = (), ) -> RustProject: return _make_rust_project( path=path, @@ -217,9 +209,6 @@ def project( components=components, base=base, cache=cache, - test_flags=test_flags, - clippy_flags=clippy_flags, - fmt_flags=fmt_flags, ) diff --git a/dsls/harmont-py/tests/test_rust.py b/dsls/harmont-py/tests/test_rust.py index e13d7905..6ccd0648 100644 --- a/dsls/harmont-py/tests/test_rust.py +++ b/dsls/harmont-py/tests/test_rust.py @@ -164,12 +164,12 @@ def test_warmup_in_pipeline(self): class TestRustProject: - def test_project_has_all_steps(self): + def test_project_has_all_methods(self): proj = hm.rust.project(path="cli") assert proj.warmup.cmd is not None - assert proj.test.cmd is not None - assert proj.clippy.cmd is not None - assert proj.fmt.cmd is not None + assert proj.test().cmd is not None + assert proj.clippy().cmd is not None + assert proj.fmt().cmd is not None def test_warmup_implicit_cache_on_change(self): proj = hm.rust.project(path="cli") @@ -186,39 +186,39 @@ def test_warmup_cache_override(self): def test_test_command(self): proj = hm.rust.project(path="cli") - assert "cargo test --workspace --locked" in proj.test.cmd + assert "cargo test --workspace --locked" in proj.test().cmd def test_test_flags(self): - proj = hm.rust.project(path=".", test_flags=("--lib", "--no-fail-fast")) - assert "cargo test --workspace --locked --lib --no-fail-fast" in proj.test.cmd + proj = hm.rust.project(path=".") + assert "cargo test --workspace --locked --lib --no-fail-fast" in proj.test(flags=("--lib", "--no-fail-fast")).cmd def test_clippy_command(self): proj = hm.rust.project(path="cli") - assert "cargo clippy --workspace --tests --locked -- -D warnings" in proj.clippy.cmd + assert "cargo clippy --workspace --tests --locked -- -D warnings" in proj.clippy().cmd def test_clippy_flags(self): - proj = hm.rust.project(path=".", clippy_flags=("--fix",)) - assert "cargo clippy --workspace --tests --locked --fix -- -D warnings" in proj.clippy.cmd + proj = hm.rust.project(path=".") + assert "cargo clippy --workspace --tests --locked --fix -- -D warnings" in proj.clippy(flags=("--fix",)).cmd def test_fmt_command(self): proj = hm.rust.project(path="cli") - assert "cargo fmt --check" in proj.fmt.cmd + assert "cargo fmt --check" in proj.fmt().cmd def test_fmt_flags(self): - proj = hm.rust.project(path=".", fmt_flags=("--all",)) - assert "cargo fmt --check --all" in proj.fmt.cmd + proj = hm.rust.project(path=".") + assert "cargo fmt --check --all" in proj.fmt(flags=("--all",)).cmd def test_test_chains_off_warmup(self): proj = hm.rust.project(path=".") - assert proj.test.parent is proj.warmup + assert proj.test().parent is proj.warmup def test_clippy_chains_off_warmup(self): proj = hm.rust.project(path=".") - assert proj.clippy.parent is proj.warmup + assert proj.clippy().parent is proj.warmup def test_fmt_chains_off_install(self): proj = hm.rust.project(path=".") - assert proj.fmt.parent is proj.toolchain.installed + assert proj.fmt().parent is proj.toolchain.installed def test_toolchain_escape_hatch(self): proj = hm.rust.project(path="cli") @@ -228,7 +228,7 @@ def test_toolchain_escape_hatch(self): def test_with_base_skips_apt(self): base = hm.scratch().sh("custom base", label="base") proj = hm.rust.project(path="cli", base=base) - p = hm.pipeline(proj.test, proj.clippy, proj.fmt, default_image="ubuntu:24.04") + p = hm.pipeline(proj.test(), proj.clippy(), proj.fmt(), default_image="ubuntu:24.04") cmds = _cmds(p) assert not any("apt-get install" in c for c in cmds) assert any("custom base" in c for c in cmds) @@ -236,13 +236,13 @@ def test_with_base_skips_apt(self): def test_labels(self): proj = hm.rust.project(path=".") assert proj.warmup.label == ":rust: warmup" - assert proj.test.label == ":rust: test" - assert proj.clippy.label == ":rust: clippy" - assert proj.fmt.label == ":rust: fmt" + assert proj.test().label == ":rust: test" + assert proj.clippy().label == ":rust: clippy" + assert proj.fmt().label == ":rust: fmt" def test_pipeline_ir(self): proj = hm.rust.project(path="cli") - p = hm.pipeline(proj.test, proj.clippy, proj.fmt, default_image="ubuntu:24.04") + p = hm.pipeline(proj.test(), proj.clippy(), proj.fmt(), default_image="ubuntu:24.04") cmds = _cmds(p) assert any("cargo build --workspace --tests --locked" in c for c in cmds) assert any("cargo test --workspace --locked" in c for c in cmds) @@ -253,6 +253,6 @@ def test_pipeline_ir(self): def test_version_forwarded(self): proj = hm.rust.project(path=".", version="1.81.0") - p = hm.pipeline(proj.test) + p = hm.pipeline(proj.test()) rustup = _step_by_substring(p, "sh.rustup.rs") assert "--default-toolchain 1.81.0" in rustup["cmd"] diff --git a/dsls/harmont-ts/src/toolchains/rust.ts b/dsls/harmont-ts/src/toolchains/rust.ts index b0a1e5eb..34790061 100644 --- a/dsls/harmont-ts/src/toolchains/rust.ts +++ b/dsls/harmont-ts/src/toolchains/rust.ts @@ -21,9 +21,6 @@ export interface RustToolchainOptions { export interface RustProjectOptions extends RustToolchainOptions { readonly cache?: CachePolicy; - readonly testFlags?: readonly string[]; - readonly clippyFlags?: readonly string[]; - readonly fmtFlags?: readonly string[]; } type ActionOptions = Omit; @@ -86,22 +83,35 @@ export class RustToolchain { export class RustProject { readonly toolchain: RustToolchain; readonly warmup: Step; - readonly test: Step; - readonly clippy: Step; - readonly fmt: Step; - - constructor( - toolchain: RustToolchain, - warmup: Step, - test: Step, - clippy: Step, - fmt: Step, - ) { + + constructor(toolchain: RustToolchain, warmup: Step) { this.toolchain = toolchain; this.warmup = warmup; - this.test = test; - this.clippy = clippy; - this.fmt = fmt; + } + + test(opts?: { flags?: readonly string[] } & ActionOptions): Step { + const extra = opts?.flags?.length ? " " + opts.flags.join(" ") : ""; + return this.warmup.sh( + `. $HOME/.cargo/env && cd ${this.toolchain.path} && cargo test --workspace --locked${extra}`, + { label: ":rust: test", ...opts }, + ); + } + + clippy(opts?: { flags?: readonly string[] } & ActionOptions): Step { + const extra = opts?.flags?.length ? " " + opts.flags.join(" ") : ""; + return this.warmup.sh( + `. $HOME/.cargo/env && cd ${this.toolchain.path} && cargo clippy --workspace --tests --locked${extra} -- -D warnings`, + { label: ":rust: clippy", ...opts }, + ); + } + + fmt(opts?: { flags?: readonly string[] } & ActionOptions): Step { + const extra = opts?.flags?.length ? " " + opts.flags.join(" ") : ""; + return this.toolchain._cargo( + `cargo fmt --check${extra}`, + ":rust: fmt", + opts, + ); } } @@ -149,28 +159,7 @@ function makeProject(opts?: RustProjectOptions): RustProject { { cache: warmupCache }, ); - const testExtra = opts?.testFlags?.length - ? " " + opts.testFlags.join(" ") - : ""; - const testStep = warm.sh( - `. $HOME/.cargo/env && cd ${path} && cargo test --workspace --locked${testExtra}`, - { label: ":rust: test" }, - ); - - const clippyExtra = opts?.clippyFlags?.length - ? " " + opts.clippyFlags.join(" ") - : ""; - const clippyStep = warm.sh( - `. $HOME/.cargo/env && cd ${path} && cargo clippy --workspace --tests --locked${clippyExtra} -- -D warnings`, - { label: ":rust: clippy" }, - ); - - const fmtExtra = opts?.fmtFlags?.length - ? " " + opts.fmtFlags.join(" ") - : ""; - const fmtStep = tc._cargo(`cargo fmt --check${fmtExtra}`, ":rust: fmt"); - - return new RustProject(tc, warm, testStep, clippyStep, fmtStep); + return new RustProject(tc, warm); } export const rust = { diff --git a/dsls/harmont-ts/tests/toolchains/rust.test.ts b/dsls/harmont-ts/tests/toolchains/rust.test.ts index acc99f58..79e9561f 100644 --- a/dsls/harmont-ts/tests/toolchains/rust.test.ts +++ b/dsls/harmont-ts/tests/toolchains/rust.test.ts @@ -151,16 +151,16 @@ describe("rust.toolchain", () => { }); describe("rust.project", () => { - it("has all steps", () => { + it("has all methods", () => { const proj = rust.project({ path: "cli" }); expect(proj.warmup._cmd).toContain( "cargo build --workspace --tests --locked", ); - expect(proj.test._cmd).toContain("cargo test --workspace --locked"); - expect(proj.clippy._cmd).toContain( + expect(proj.test()._cmd).toContain("cargo test --workspace --locked"); + expect(proj.clippy()._cmd).toContain( "cargo clippy --workspace --tests --locked", ); - expect(proj.fmt._cmd).toContain("cargo fmt --check"); + expect(proj.fmt()._cmd).toContain("cargo fmt --check"); }); it("warmup has implicit CacheOnChange on Cargo.lock", () => { @@ -191,54 +191,53 @@ describe("rust.project", () => { }); it("test flags are appended", () => { - const proj = rust.project({ - path: ".", - testFlags: ["--lib", "--no-fail-fast"], - }); - expect(proj.test._cmd).toContain( + const proj = rust.project({ path: "." }); + expect(proj.test({ flags: ["--lib", "--no-fail-fast"] })._cmd).toContain( "cargo test --workspace --locked --lib --no-fail-fast", ); }); it("clippy flags are inserted before --", () => { - const proj = rust.project({ path: ".", clippyFlags: ["--fix"] }); - expect(proj.clippy._cmd).toContain( + const proj = rust.project({ path: "." }); + expect(proj.clippy({ flags: ["--fix"] })._cmd).toContain( "cargo clippy --workspace --tests --locked --fix -- -D warnings", ); }); it("fmt flags are appended", () => { - const proj = rust.project({ path: ".", fmtFlags: ["--all"] }); - expect(proj.fmt._cmd).toContain("cargo fmt --check --all"); + const proj = rust.project({ path: "." }); + expect(proj.fmt({ flags: ["--all"] })._cmd).toContain( + "cargo fmt --check --all", + ); }); it("test chains off warmup", () => { const proj = rust.project(); - expect(proj.test._parent).toBe(proj.warmup); + expect(proj.test()._parent).toBe(proj.warmup); }); it("clippy chains off warmup", () => { const proj = rust.project(); - expect(proj.clippy._parent).toBe(proj.warmup); + expect(proj.clippy()._parent).toBe(proj.warmup); }); it("fmt chains off install (not warmup)", () => { const proj = rust.project(); - expect(proj.fmt._parent).toBe(proj.toolchain.install()); + expect(proj.fmt()._parent).toBe(proj.toolchain.install()); }); it("labels are correct", () => { const proj = rust.project(); expect(proj.warmup._label).toBe(":rust: warmup"); - expect(proj.test._label).toBe(":rust: test"); - expect(proj.clippy._label).toBe(":rust: clippy"); - expect(proj.fmt._label).toBe(":rust: fmt"); + expect(proj.test()._label).toBe(":rust: test"); + expect(proj.clippy()._label).toBe(":rust: clippy"); + expect(proj.fmt()._label).toBe(":rust: fmt"); }); it("with base skips apt", () => { const base = sh("custom base"); const proj = rust.project({ path: "cli", base }); - const ir = pipeline(proj.test, proj.clippy, proj.fmt, { + const ir = pipeline(proj.test(), proj.clippy(), proj.fmt(), { defaultImage: "ubuntu:24.04", }); const c = cmds(ir); @@ -250,7 +249,7 @@ describe("rust.project", () => { it("produces valid pipeline IR", () => { const proj = rust.project({ path: "cli" }); - const ir = pipeline(proj.test, proj.clippy, proj.fmt, { + const ir = pipeline(proj.test(), proj.clippy(), proj.fmt(), { defaultImage: "ubuntu:24.04", }); expect(ir.version).toBe("0"); @@ -267,7 +266,7 @@ describe("rust.project", () => { it("version forwarded", () => { const proj = rust.project({ path: ".", version: "1.81.0" }); - const ir = pipeline(proj.test); + const ir = pipeline(proj.test()); const rustup = stepBySubstring(ir, "sh.rustup.rs"); expect(rustup.cmd).toContain("--default-toolchain 1.81.0"); }); From efa214c7a0039a6606ef1f6c059029b3980e9a4a Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 16:16:51 -0700 Subject: [PATCH 18/41] fix: update Python Rust example to use rust.toolchain() --- examples/rust/.harmont/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/rust/.harmont/pipeline.py b/examples/rust/.harmont/pipeline.py index 1ee15f80..0cca8c32 100644 --- a/examples/rust/.harmont/pipeline.py +++ b/examples/rust/.harmont/pipeline.py @@ -7,7 +7,7 @@ @hm.target() def project() -> RustToolchain: - return hm.rust(path=".") + return hm.rust.toolchain(path=".") @hm.pipeline( From 508eebf929fe7233189b9666e84aec52423df2ac Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 16:18:11 -0700 Subject: [PATCH 19/41] fix: ruff import sorting + SLF001 noqa for friend access --- dsls/harmont-py/harmont/__init__.py | 3 ++- dsls/harmont-py/harmont/rust.py | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/dsls/harmont-py/harmont/__init__.py b/dsls/harmont-py/harmont/__init__.py index 67a08328..d39f3d01 100644 --- a/dsls/harmont-py/harmont/__init__.py +++ b/dsls/harmont-py/harmont/__init__.py @@ -59,7 +59,8 @@ from .python import python from .ruby import ruby from .rust import RustProject, rust -from .triggers import pull_request, pull_request as pr, push, schedule +from .triggers import pull_request, push, schedule +from .triggers import pull_request as pr from .types import Pipeline from .zig import zig diff --git a/dsls/harmont-py/harmont/rust.py b/dsls/harmont-py/harmont/rust.py index 9caa102c..a9c90322 100644 --- a/dsls/harmont-py/harmont/rust.py +++ b/dsls/harmont-py/harmont/rust.py @@ -94,7 +94,7 @@ class RustProject: def test(self, *, flags: tuple[str, ...] = (), **kw: Any) -> Step: extra = (" " + " ".join(flags)) if flags else "" return self.warmup.sh( - self.toolchain._wrap(f"cargo test --workspace --locked{extra}"), + self.toolchain._wrap(f"cargo test --workspace --locked{extra}"), # noqa: SLF001 label=kw.pop("label", ":rust: test"), **kw, ) @@ -102,7 +102,7 @@ def test(self, *, flags: tuple[str, ...] = (), **kw: Any) -> Step: def clippy(self, *, flags: tuple[str, ...] = (), **kw: Any) -> Step: extra = (" " + " ".join(flags)) if flags else "" return self.warmup.sh( - self.toolchain._wrap( + self.toolchain._wrap( # noqa: SLF001 f"cargo clippy --workspace --tests --locked{extra} -- -D warnings" ), label=kw.pop("label", ":rust: clippy"), @@ -111,7 +111,7 @@ def clippy(self, *, flags: tuple[str, ...] = (), **kw: Any) -> Step: def fmt(self, *, flags: tuple[str, ...] = (), **kw: Any) -> Step: extra = (" " + " ".join(flags)) if flags else "" - return self.toolchain._emit( + return self.toolchain._emit( # noqa: SLF001 f"cargo fmt --check{extra}", ":rust: fmt", **kw ) @@ -163,7 +163,7 @@ def _make_rust_project( lock_path = f"{path}/Cargo.lock" if path != "." else "Cargo.lock" warmup_cache = cache if cache is not None else CacheOnChange(paths=(lock_path,)) - warm = tc._emit( + warm = tc._emit( # noqa: SLF001 "cargo build --workspace --tests --locked", ":rust: warmup", cache=warmup_cache, From c68ecb4c833388cb73542d7658c6ad9264cdbfb9 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 16:20:10 -0700 Subject: [PATCH 20/41] fix: line length in Rust project tests --- dsls/harmont-py/tests/test_rust.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dsls/harmont-py/tests/test_rust.py b/dsls/harmont-py/tests/test_rust.py index 6ccd0648..4e8f70ac 100644 --- a/dsls/harmont-py/tests/test_rust.py +++ b/dsls/harmont-py/tests/test_rust.py @@ -190,7 +190,8 @@ def test_test_command(self): def test_test_flags(self): proj = hm.rust.project(path=".") - assert "cargo test --workspace --locked --lib --no-fail-fast" in proj.test(flags=("--lib", "--no-fail-fast")).cmd + step = proj.test(flags=("--lib", "--no-fail-fast")) + assert "cargo test --workspace --locked --lib --no-fail-fast" in step.cmd def test_clippy_command(self): proj = hm.rust.project(path="cli") @@ -198,7 +199,8 @@ def test_clippy_command(self): def test_clippy_flags(self): proj = hm.rust.project(path=".") - assert "cargo clippy --workspace --tests --locked --fix -- -D warnings" in proj.clippy(flags=("--fix",)).cmd + step = proj.clippy(flags=("--fix",)) + assert "cargo clippy --workspace --tests --locked --fix -- -D warnings" in step.cmd def test_fmt_command(self): proj = hm.rust.project(path="cli") From 962e4f5e590d77ee4576c6b0a48e687d8b631e29 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 16:25:25 -0700 Subject: [PATCH 21/41] style: ruff format test_rust.py --- dsls/harmont-py/tests/test_rust.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dsls/harmont-py/tests/test_rust.py b/dsls/harmont-py/tests/test_rust.py index 4e8f70ac..d6a9a889 100644 --- a/dsls/harmont-py/tests/test_rust.py +++ b/dsls/harmont-py/tests/test_rust.py @@ -35,7 +35,11 @@ def test_full_chain(self): def test_actions_share_install_step(self): tc = hm.rust.toolchain(path="cli") p = hm.pipeline( - tc.build(), tc.test(), tc.clippy(), tc.fmt(), tc.doc(), + tc.build(), + tc.test(), + tc.clippy(), + tc.fmt(), + tc.doc(), default_image="ubuntu:24.04", ) cmds = _cmds(p) From 53671e3fa2c8bb6ebd174d5bb318570ae5b161ba Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 18:34:02 -0700 Subject: [PATCH 22/41] ci: add 15min timeout to integration test job --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c7c68dc..c45ec03e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,6 +141,7 @@ jobs: integration: name: docker-gated integration test runs-on: ubuntu-latest + timeout-minutes: 15 # Skip the heavy job on draft PRs to save runner minutes. Push to # main always runs it. if: github.event_name == 'push' || (github.event_name == 'pull_request' && !github.event.pull_request.draft) From 0e65c1b73af04e68102f90af94c9bf35d8d1dfac Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 19:39:01 -0700 Subject: [PATCH 23/41] feat: add export_image, import_image, list_images_by_prefix to DockerClient --- crates/hm/src/orchestrator/docker_client.rs | 92 ++++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) diff --git a/crates/hm/src/orchestrator/docker_client.rs b/crates/hm/src/orchestrator/docker_client.rs index ae5657d3..2c86b508 100644 --- a/crates/hm/src/orchestrator/docker_client.rs +++ b/crates/hm/src/orchestrator/docker_client.rs @@ -15,7 +15,8 @@ use bollard::container::{ }; use bollard::exec::{CreateExecOptions, StartExecResults}; use bollard::image::{ - CommitContainerOptions, CreateImageOptions, ListImagesOptions, RemoveImageOptions, + CommitContainerOptions, CreateImageOptions, ImportImageOptions, ListImagesOptions, + RemoveImageOptions, }; use futures_util::StreamExt; use tokio::io::AsyncWrite; @@ -343,6 +344,95 @@ impl DockerClient { Ok(()) } + /// Export a Docker image to a tar file on disk. + /// + /// Streams the image layer data from the daemon and writes it to + /// `dest` using a buffered writer. + /// + /// # Errors + /// + /// Returns [`HmError::Docker`] if the daemon's export stream fails, + /// or an I/O error if writing to `dest` fails. + pub async fn export_image(&self, image: &str, dest: &std::path::Path) -> Result<()> { + use tokio::io::AsyncWriteExt; + + let mut stream = self.inner.export_image(image); + let file = tokio::fs::File::create(dest) + .await + .with_context(|| format!("create export file '{}'", dest.display()))?; + let mut writer = tokio::io::BufWriter::new(file); + while let Some(chunk) = stream.next().await { + let bytes = + chunk.map_err(|e| HmError::Docker(format!("export_image '{image}': {e}")))?; + writer + .write_all(&bytes) + .await + .with_context(|| format!("write export data to '{}'", dest.display()))?; + } + writer + .flush() + .await + .with_context(|| format!("flush export file '{}'", dest.display()))?; + Ok(()) + } + + /// Import a Docker image from a tar file on disk. + /// + /// Reads the full tar file into memory and loads it into the + /// daemon via the image import API. + /// + /// # Errors + /// + /// Returns [`HmError::Docker`] if the daemon rejects the import + /// stream, or an I/O error if reading `src` fails. + pub async fn import_image(&self, src: &std::path::Path) -> Result<()> { + let body = tokio::fs::read(src) + .await + .with_context(|| format!("read import file '{}'", src.display()))?; + let mut stream = self.inner.import_image( + ImportImageOptions { quiet: true }, + body.into(), + None, + ); + while let Some(item) = stream.next().await { + item.map_err(|e| { + HmError::Docker(format!("import_image '{}': {e}", src.display())) + })?; + } + Ok(()) + } + + /// List all image tags whose name starts with `prefix`. + /// + /// Uses the Docker `reference` filter with a glob pattern and then + /// post-filters the returned `repo_tags` to those that truly begin + /// with `prefix`. The result is sorted lexicographically. + /// + /// # Errors + /// + /// Returns [`HmError::Docker`] if the `list_images` API call + /// fails (daemon unreachable, malformed filter). + pub async fn list_images_by_prefix(&self, prefix: &str) -> Result> { + let mut filters = HashMap::new(); + filters.insert("reference".to_string(), vec![format!("{prefix}*")]); + let images = self + .inner + .list_images(Some(ListImagesOptions { + filters, + ..Default::default() + })) + .await + .map_err(|e| HmError::Docker(format!("list_images: {e}")))?; + let mut tags: Vec = images + .iter() + .flat_map(|img| &img.repo_tags) + .filter(|tag| tag.starts_with(prefix)) + .cloned() + .collect(); + tags.sort(); + Ok(tags) + } + pub async fn stop_remove(&self, container_id: &str) { let _ = self .inner From 8fe4e5f180ebda393816e055619b2a5d01de4a16 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 19:40:50 -0700 Subject: [PATCH 24/41] feat: add cache manifest types and filename helpers --- crates/hm/src/commands/cache/manifest.rs | 105 +++++++++++++++++++++++ crates/hm/src/commands/cache/mod.rs | 6 ++ crates/hm/src/commands/cache/restore.rs | 11 +++ crates/hm/src/commands/cache/save.rs | 11 +++ crates/hm/src/commands/mod.rs | 1 + 5 files changed, 134 insertions(+) create mode 100644 crates/hm/src/commands/cache/manifest.rs create mode 100644 crates/hm/src/commands/cache/mod.rs create mode 100644 crates/hm/src/commands/cache/restore.rs create mode 100644 crates/hm/src/commands/cache/save.rs diff --git a/crates/hm/src/commands/cache/manifest.rs b/crates/hm/src/commands/cache/manifest.rs new file mode 100644 index 00000000..7e3be499 --- /dev/null +++ b/crates/hm/src/commands/cache/manifest.rs @@ -0,0 +1,105 @@ +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Manifest { + pub version: u32, + pub images: BTreeMap, +} + +impl Manifest { + #[must_use] + pub const fn new() -> Self { + Self { + version: 1, + images: BTreeMap::new(), + } + } + + /// SHA-256 content hash of the JSON-serialized manifest, truncated to 16 + /// hex characters. + /// + /// # Panics + /// + /// Panics if the manifest cannot be serialized to JSON (should never + /// happen for this type). + #[must_use] + #[allow(clippy::expect_used)] + pub fn content_hash(&self) -> String { + let json = serde_json::to_string(self).expect("manifest serialization cannot fail"); + let hash = Sha256::digest(json.as_bytes()); + hex::encode(&hash[..8]) + } +} + +/// Convert a Docker image tag to the corresponding tar filename. +/// +/// `"harmont-local/base:a1b2c3d4"` → `"base--a1b2c3d4.tar"` +#[must_use] +pub fn tar_name_for_tag(tag: &str) -> String { + let stripped = tag.strip_prefix("harmont-local/").unwrap_or(tag); + format!("{}.tar", stripped.replace(':', "--")) +} + +/// Inverse of [`tar_name_for_tag`]. +/// +/// `"base--a1b2c3d4.tar"` → `Some("harmont-local/base:a1b2c3d4")` +#[must_use] +pub fn tag_from_tar_name(filename: &str) -> Option { + let stem = filename.strip_suffix(".tar")?; + let (name, hash) = stem.split_once("--")?; + Some(format!("harmont-local/{name}:{hash}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tar_filename_from_tag() { + assert_eq!( + tar_name_for_tag("harmont-local/base:a1b2c3d4"), + "base--a1b2c3d4.tar" + ); + } + + #[test] + fn tag_from_tar_filename() { + assert_eq!( + tag_from_tar_name("base--a1b2c3d4.tar"), + Some("harmont-local/base:a1b2c3d4".to_string()) + ); + } + + #[test] + fn tag_from_bad_filename_returns_none() { + assert_eq!(tag_from_tar_name("random-file.tar"), None); + assert_eq!(tag_from_tar_name("no-extension"), None); + } + + #[test] + fn manifest_round_trip() { + let mut m = Manifest::new(); + m.images + .insert("base".to_string(), "harmont-local/base:abc123".to_string()); + + let json = serde_json::to_string(&m).unwrap(); + let m2: Manifest = serde_json::from_str(&json).unwrap(); + assert_eq!(m, m2); + } + + #[test] + fn manifest_content_hash_is_deterministic() { + let mut m = Manifest::new(); + m.images + .insert("step1".to_string(), "harmont-local/step1:deadbeef".to_string()); + + let h1 = m.content_hash(); + let h2 = m.content_hash(); + assert_eq!(h1, h2); + assert_eq!(h1.len(), 16); + assert!(h1.chars().all(|c| c.is_ascii_hexdigit())); + } +} diff --git a/crates/hm/src/commands/cache/mod.rs b/crates/hm/src/commands/cache/mod.rs new file mode 100644 index 00000000..c011ea79 --- /dev/null +++ b/crates/hm/src/commands/cache/mod.rs @@ -0,0 +1,6 @@ +pub mod manifest; +mod restore; +mod save; + +pub use restore::handle_restore; +pub use save::handle_save; diff --git a/crates/hm/src/commands/cache/restore.rs b/crates/hm/src/commands/cache/restore.rs new file mode 100644 index 00000000..16c81657 --- /dev/null +++ b/crates/hm/src/commands/cache/restore.rs @@ -0,0 +1,11 @@ +use std::path::Path; + +use anyhow::Result; + +/// # Errors +/// +/// Returns an error if the restore operation fails. +#[allow(clippy::todo)] +pub async fn handle_restore(_dir: &Path) -> Result { + todo!() +} diff --git a/crates/hm/src/commands/cache/save.rs b/crates/hm/src/commands/cache/save.rs new file mode 100644 index 00000000..d0076865 --- /dev/null +++ b/crates/hm/src/commands/cache/save.rs @@ -0,0 +1,11 @@ +use std::path::Path; + +use anyhow::Result; + +/// # Errors +/// +/// Returns an error if the save operation fails. +#[allow(clippy::todo)] +pub async fn handle_save(_dir: &Path) -> Result { + todo!() +} diff --git a/crates/hm/src/commands/mod.rs b/crates/hm/src/commands/mod.rs index cead5fb1..957e4112 100644 --- a/crates/hm/src/commands/mod.rs +++ b/crates/hm/src/commands/mod.rs @@ -1,2 +1,3 @@ +pub mod cache; pub mod dev; pub mod run; From e9b92968965d6799a39ef72489a50e2c1d2ba573 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 19:42:42 -0700 Subject: [PATCH 25/41] feat: implement hm cache save --- crates/hm/src/commands/cache/save.rs | 72 ++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 5 deletions(-) diff --git a/crates/hm/src/commands/cache/save.rs b/crates/hm/src/commands/cache/save.rs index d0076865..1fe196ba 100644 --- a/crates/hm/src/commands/cache/save.rs +++ b/crates/hm/src/commands/cache/save.rs @@ -1,11 +1,73 @@ use std::path::Path; -use anyhow::Result; +use anyhow::{Context, Result}; +use tracing::info; +use super::manifest::{self, Manifest}; +use crate::orchestrator::docker_client::DockerClient; + +/// Save all `harmont-local/*` images to a cache directory as tar files, +/// write a manifest, and prune stale tars that no longer correspond to +/// any known image. +/// +/// Prints the manifest's content hash to stdout so CI runners (e.g. +/// GitHub Actions) can capture it for use as a cache key. +/// /// # Errors /// -/// Returns an error if the save operation fails. -#[allow(clippy::todo)] -pub async fn handle_save(_dir: &Path) -> Result { - todo!() +/// Returns an error if the Docker daemon is unreachable, an image +/// export fails, or any filesystem operation on `dir` fails. +pub async fn handle_save(dir: &Path) -> Result { + let docker = DockerClient::connect()?; + docker.ping().await?; + + tokio::fs::create_dir_all(dir) + .await + .with_context(|| format!("create cache dir {}", dir.display()))?; + + // 1. List all harmont-local/* images (list_images_by_prefix already skips ephemeral) + let tags = docker.list_images_by_prefix("harmont-local/").await?; + + let mut manifest = Manifest::new(); + + // 2. For each image, save to tar if not already on disk + for tag in &tags { + let filename = manifest::tar_name_for_tag(tag); + let tar_path = dir.join(&filename); + + if tar_path.exists() { + info!("skip (exists): {filename}"); + } else { + info!("save: {tag} → {filename}"); + docker.export_image(tag, &tar_path).await?; + } + + manifest.images.insert(filename, tag.clone()); + } + + // 3. Write manifest.json + let manifest_json = serde_json::to_string_pretty(&manifest)?; + tokio::fs::write(dir.join("manifest.json"), &manifest_json) + .await + .context("write manifest.json")?; + + // 4. Prune stale tars + let mut entries = tokio::fs::read_dir(dir).await?; + while let Some(entry) = entries.next_entry().await? { + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + if name_str.ends_with(".tar") && !manifest.images.contains_key(name_str.as_ref()) { + info!("prune stale: {name_str}"); + tokio::fs::remove_file(entry.path()).await.ok(); + } + } + + // 5. Print content hash to stdout (GHA captures this for the cache key) + let hash = manifest.content_hash(); + #[allow(clippy::print_stdout, reason = "hash must go to stdout for CI capture")] + { + println!("{hash}"); + } + + Ok(0) } From d57416f392b86ecbf6bc5a4c4a78068e12db1df9 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 19:44:20 -0700 Subject: [PATCH 26/41] feat: implement hm cache restore --- crates/hm/src/commands/cache/restore.rs | 72 +++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 5 deletions(-) diff --git a/crates/hm/src/commands/cache/restore.rs b/crates/hm/src/commands/cache/restore.rs index 16c81657..c56213fa 100644 --- a/crates/hm/src/commands/cache/restore.rs +++ b/crates/hm/src/commands/cache/restore.rs @@ -1,11 +1,73 @@ use std::path::Path; -use anyhow::Result; +use anyhow::{Context, Result}; +use tracing::{info, warn}; +use super::manifest; +use crate::orchestrator::docker_client::DockerClient; + +/// Restore cached Docker images from tar files in the given directory. +/// +/// Each `.tar` file is mapped back to its `harmont-local/*` tag via +/// [`manifest::tag_from_tar_name`]. Images that already exist in the +/// local Docker daemon are skipped. +/// /// # Errors /// -/// Returns an error if the restore operation fails. -#[allow(clippy::todo)] -pub async fn handle_restore(_dir: &Path) -> Result { - todo!() +/// Returns an error if the Docker daemon is unreachable or a filesystem +/// operation on `dir` fails. +#[allow(clippy::print_stderr, reason = "status summary must go to stderr for CI visibility")] +pub async fn handle_restore(dir: &Path) -> Result { + let docker = DockerClient::connect()?; + docker.ping().await?; + + if !dir.exists() { + info!("cache dir does not exist, nothing to restore"); + eprintln!("restored 0/0 images (cache dir missing)"); + return Ok(0); + } + + // Scan for .tar files + let mut tars = Vec::new(); + let mut entries = tokio::fs::read_dir(dir) + .await + .with_context(|| format!("read cache dir {}", dir.display()))?; + while let Some(entry) = entries.next_entry().await? { + let name = entry.file_name(); + let name_str = name.to_string_lossy().to_string(); + if std::path::Path::new(&name_str) + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("tar")) + { + tars.push((name_str, entry.path())); + } + } + + let total = tars.len(); + let mut restored = 0u32; + let mut skipped = 0u32; + + for (filename, tar_path) in &tars { + let Some(tag) = manifest::tag_from_tar_name(filename) else { + warn!("skip unrecognized tar: {filename}"); + continue; + }; + + if docker.image_exists(&tag).await? { + info!("skip (present): {tag}"); + skipped += 1; + continue; + } + + info!("restore: {filename} → {tag}"); + match docker.import_image(tar_path).await { + Ok(()) => restored += 1, + Err(e) => { + warn!("failed to load {filename}: {e}"); + } + } + } + + eprintln!("restored {restored}/{total} images ({skipped} already present)"); + Ok(0) } From 739b8b51a99be2f1c5f5503559d956b081d1fd8e Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 19:45:31 -0700 Subject: [PATCH 27/41] feat: wire hm cache save/restore CLI subcommands --- crates/hm/src/cli/mod.rs | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/crates/hm/src/cli/mod.rs b/crates/hm/src/cli/mod.rs index 80c9d07f..8cf5e0e1 100644 --- a/crates/hm/src/cli/mod.rs +++ b/crates/hm/src/cli/mod.rs @@ -7,6 +7,8 @@ pub use dev::{DevCommand, DevDownArgs, DevExecArgs, DevLogsArgs, DevPortOfArgs, pub use plugin::PluginCommand; pub use run::RunArgs; +use std::path::PathBuf; + use anyhow::Result; use clap::{Parser, Subcommand}; @@ -58,11 +60,35 @@ pub enum Command { #[command(subcommand)] Dev(DevCommand), + /// Manage harmont Docker image cache. + #[command(subcommand)] + Cache(CacheCommand), + /// Interact with the Harmont cloud API. #[command(subcommand)] Cloud(hm_plugin_cloud::cli::CloudCommand), } +#[derive(Debug, Clone, Subcommand)] +pub enum CacheCommand { + /// Save harmont Docker images to a cache directory. + Save(CacheSaveArgs), + /// Restore harmont Docker images from a cache directory. + Restore(CacheRestoreArgs), +} + +#[derive(Debug, Clone, clap::Args)] +pub struct CacheSaveArgs { + /// Directory to save image tars into. + pub dir: PathBuf, +} + +#[derive(Debug, Clone, clap::Args)] +pub struct CacheRestoreArgs { + /// Directory containing cached image tars. + pub dir: PathBuf, +} + /// Dispatch a parsed CLI command to the appropriate handler. Returns an exit code. /// /// # Errors @@ -72,6 +98,12 @@ pub async fn dispatch(command: Command, ctx: RunContext) -> Result { match command { Command::Run(args) => crate::commands::run::handle(args, ctx).await, Command::Dev(cmd) => dev::dispatch(cmd, ctx).await, + Command::Cache(cmd) => match cmd { + CacheCommand::Save(args) => crate::commands::cache::handle_save(&args.dir).await, + CacheCommand::Restore(args) => { + crate::commands::cache::handle_restore(&args.dir).await + } + }, Command::Version => version::run().await.map(|()| 0), Command::Plugin(cmd) => plugin::run(cmd).await.map(|()| 0), Command::Cloud(cmd) => { From 65ef6e6d299691fc7560142afe75f86f72cf7f0f Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 19:45:52 -0700 Subject: [PATCH 28/41] ci: use hm cache save/restore for per-image Docker caching --- .github/workflows/ci.yml | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c45ec03e..02cd7186 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -113,17 +113,15 @@ jobs: /usr/bin/python3 -c "import harmont; print('harmont', harmont.__file__)" - name: Restore harmont Docker cache - id: docker-cache - uses: actions/cache@v4 + uses: actions/cache/restore@v4 with: - path: /tmp/harmont-docker-cache.tar - key: harmont-docker-${{ hashFiles('Cargo.lock') }} + path: .harmont-cache/ + key: harmont-v1-will-never-match restore-keys: | - harmont-docker- + harmont-v1- - name: Load cached Docker images - if: steps.docker-cache.outputs.cache-hit == 'true' - run: docker load -i /tmp/harmont-docker-cache.tar || true + run: ./target/debug/hm cache restore .harmont-cache/ - name: hm run ci env: @@ -131,12 +129,16 @@ jobs: run: ./target/debug/hm run ci - name: Save harmont Docker images + id: cache-manifest if: always() - run: | - images=$(docker images --format '{{.Repository}}:{{.Tag}}' | grep '^harmont-local/' || true) - if [ -n "$images" ]; then - docker save $images -o /tmp/harmont-docker-cache.tar - fi + run: echo "key=harmont-v1-$(./target/debug/hm cache save .harmont-cache/)" >> "$GITHUB_OUTPUT" + + - name: Upload Docker cache + if: always() + uses: actions/cache/save@v4 + with: + path: .harmont-cache/ + key: ${{ steps.cache-manifest.outputs.key }} integration: name: docker-gated integration test From 2ad0bee0edbc39007f979a22b04d023133fbf023 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 19:48:03 -0700 Subject: [PATCH 29/41] test: add docker-gated integration tests for hm cache save/restore --- crates/hm/src/commands/cache/manifest.rs | 1 + crates/hm/tests/cache_round_trip.rs | 103 +++++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 crates/hm/tests/cache_round_trip.rs diff --git a/crates/hm/src/commands/cache/manifest.rs b/crates/hm/src/commands/cache/manifest.rs index 7e3be499..71ab78fd 100644 --- a/crates/hm/src/commands/cache/manifest.rs +++ b/crates/hm/src/commands/cache/manifest.rs @@ -54,6 +54,7 @@ pub fn tag_from_tar_name(filename: &str) -> Option { } #[cfg(test)] +#[allow(clippy::unwrap_used, reason = "unit tests")] mod tests { use super::*; diff --git a/crates/hm/tests/cache_round_trip.rs b/crates/hm/tests/cache_round_trip.rs new file mode 100644 index 00000000..2ab821f0 --- /dev/null +++ b/crates/hm/tests/cache_round_trip.rs @@ -0,0 +1,103 @@ +//! Docker-gated integration test for `hm cache save` / `hm cache restore`. +//! +//! Run: `cargo test -p harmont-cli --features docker-integration -- --ignored cache` + +#![cfg(feature = "docker-integration")] +#![allow( + clippy::unwrap_used, + reason = "integration tests panic on unexpected failures" +)] +#![allow( + clippy::expect_used, + reason = "integration tests panic on unexpected failures" +)] +#![allow( + clippy::ignore_without_reason, + reason = "reason is in the test name and doc comment above" +)] + +use assert_cmd::Command; +use predicates::str::contains; +use tempfile::TempDir; + +#[test] +#[ignore] +fn cache_save_creates_manifest() { + let cache_dir = TempDir::new().unwrap(); + let cache_path = cache_dir.path(); + + let out = Command::cargo_bin("hm") + .unwrap() + .args(["cache", "save", cache_path.to_str().unwrap()]) + .assert() + .success(); + + // manifest.json must exist + let manifest_path = cache_path.join("manifest.json"); + assert!(manifest_path.exists(), "manifest.json should exist"); + + let content = std::fs::read_to_string(&manifest_path).unwrap(); + let manifest: serde_json::Value = serde_json::from_str(&content).unwrap(); + assert_eq!(manifest["version"], 1); + + // stdout has the 16-char hex content hash + let stdout = String::from_utf8(out.get_output().stdout.clone()).unwrap(); + let hash = stdout.trim(); + assert_eq!(hash.len(), 16, "content hash should be 16 hex chars"); + assert!(hash.chars().all(|c| c.is_ascii_hexdigit()), "should be hex"); +} + +#[test] +#[ignore] +fn cache_save_is_deterministic() { + let cache_dir = TempDir::new().unwrap(); + let path = cache_dir.path().to_str().unwrap(); + + let out1 = Command::cargo_bin("hm") + .unwrap() + .args(["cache", "save", path]) + .assert() + .success(); + let out2 = Command::cargo_bin("hm") + .unwrap() + .args(["cache", "save", path]) + .assert() + .success(); + + let h1 = String::from_utf8(out1.get_output().stdout.clone()).unwrap(); + let h2 = String::from_utf8(out2.get_output().stdout.clone()).unwrap(); + assert_eq!(h1.trim(), h2.trim(), "content hash should be deterministic"); +} + +#[test] +#[ignore] +fn cache_restore_after_save() { + let cache_dir = TempDir::new().unwrap(); + let path = cache_dir.path().to_str().unwrap(); + + // Save first + Command::cargo_bin("hm") + .unwrap() + .args(["cache", "save", path]) + .assert() + .success(); + + // Restore — all images already present + Command::cargo_bin("hm") + .unwrap() + .args(["cache", "restore", path]) + .assert() + .success() + .stderr(contains("already present")); +} + +#[test] +#[ignore] +fn cache_restore_missing_dir() { + Command::cargo_bin("hm") + .unwrap() + .args(["cache", "restore", "/tmp/harmont-nonexistent-cache-dir-test"]) + .assert() + .success() + .stderr(contains("0/0")); +} From f9ac8e31819f3bd38cfae006c7126f0f72d6df84 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 20:02:42 -0700 Subject: [PATCH 30/41] fix: use eprintln for cache progress, keep only hash on stdout --- .github/workflows/ci.yml | 4 +++- crates/hm/src/commands/cache/restore.rs | 15 ++++++--------- crates/hm/src/commands/cache/save.rs | 22 ++++++++-------------- 3 files changed, 17 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02cd7186..55f4f9c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -131,7 +131,9 @@ jobs: - name: Save harmont Docker images id: cache-manifest if: always() - run: echo "key=harmont-v1-$(./target/debug/hm cache save .harmont-cache/)" >> "$GITHUB_OUTPUT" + run: | + hash=$(./target/debug/hm cache save .harmont-cache/) + echo "key=harmont-v1-${hash}" >> "$GITHUB_OUTPUT" - name: Upload Docker cache if: always() diff --git a/crates/hm/src/commands/cache/restore.rs b/crates/hm/src/commands/cache/restore.rs index c56213fa..6f68afc8 100644 --- a/crates/hm/src/commands/cache/restore.rs +++ b/crates/hm/src/commands/cache/restore.rs @@ -1,7 +1,6 @@ use std::path::Path; use anyhow::{Context, Result}; -use tracing::{info, warn}; use super::manifest; use crate::orchestrator::docker_client::DockerClient; @@ -10,24 +9,22 @@ use crate::orchestrator::docker_client::DockerClient; /// /// Each `.tar` file is mapped back to its `harmont-local/*` tag via /// [`manifest::tag_from_tar_name`]. Images that already exist in the -/// local Docker daemon are skipped. +/// local Docker daemon are skipped. All progress goes to stderr. /// /// # Errors /// /// Returns an error if the Docker daemon is unreachable or a filesystem /// operation on `dir` fails. -#[allow(clippy::print_stderr, reason = "status summary must go to stderr for CI visibility")] +#[allow(clippy::print_stderr)] pub async fn handle_restore(dir: &Path) -> Result { let docker = DockerClient::connect()?; docker.ping().await?; if !dir.exists() { - info!("cache dir does not exist, nothing to restore"); eprintln!("restored 0/0 images (cache dir missing)"); return Ok(0); } - // Scan for .tar files let mut tars = Vec::new(); let mut entries = tokio::fs::read_dir(dir) .await @@ -49,21 +46,21 @@ pub async fn handle_restore(dir: &Path) -> Result { for (filename, tar_path) in &tars { let Some(tag) = manifest::tag_from_tar_name(filename) else { - warn!("skip unrecognized tar: {filename}"); + eprintln!("skip unrecognized tar: {filename}"); continue; }; if docker.image_exists(&tag).await? { - info!("skip (present): {tag}"); + eprintln!("skip (present): {tag}"); skipped += 1; continue; } - info!("restore: {filename} → {tag}"); + eprintln!("restore: {filename} → {tag}"); match docker.import_image(tar_path).await { Ok(()) => restored += 1, Err(e) => { - warn!("failed to load {filename}: {e}"); + eprintln!("warning: failed to load {filename}: {e}"); } } } diff --git a/crates/hm/src/commands/cache/save.rs b/crates/hm/src/commands/cache/save.rs index 1fe196ba..cf74e0fc 100644 --- a/crates/hm/src/commands/cache/save.rs +++ b/crates/hm/src/commands/cache/save.rs @@ -1,7 +1,6 @@ use std::path::Path; use anyhow::{Context, Result}; -use tracing::info; use super::manifest::{self, Manifest}; use crate::orchestrator::docker_client::DockerClient; @@ -11,12 +10,14 @@ use crate::orchestrator::docker_client::DockerClient; /// any known image. /// /// Prints the manifest's content hash to stdout so CI runners (e.g. -/// GitHub Actions) can capture it for use as a cache key. +/// GitHub Actions) can capture it for use as a cache key. Progress +/// messages go to stderr. /// /// # Errors /// /// Returns an error if the Docker daemon is unreachable, an image /// export fails, or any filesystem operation on `dir` fails. +#[allow(clippy::print_stdout, clippy::print_stderr)] pub async fn handle_save(dir: &Path) -> Result { let docker = DockerClient::connect()?; docker.ping().await?; @@ -25,49 +26,42 @@ pub async fn handle_save(dir: &Path) -> Result { .await .with_context(|| format!("create cache dir {}", dir.display()))?; - // 1. List all harmont-local/* images (list_images_by_prefix already skips ephemeral) let tags = docker.list_images_by_prefix("harmont-local/").await?; let mut manifest = Manifest::new(); - // 2. For each image, save to tar if not already on disk for tag in &tags { let filename = manifest::tar_name_for_tag(tag); let tar_path = dir.join(&filename); if tar_path.exists() { - info!("skip (exists): {filename}"); + eprintln!("skip (exists): {filename}"); } else { - info!("save: {tag} → {filename}"); + eprintln!("save: {tag} → {filename}"); docker.export_image(tag, &tar_path).await?; } manifest.images.insert(filename, tag.clone()); } - // 3. Write manifest.json let manifest_json = serde_json::to_string_pretty(&manifest)?; tokio::fs::write(dir.join("manifest.json"), &manifest_json) .await .context("write manifest.json")?; - // 4. Prune stale tars + // Prune stale tars not in current manifest let mut entries = tokio::fs::read_dir(dir).await?; while let Some(entry) = entries.next_entry().await? { let name = entry.file_name(); let name_str = name.to_string_lossy(); if name_str.ends_with(".tar") && !manifest.images.contains_key(name_str.as_ref()) { - info!("prune stale: {name_str}"); + eprintln!("prune stale: {name_str}"); tokio::fs::remove_file(entry.path()).await.ok(); } } - // 5. Print content hash to stdout (GHA captures this for the cache key) let hash = manifest.content_hash(); - #[allow(clippy::print_stdout, reason = "hash must go to stdout for CI capture")] - { - println!("{hash}"); - } + println!("{hash}"); Ok(0) } From 9b0ab25aec3808fb6ec45d1cbe387a7a25fd2d90 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 20:03:28 -0700 Subject: [PATCH 31/41] style: cargo fmt --- crates/hm/src/cli/mod.rs | 4 +--- crates/hm/src/commands/cache/manifest.rs | 6 ++++-- crates/hm/src/orchestrator/docker_client.rs | 12 ++++-------- crates/hm/tests/cache_round_trip.rs | 6 +++++- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/crates/hm/src/cli/mod.rs b/crates/hm/src/cli/mod.rs index 8cf5e0e1..6213b6f9 100644 --- a/crates/hm/src/cli/mod.rs +++ b/crates/hm/src/cli/mod.rs @@ -100,9 +100,7 @@ pub async fn dispatch(command: Command, ctx: RunContext) -> Result { Command::Dev(cmd) => dev::dispatch(cmd, ctx).await, Command::Cache(cmd) => match cmd { CacheCommand::Save(args) => crate::commands::cache::handle_save(&args.dir).await, - CacheCommand::Restore(args) => { - crate::commands::cache::handle_restore(&args.dir).await - } + CacheCommand::Restore(args) => crate::commands::cache::handle_restore(&args.dir).await, }, Command::Version => version::run().await.map(|()| 0), Command::Plugin(cmd) => plugin::run(cmd).await.map(|()| 0), diff --git a/crates/hm/src/commands/cache/manifest.rs b/crates/hm/src/commands/cache/manifest.rs index 71ab78fd..c0f11a4e 100644 --- a/crates/hm/src/commands/cache/manifest.rs +++ b/crates/hm/src/commands/cache/manifest.rs @@ -94,8 +94,10 @@ mod tests { #[test] fn manifest_content_hash_is_deterministic() { let mut m = Manifest::new(); - m.images - .insert("step1".to_string(), "harmont-local/step1:deadbeef".to_string()); + m.images.insert( + "step1".to_string(), + "harmont-local/step1:deadbeef".to_string(), + ); let h1 = m.content_hash(); let h2 = m.content_hash(); diff --git a/crates/hm/src/orchestrator/docker_client.rs b/crates/hm/src/orchestrator/docker_client.rs index 2c86b508..83d064fd 100644 --- a/crates/hm/src/orchestrator/docker_client.rs +++ b/crates/hm/src/orchestrator/docker_client.rs @@ -389,15 +389,11 @@ impl DockerClient { let body = tokio::fs::read(src) .await .with_context(|| format!("read import file '{}'", src.display()))?; - let mut stream = self.inner.import_image( - ImportImageOptions { quiet: true }, - body.into(), - None, - ); + let mut stream = + self.inner + .import_image(ImportImageOptions { quiet: true }, body.into(), None); while let Some(item) = stream.next().await { - item.map_err(|e| { - HmError::Docker(format!("import_image '{}': {e}", src.display())) - })?; + item.map_err(|e| HmError::Docker(format!("import_image '{}': {e}", src.display())))?; } Ok(()) } diff --git a/crates/hm/tests/cache_round_trip.rs b/crates/hm/tests/cache_round_trip.rs index 2ab821f0..226625e9 100644 --- a/crates/hm/tests/cache_round_trip.rs +++ b/crates/hm/tests/cache_round_trip.rs @@ -96,7 +96,11 @@ fn cache_restore_after_save() { fn cache_restore_missing_dir() { Command::cargo_bin("hm") .unwrap() - .args(["cache", "restore", "/tmp/harmont-nonexistent-cache-dir-test"]) + .args([ + "cache", + "restore", + "/tmp/harmont-nonexistent-cache-dir-test", + ]) .assert() .success() .stderr(contains("0/0")); From 28e8fd17565a3a1a74cfb7d165c9721d6ac13d1a Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 20:15:43 -0700 Subject: [PATCH 32/41] fix: revert to tracing::info!, direct tracing subscriber to stderr --- crates/hm/src/commands/cache/restore.rs | 12 +++++++----- crates/hm/src/commands/cache/save.rs | 13 ++++++------- crates/hm/src/main.rs | 1 + 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/crates/hm/src/commands/cache/restore.rs b/crates/hm/src/commands/cache/restore.rs index 6f68afc8..86815cdc 100644 --- a/crates/hm/src/commands/cache/restore.rs +++ b/crates/hm/src/commands/cache/restore.rs @@ -1,6 +1,7 @@ use std::path::Path; use anyhow::{Context, Result}; +use tracing::{info, warn}; use super::manifest; use crate::orchestrator::docker_client::DockerClient; @@ -9,7 +10,7 @@ use crate::orchestrator::docker_client::DockerClient; /// /// Each `.tar` file is mapped back to its `harmont-local/*` tag via /// [`manifest::tag_from_tar_name`]. Images that already exist in the -/// local Docker daemon are skipped. All progress goes to stderr. +/// local Docker daemon are skipped. /// /// # Errors /// @@ -21,6 +22,7 @@ pub async fn handle_restore(dir: &Path) -> Result { docker.ping().await?; if !dir.exists() { + info!("cache dir does not exist, nothing to restore"); eprintln!("restored 0/0 images (cache dir missing)"); return Ok(0); } @@ -46,21 +48,21 @@ pub async fn handle_restore(dir: &Path) -> Result { for (filename, tar_path) in &tars { let Some(tag) = manifest::tag_from_tar_name(filename) else { - eprintln!("skip unrecognized tar: {filename}"); + warn!("skip unrecognized tar: {filename}"); continue; }; if docker.image_exists(&tag).await? { - eprintln!("skip (present): {tag}"); + info!("skip (present): {tag}"); skipped += 1; continue; } - eprintln!("restore: {filename} → {tag}"); + info!("restore: {filename} → {tag}"); match docker.import_image(tar_path).await { Ok(()) => restored += 1, Err(e) => { - eprintln!("warning: failed to load {filename}: {e}"); + warn!("failed to load {filename}: {e}"); } } } diff --git a/crates/hm/src/commands/cache/save.rs b/crates/hm/src/commands/cache/save.rs index cf74e0fc..195fea0d 100644 --- a/crates/hm/src/commands/cache/save.rs +++ b/crates/hm/src/commands/cache/save.rs @@ -1,6 +1,7 @@ use std::path::Path; use anyhow::{Context, Result}; +use tracing::info; use super::manifest::{self, Manifest}; use crate::orchestrator::docker_client::DockerClient; @@ -10,14 +11,13 @@ use crate::orchestrator::docker_client::DockerClient; /// any known image. /// /// Prints the manifest's content hash to stdout so CI runners (e.g. -/// GitHub Actions) can capture it for use as a cache key. Progress -/// messages go to stderr. +/// GitHub Actions) can capture it for use as a cache key. /// /// # Errors /// /// Returns an error if the Docker daemon is unreachable, an image /// export fails, or any filesystem operation on `dir` fails. -#[allow(clippy::print_stdout, clippy::print_stderr)] +#[allow(clippy::print_stdout)] pub async fn handle_save(dir: &Path) -> Result { let docker = DockerClient::connect()?; docker.ping().await?; @@ -35,9 +35,9 @@ pub async fn handle_save(dir: &Path) -> Result { let tar_path = dir.join(&filename); if tar_path.exists() { - eprintln!("skip (exists): {filename}"); + info!("skip (exists): {filename}"); } else { - eprintln!("save: {tag} → {filename}"); + info!("save: {tag} → {filename}"); docker.export_image(tag, &tar_path).await?; } @@ -49,13 +49,12 @@ pub async fn handle_save(dir: &Path) -> Result { .await .context("write manifest.json")?; - // Prune stale tars not in current manifest let mut entries = tokio::fs::read_dir(dir).await?; while let Some(entry) = entries.next_entry().await? { let name = entry.file_name(); let name_str = name.to_string_lossy(); if name_str.ends_with(".tar") && !manifest.images.contains_key(name_str.as_ref()) { - eprintln!("prune stale: {name_str}"); + info!("prune stale: {name_str}"); tokio::fs::remove_file(entry.path()).await.ok(); } } diff --git a/crates/hm/src/main.rs b/crates/hm/src/main.rs index ae5a96e7..f104b7b6 100644 --- a/crates/hm/src/main.rs +++ b/crates/hm/src/main.rs @@ -23,6 +23,7 @@ async fn main() { // Initialize tracing if --verbose. if args.verbose { tracing_subscriber::fmt() + .with_writer(std::io::stderr) .with_env_filter( EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("debug")), ) From def73ff6db28a4ade1f6fbf38792cd35636234ce Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 20:26:01 -0700 Subject: [PATCH 33/41] ci: retrigger From a9121d42b35c92f31a31d7d420e8c914bf77af74 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 20:31:27 -0700 Subject: [PATCH 34/41] ci: retrigger (2) From 89e93cc32b3cb4ac41f5a4b67f30a9e65c487659 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 20:51:32 -0700 Subject: [PATCH 35/41] chore: remove haskell example --- .github/workflows/examples.yml | 1 - examples/haskell/.harmont/pipeline.py | 30 --------------------------- examples/haskell/.harmont/pipeline.ts | 21 ------------------- examples/haskell/README.md | 12 ----------- examples/haskell/cabal.project | 1 - examples/haskell/example.cabal | 18 ---------------- examples/haskell/src/Lib.hs | 4 ---- examples/haskell/test/Spec.hs | 8 ------- 8 files changed, 95 deletions(-) delete mode 100644 examples/haskell/.harmont/pipeline.py delete mode 100644 examples/haskell/.harmont/pipeline.ts delete mode 100644 examples/haskell/README.md delete mode 100644 examples/haskell/cabal.project delete mode 100644 examples/haskell/example.cabal delete mode 100644 examples/haskell/src/Lib.hs delete mode 100644 examples/haskell/test/Spec.hs diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index eccaebe0..e1699e7d 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -85,7 +85,6 @@ jobs: - cpp - csharp - go - - haskell - java - kotlin - nextjs diff --git a/examples/haskell/.harmont/pipeline.py b/examples/haskell/.harmont/pipeline.py deleted file mode 100644 index fa7a51a8..00000000 --- a/examples/haskell/.harmont/pipeline.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Haskell example pipeline.""" -from __future__ import annotations - -import harmont as hm -from harmont.haskell import HaskellPackage, HaskellToolchain - - -@hm.target() -def ghc() -> HaskellToolchain: - return hm.haskell(ghc="9.6.7") - - -@hm.target() -def project(ghc: hm.Target[HaskellToolchain]) -> HaskellPackage: - return ghc.cabal(path=".") - - -@hm.pipeline( - "ci", - env={"CI": "true"}, - default_image="ubuntu:24.04", - triggers=[hm.push(branch="main")], -) -def ci(project: hm.Target[HaskellPackage]) -> tuple[hm.Step, ...]: - return ( - project.build(), - project.test(), - project.lint(), - project.fmt(), - ) diff --git a/examples/haskell/.harmont/pipeline.ts b/examples/haskell/.harmont/pipeline.ts deleted file mode 100644 index ed64b3a7..00000000 --- a/examples/haskell/.harmont/pipeline.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { pipeline, push, target, type PipelineDefinition } from "harmont"; -import { haskell } from "harmont/toolchains"; - -const ghc = target("ghc", () => haskell({ ghc: "9.6.7" })); -const project = target("project", () => ghc().cabal(".")); - -const pipelines: PipelineDefinition[] = [ - { - slug: "ci", - triggers: [push({ branch: "main" })], - pipeline: pipeline( - project().build(), - project().test(), - project().lint(), - project().fmt(), - { env: { CI: "true" }, defaultImage: "ubuntu:24.04" }, - ), - }, -]; - -export default pipelines; diff --git a/examples/haskell/README.md b/examples/haskell/README.md deleted file mode 100644 index 27533977..00000000 --- a/examples/haskell/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Haskell example - -Single cabal package with an inline test-suite. Pipeline pins GHC 9.6.7 via ghcup and runs build + test + lint (--flag werror) + fmt (fourmolu check). - -## Run the pipeline - -```sh -cd examples/haskell -hm run ci --local -``` - -See `.harmont/pipeline.py` for the definition; `examples/README.md` for the full index. diff --git a/examples/haskell/cabal.project b/examples/haskell/cabal.project deleted file mode 100644 index e6fdbadb..00000000 --- a/examples/haskell/cabal.project +++ /dev/null @@ -1 +0,0 @@ -packages: . diff --git a/examples/haskell/example.cabal b/examples/haskell/example.cabal deleted file mode 100644 index 6c4caacd..00000000 --- a/examples/haskell/example.cabal +++ /dev/null @@ -1,18 +0,0 @@ -cabal-version: 3.0 -name: example -version: 0.1.0.0 -build-type: Simple - -library - exposed-modules: Lib - hs-source-dirs: src - build-depends: base >=4.18 && <5 - default-language: Haskell2010 - ghc-options: -Wall - -test-suite example-test - type: exitcode-stdio-1.0 - main-is: Spec.hs - hs-source-dirs: test - build-depends: base, example - default-language: Haskell2010 diff --git a/examples/haskell/src/Lib.hs b/examples/haskell/src/Lib.hs deleted file mode 100644 index 665c64d0..00000000 --- a/examples/haskell/src/Lib.hs +++ /dev/null @@ -1,4 +0,0 @@ -module Lib (add) where - -add :: Int -> Int -> Int -add a b = a + b diff --git a/examples/haskell/test/Spec.hs b/examples/haskell/test/Spec.hs deleted file mode 100644 index 2ce5c26d..00000000 --- a/examples/haskell/test/Spec.hs +++ /dev/null @@ -1,8 +0,0 @@ -module Main where - -import Lib (add) -import System.Exit (exitFailure, exitSuccess) - -main :: IO () -main = - if add 2 3 == 5 then exitSuccess else exitFailure From a8c32b3b3310c7e4fd569f04f230fe20e85fe5a9 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 21:07:11 -0700 Subject: [PATCH 36/41] style: cargo fmt main.rs (remove leading blank line) --- crates/hm/src/main.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/hm/src/main.rs b/crates/hm/src/main.rs index c5b32d20..32f0ad9c 100644 --- a/crates/hm/src/main.rs +++ b/crates/hm/src/main.rs @@ -1,4 +1,3 @@ - #![allow( clippy::multiple_crate_versions, reason = "transitive dependency version conflicts in rand/windows-sys/thiserror chains; not fixable without upstream updates" From cbf45a957f09a4c1729f6b20dc27d41955fed0d4 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 21:08:52 -0700 Subject: [PATCH 37/41] fix: update TS example count after haskell removal --- dsls/harmont-ts/tests/examples.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dsls/harmont-ts/tests/examples.test.ts b/dsls/harmont-ts/tests/examples.test.ts index 9cc8a6c3..6d9994c2 100644 --- a/dsls/harmont-ts/tests/examples.test.ts +++ b/dsls/harmont-ts/tests/examples.test.ts @@ -62,7 +62,7 @@ describe("examples render to v0 IR", () => { }); } - it("discovered at least 18 example pipeline.ts files", () => { - expect(examples.length).toBeGreaterThanOrEqual(18); + it("discovered at least 17 example pipeline.ts files", () => { + expect(examples.length).toBeGreaterThanOrEqual(17); }); }); From d6b2f7b9e7b0c045d90733ff6a7c64860a286776 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 21:20:37 -0700 Subject: [PATCH 38/41] fix: port-of outputs to stdout, not tracing --- crates/hm/src/commands/dev/port_of.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/hm/src/commands/dev/port_of.rs b/crates/hm/src/commands/dev/port_of.rs index e7870df8..b9e41917 100644 --- a/crates/hm/src/commands/dev/port_of.rs +++ b/crates/hm/src/commands/dev/port_of.rs @@ -84,7 +84,10 @@ pub async fn handle(args: DevPortOfArgs, _ctx: RunContext) -> Result { ); return Ok(5); }; - tracing::info!("{host_port}"); + #[allow(clippy::print_stdout)] + { + println!("{host_port}"); + } Ok(0) } From 5b2d20da11e088d6157860eabe52a4a7f00086dc Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 22:22:47 -0700 Subject: [PATCH 39/41] ci: retrigger for clean PR checks From 83d168f939e55e13e0de6048a603619014b77564 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 22:43:25 -0700 Subject: [PATCH 40/41] ci: restore push trigger to main-only (avoid double runs on PRs) --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a9d8bd49..55f4f9c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,6 +3,7 @@ name: CI on: pull_request: push: + branches: [main] permissions: contents: read From 5143af1e4a3dd33b18e59ee60074bc3736d0dce1 Mon Sep 17 00:00:00 2001 From: Marko Vejnovic Date: Sun, 24 May 2026 23:27:08 -0700 Subject: [PATCH 41/41] ci: retrigger PR checks