From 219dd08e8e62c820555357090b99c831ef263203 Mon Sep 17 00:00:00 2001 From: Anastassios Nanos Date: Sun, 4 Oct 2026 01:29:19 +0300 Subject: [PATCH 1/7] feat(run): Add overlay upper layers for hvi roots HULL_HVI_ROOTFS_UPPER selects how an hvi container boot gets a writable root. Unset keeps the per-instance APFS clone. "virtiofs" and "block" share the cached image read-only as the lower and give the instance its own overlay upper: a directory exported as the writable tag rootfs-upper, or a sparse 16G ext4 image attached as the guest's disk. Nothing is copied before the VMM starts. The upper is created on first boot and reused after; mke2fs is told the fresh sparse file is already zeroed, so it writes only metadata. vz-init picks the upper from /urunc-rootfs-upper. Signed-off-by: Anastassios Nanos --- cmd/hull/rootfsupper.go | 56 +++++++++++++++++++++++++++++++++++++++++ cmd/hull/run.go | 36 ++++++++++++++++++++++++-- 2 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 cmd/hull/rootfsupper.go diff --git a/cmd/hull/rootfsupper.go b/cmd/hull/rootfsupper.go new file mode 100644 index 00000000..e808eb05 --- /dev/null +++ b/cmd/hull/rootfsupper.go @@ -0,0 +1,56 @@ +package main + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// rootfsUpperDiskSize is the apparent size of an ext4 upper layer. The file is +// sparse, so it costs only what the guest writes, and it caps what one instance +// can add on top of its image. +const rootfsUpperDiskSize = "16G" + +// prepareRootfsUpper returns the instance's overlay upper layer, creating it on +// first boot and reusing it on every later one. +// +// kind "virtiofs" is a directory hvi exports read-write; vz-init makes the +// upper and work directories inside it. kind "block" is a sparse ext4 image +// hvi attaches as the guest's only disk. mke2fs is told the storage is already +// zeroed, which a fresh sparse file is, so it writes only the metadata it needs +// and the guest has no inode tables left to initialise. +func prepareRootfsUpper(instanceDir, kind string) (string, error) { + switch kind { + case "virtiofs": + dir := filepath.Join(instanceDir, "rootfs-upper") + if err := os.MkdirAll(dir, 0o700); err != nil { + return "", fmt.Errorf("create overlay upper directory: %w", err) + } + return dir, nil + case "block": + disk := filepath.Join(instanceDir, "rootfs-upper.ext4") + if _, err := os.Stat(disk); err == nil { + return disk, nil + } else if !errors.Is(err, os.ErrNotExist) { + return "", fmt.Errorf("inspect overlay upper disk: %w", err) + } + staging := disk + ".tmp" + _ = os.Remove(staging) + out, err := exec.Command(mke2fsBin, "-q", "-F", "-t", "ext4", "-m", "0", + "-E", "assume_storage_prezeroed=1,root_owner=0:0", + staging, rootfsUpperDiskSize).CombinedOutput() + if err != nil { + _ = os.Remove(staging) + return "", fmt.Errorf("create overlay upper disk: %s: %w", strings.TrimSpace(string(out)), err) + } + if err := os.Rename(staging, disk); err != nil { + _ = os.Remove(staging) + return "", fmt.Errorf("install overlay upper disk: %w", err) + } + return disk, nil + } + return "", fmt.Errorf("HULL_HVI_ROOTFS_UPPER must be virtiofs or block (got %q)", kind) +} diff --git a/cmd/hull/run.go b/cmd/hull/run.go index 110d94fd..2fb47495 100644 --- a/cmd/hull/run.go +++ b/cmd/hull/run.go @@ -754,6 +754,10 @@ func runInstance(ctx context.Context, cmd *cli.Command) error { rootfsDiskImage := "" // path to ext4 disk image (block mode) containerRootfsDirect := false containerMetadataRootfs := "" + // The overlay upper layer an hvi container boot keeps per instance: its + // kind ("virtiofs" or "block") and the directory or disk that holds it. + containerRootfsUpper := "" + containerRootfsUpperPath := "" mountRootfs := ociSpec.Annotations["com.urunc.unikernel.mountRootfs"] == "true" if containerBoot { @@ -818,7 +822,19 @@ func runInstance(ctx context.Context, cmd *cli.Command) error { if resolveErr != nil { return fmt.Errorf("resolve unpacked OCI rootfs: %w", resolveErr) } - if vmmType == hypervisors.HviVmm { + if upper := os.Getenv("HULL_HVI_ROOTFS_UPPER"); vmmType == hypervisors.HviVmm && upper != "" { + // Overlay: the cached image is the read-only lower and every + // guest write lands in the instance's own upper layer, so + // nothing is copied before the VMM starts. + upperPath, err := prepareRootfsUpper(s.InstanceDir(instanceName), upper) + if err != nil { + return err + } + rootfsDir = resolved + containerMetadataRootfs = resolved + containerRootfsUpper = upper + containerRootfsUpperPath = upperPath + } else if vmmType == hypervisors.HviVmm { // HVI's writable virtio-fs backend can use an unpacked directory as // the real root. Replace only the instance bundle's cache symlink // with an APFS copy-on-write clone: guest changes then persist with @@ -1015,6 +1031,11 @@ exec %s "$@" return fmt.Errorf("select direct writable rootfs in boot initrd: %w", err) } } + if containerRootfsUpper != "" { + if err := initrd.AddFileToInitrd(initrdPath, containerRootfsUpper+"\n", "/urunc-rootfs-upper"); err != nil { + return fmt.Errorf("select overlay upper layer in boot initrd: %w", err) + } + } resolver, err := containerBootResolver(gatewayIP) if err != nil { return err @@ -1383,6 +1404,17 @@ exec %s "$@" log.Debugf("Shared directory: host=%s guest=%s tag=%s readOnly=%t", sh.host, sh.guest, sh.tag, sh.readOnly) } + // vz-init mounts the upper layer by kind: a writable tag named + // rootfs-upper, or the guest's only disk. + blockDevPath := rootfsDiskImage + switch containerRootfsUpper { + case "virtiofs": + sharedDirParams = append(sharedDirParams, types.SharedDirParams{ + Path: containerRootfsUpperPath, Tag: "rootfs-upper", + }) + case "block": + blockDevPath = containerRootfsUpperPath + } // "unikernel" is hull's own default name, but the file it names would come // out of the image and is handed to the monitor as a binary to load, so it @@ -1406,7 +1438,7 @@ exec %s "$@" KernelPath: kernelPath, InitrdPath: initrdPath, RootfsPath: rootfsDir, - BlockDevPath: rootfsDiskImage, + BlockDevPath: blockDevPath, LogFile: logFile, AgentSockPath: s.InstanceAgentSocket(instanceName), GUI: gui, From d7f2357dcc2bcbb56f547a1c8e97fc1e671a690f Mon Sep 17 00:00:00 2001 From: Anastassios Nanos Date: Fri, 9 Oct 2026 15:00:09 +0300 Subject: [PATCH 2/7] chore(deps): Pin urunc to the darwin series with the single binary urunc moves to 770a597 on feat/urunit-single-binary. That is the darwin series (feat/hvi-darwin-linux-parity 71c4d2d, on feat/unchanged_containers-exec-fixes) with urunit's single binary on top. hull leaves the v0.8.0 line it tracked so far. For the disks of an overlay-block root: - ExecArgs gains BlockDevs, an ordered list of BlockDevSpec{Path, ReadOnly}. BlockDevPath stays as the single writable disk when BlockDevs is empty. - hvi gets one --disk path=

[,readonly=on],serial=disk per entry, the form hvi-vmm takes since its multi-disk support landed. Shares go out as --fs tag=,path=

[,readonly=on][,uid=..,gid=..]. - vz gets --disk or --disk-ro per entry, after the root share, so a root share is no longer dropped when disks are present. - Both refuse an empty or a duplicate disk path; hvi refuses a path with a comma. The newer urunc requires newer modules than hull did, and minimum version selection takes them: urfave/cli v3.13.0, logrus v1.10.2, the golang.org/x modules, grpc v1.84.0 and their dependencies. urunc now writes urunit.conf as NUL-terminated URUNIT1 records. An image that boots its own urunit (the BuildUrunitConfig paths in run.go) needs a urunit from NOFireAI/urunit da5e62f or later. An older one runs the command without its environment, user and working directory. Signed-off-by: Anastassios Nanos --- go.mod | 50 ++++++++++++++++++++++++++------------------------ go.sum | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 43fba3f8..064c07d0 100644 --- a/go.mod +++ b/go.mod @@ -12,17 +12,17 @@ require ( github.com/inetaf/tcpproxy v0.0.0-20250222171855-c4b9df066048 github.com/miekg/dns v1.1.72 github.com/opencontainers/runtime-spec v1.2.1 - github.com/sirupsen/logrus v1.9.4 - github.com/urfave/cli/v3 v3.10.1 - github.com/urunc-dev/urunc v0.8.1-0.20261006162953-6c062f39c5bb - golang.org/x/sys v0.47.0 - golang.org/x/term v0.45.0 + github.com/sirupsen/logrus v1.10.2 + github.com/urfave/cli/v3 v3.13.0 + github.com/urunc-dev/urunc v0.8.2-0.20261018212029-770a597f0a1d + golang.org/x/sys v0.48.0 + golang.org/x/term v0.46.0 gopkg.in/yaml.v3 v3.0.1 gvisor.dev/gvisor v0.0.0-20240916094835-a174eb65023f ) require ( - cyphar.com/go-pathrs v0.2.5 // indirect + cyphar.com/go-pathrs v0.2.6 // indirect github.com/BurntSushi/toml v1.6.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Microsoft/hcsshim v0.13.0 // indirect @@ -35,7 +35,7 @@ require ( github.com/containerd/continuity v0.5.0 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect - github.com/containerd/log v0.1.0 // indirect + github.com/containerd/log v0.2.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.16.3 // indirect github.com/containerd/typeurl/v2 v2.3.0 // indirect github.com/coreos/go-systemd/v22 v22.7.0 // indirect @@ -54,23 +54,23 @@ require ( github.com/google/gopacket v1.1.19 // indirect github.com/insomniacslk/dhcp v0.0.0-20240710054256-ddd8a41251c9 // indirect github.com/jackpal/gateway v1.2.0 // indirect - github.com/klauspost/compress v1.18.6 // indirect + github.com/klauspost/compress v1.20.1 // indirect github.com/mattn/go-colorable v0.1.15 // indirect - github.com/mattn/go-isatty v0.0.22 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect github.com/mattn/go-shellwords v1.0.12 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/moby/sys/capability v0.4.0 // indirect github.com/moby/sys/mountinfo v0.7.2 // indirect github.com/moby/sys/sequential v0.7.0 // indirect github.com/moby/sys/user v0.4.1 // indirect - github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/sys/userns v0.2.1 // indirect github.com/mrunalp/fileutils v0.5.1 // indirect github.com/nubificus/hedge_cli v0.0.3 // indirect github.com/opencontainers/cgroups v0.0.4 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/opencontainers/runc v1.3.6 // indirect - github.com/opencontainers/selinux v1.13.1 // indirect + github.com/opencontainers/selinux v1.15.1 // indirect github.com/pierrec/lz4/v4 v4.1.14 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/rs/zerolog v1.35.1 // indirect @@ -83,23 +83,25 @@ require ( github.com/xhit/go-str2duration/v2 v2.1.0 // indirect go.opencensus.io v0.24.0 // indirect go.yaml.in/yaml/v4 v4.0.0-rc.4 // indirect - golang.org/x/crypto v0.56.0 // indirect - golang.org/x/mod v0.38.0 // indirect - golang.org/x/net v0.58.0 // indirect - golang.org/x/sync v0.22.0 // indirect - golang.org/x/text v0.41.0 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/mod v0.41.0 // indirect + golang.org/x/net v0.59.0 // indirect + golang.org/x/sync v0.23.0 // indirect + golang.org/x/text v0.42.0 // indirect golang.org/x/time v0.12.0 // indirect - golang.org/x/tools v0.48.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260622175928-b703f567277d // indirect - google.golang.org/grpc v1.83.2 // indirect - google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect + golang.org/x/tools v0.49.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260921155816-b14227669459 // indirect + google.golang.org/grpc v1.84.0 // indirect + google.golang.org/protobuf v1.36.12 // indirect ) // The darwin work hull depends on -- the Vz backend, exec, console, the GUI -// window flags, the HVI backend and the generic container initrd -- is the -// urunc-dev/urunc branch feat/initrd-hvi-backend-v0.8.0: the darwin commits -// rebased onto the v0.8.0 release. The require below is that branch's tip. -// No replace: the module is the upstream repository itself. +// window flags, the HVI backend, the disks of an overlay-block root and the +// generic container initrd -- is the urunc-dev/urunc branch +// feat/hvi-darwin-linux-parity, on feat/unchanged_containers-exec-fixes. The +// require below is feat/urunit-single-binary, that branch with urunit's +// single binary on top. No replace: the module is the upstream repository +// itself. // // hull tracks that branch rather than urunc main. main has no hvi backend at // all and renders netdev.* with no guard for an empty address, so it cannot diff --git a/go.sum b/go.sum index b0186b98..64643d76 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cyphar.com/go-pathrs v0.2.5 h1:SnX9FBvnoyn3lUs1dkMgZ52bAETpirNu3FTRh5HlRik= cyphar.com/go-pathrs v0.2.5/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc= +cyphar.com/go-pathrs v0.2.6 h1:+btfBJ7qWKZo+TDJlDZ2tPqToKdN8QD4BAOqomueXZg= +cyphar.com/go-pathrs v0.2.6/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= @@ -36,6 +38,8 @@ github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151X github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/log v0.2.0 h1:BewD/umNgVnoczglOpX8eRMyEy5t5iPlu5AIpnWDONc= +github.com/containerd/log v0.2.0/go.mod h1:/M7L7CXKcPTfNC74XzaK+5H5KbO5+4lJVpuVI6vRLoM= github.com/containerd/stargz-snapshotter/estargz v0.16.3 h1:7evrXtoh1mSbGj/pfRccTampEyKpjpOnS3CyiV1Ebr8= github.com/containerd/stargz-snapshotter/estargz v0.16.3/go.mod h1:uyr4BfYfOj3G9WBVE8cOlQmXAbPN9VEQpBBeJIuOipU= github.com/containerd/typeurl/v2 v2.3.0 h1:HZHPhRWo5XMy3QGQoPrUzbW/2ckwjfweHmOwlkIrPAQ= @@ -126,6 +130,8 @@ github.com/jsimonetti/rtnetlink/v2 v2.0.1 h1:xda7qaHDSVOsADNouv7ukSuicKZO7GgVUCX github.com/jsimonetti/rtnetlink/v2 v2.0.1/go.mod h1:7MoNYNbb3UaDHtF8udiJo/RH6VsTKP1pqKLUTVCvToE= github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ= +github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -134,6 +140,8 @@ github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g= @@ -156,6 +164,8 @@ github.com/moby/sys/user v0.4.1 h1:RgjRlaDKi/Xmyrz4t8lyzXT6v2ooFeO/7xtchmhVWE0= github.com/moby/sys/user v0.4.1/go.mod h1:E9QsW5WRe1kUAf7kW8hXKwu1uhsZEAdPLYHYSDudF4Y= github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/sys/userns v0.2.1 h1:4OvdM7BcPkASbuouHsbW3aeMJSFlYDldBRnXVZhaRk8= +github.com/moby/sys/userns v0.2.1/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/mrunalp/fileutils v0.5.1 h1:F+S7ZlNKnrwHfSwdlgNSkKo67ReVf8o9fel6C3dkm/Q= github.com/mrunalp/fileutils v0.5.1/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/nubificus/hedge_cli v0.0.3 h1:psu0tXb9XbzREp1S6AW4VJytbgdKP7tZ+HOXcz4W8Xk= @@ -172,6 +182,8 @@ github.com/opencontainers/runtime-spec v1.2.1 h1:S4k4ryNgEpxW1dzyqffOmhI1BHYcjzU github.com/opencontainers/runtime-spec v1.2.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/selinux v1.13.1 h1:A8nNeceYngH9Ow++M+VVEwJVpdFmrlxsN22F+ISDCJE= github.com/opencontainers/selinux v1.13.1/go.mod h1:S10WXZ/osk2kWOYKy1x2f/eXF5ZHJoUs8UU/2caNRbg= +github.com/opencontainers/selinux v1.15.1 h1:ERxeh5caJvCzNAKdI8WQbJmB1LDTn4BuaAg8wihLBpA= +github.com/opencontainers/selinux v1.15.1/go.mod h1:LenyElirjUHszfxrjuFqC85HIeXZKumHcKMQtnaDlQQ= github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE= github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -191,6 +203,8 @@ github.com/seccomp/libseccomp-golang v0.10.0 h1:aA4bp+/Zzi0BnWZ2F1wgNBs5gTpm+na2 github.com/seccomp/libseccomp-golang v0.10.0/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo= +github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q= github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -203,12 +217,17 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 h1:pyC9PaHYZFgEKFdlp3G8RaCKgVpHZnecvArXvPXcFkM= github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701/go.mod h1:P3a5rG4X7tI17Nn3aOIAYr5HbIMukwXG0urG0WuL8OA= github.com/urfave/cli/v3 v3.10.1 h1:7Kx9H50hrHbRbyxgO1KP6/BcbiGRz0uYh5YyQ30JEEY= github.com/urfave/cli/v3 v3.10.1/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso= +github.com/urfave/cli/v3 v3.13.0 h1:Dr6jqMfIyyFsRVn7Nz5mqLsMY+ZMpfh3a0aMs+umPVY= +github.com/urfave/cli/v3 v3.13.0/go.mod h1:vXn6HxPNccJSzQr2QvwVncOKrgYGIHU0HY5h8B2nQj4= github.com/urunc-dev/urunc v0.8.1-0.20261006162953-6c062f39c5bb h1:ZuHAauDqufq5kSGD4GnHb6+AOcGycDUAFwL55BfKDmE= github.com/urunc-dev/urunc v0.8.1-0.20261006162953-6c062f39c5bb/go.mod h1:CqCuV1DG/U4HDOg0TDUS1lENsT5xE1UbOuLUT8xnhp8= +github.com/urunc-dev/urunc v0.8.2-0.20261018212029-770a597f0a1d h1:hOfv62RfSEhdeOt/JWaCOQPMrNYmn8Itd2i6iQfdr1M= +github.com/urunc-dev/urunc v0.8.2-0.20261018212029-770a597f0a1d/go.mod h1:CA20jINXKdw8u7n5KlrChwKuYmAfbX2KJPOXPdKk00Q= github.com/vbatts/tar-split v0.12.1 h1:CqKoORW7BUWBe7UL/iqTVvkTBOF8UvOMKOIZykxnnbo= github.com/vbatts/tar-split v0.12.1/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA= github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0= @@ -226,6 +245,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y= golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -234,6 +255,8 @@ golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPI golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -243,12 +266,16 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= +golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -259,12 +286,18 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= +golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= +golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -275,6 +308,8 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -284,6 +319,8 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto/googleapis/rpc v0.0.0-20260622175928-b703f567277d h1:mpAgMyM9vQHxycBlDq50y1VHpfSfVwzXvrQKtYbXuUY= google.golang.org/genproto/googleapis/rpc v0.0.0-20260622175928-b703f567277d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260921155816-b14227669459 h1:b0xCahf3FK2m2Cv0p4vTozGPWncCvLfwV86UNg8xWU8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260921155816-b14227669459/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= @@ -291,6 +328,8 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= +google.golang.org/grpc v1.84.0 h1:soMyaPJ8pAak5PIQ0DGBUir0XRo2fRoMqhNWMLlLxO0= +google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -304,6 +343,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= From 1ba952f2293c13f1b4b8ee4650bbdc9ed948136b Mon Sep 17 00:00:00 2001 From: Anastassios Nanos Date: Fri, 9 Oct 2026 15:00:26 +0300 Subject: [PATCH 3/7] chore(deps): Bump hvi-vmm to main for disks and --fs hvi-vmm moves to 590c45d, brig-sh/hvi-vmm main, from c53a857. For this branch: - --disk path=

[,readonly=on|off][,serial=] attaches up to four disks, each its own virtio-mmio device. A fifth fails the boot. - A read-only disk is opened O_RDONLY and advertises VIRTIO_BLK_F_RO; writes get VIRTIO_BLK_S_IOERR. GET_ID answers the serial, so the guest finds its disks in /sys/block/vd*/serial. - --fs tag=,path=

[,readonly=on|off][,uid=..,gid=..] names a virtio-fs export and its owner. - arm64 guests get the host's wall-clock time from a PL031 RTC. It also brings main's refactors since c53a857: the vCPU lifecycle and kick signal shared across backends, and a run that owns its stop. Signed-off-by: Anastassios Nanos --- hvi-vmm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hvi-vmm b/hvi-vmm index c53a857a..590c45de 160000 --- a/hvi-vmm +++ b/hvi-vmm @@ -1 +1 @@ -Subproject commit c53a857a1e481a7dd366d67eee3e7768d1e59ad0 +Subproject commit 590c45de0806f04ddf8a0140421ad8c9a6292635 From 254f25682ba3dd50d924543ec109d722b5afa9fb Mon Sep 17 00:00:00 2001 From: Anastassios Nanos Date: Fri, 9 Oct 2026 14:31:12 +0300 Subject: [PATCH 4/7] feat(vz-runner): Add --disk and --disk-ro block devices - --disk and --disk-ro , repeatable, append virtio-blk devices after the --rootfs/EFI disk, with blockDeviceIdentifier disk counted over --disk* only. --disk-ro attaches read-only. - Checkpoint clones each writable --disk to disk.img and lists all of them under "disks" in latest.json; read-only ones are recorded by path. Restore puts the writable clones back and fails when one is missing. - Without --disk* the devices and the manifest are unchanged. Signed-off-by: Anastassios Nanos --- vz-runner/Sources/main.swift | 98 +++++++++++++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 2 deletions(-) diff --git a/vz-runner/Sources/main.swift b/vz-runner/Sources/main.swift index 395615eb..14faceb2 100644 --- a/vz-runner/Sources/main.swift +++ b/vz-runner/Sources/main.swift @@ -11,6 +11,14 @@ struct SharedDir { var readOnly: Bool } +// DiskSpec is one --disk or --disk-ro image. The guest finds it by its +// virtio-blk serial, disk, where N is its position among the --disk* +// flags; /dev/vdX order is not part of the contract. +struct DiskSpec { + var path: String + var readOnly: Bool +} + // Ceilings for the numeric flags. They are deliberately far above anything a // real host can satisfy -- Virtualization.framework clamps to the machine's // actual limits later -- and exist only to keep a hostile or fat-fingered @@ -67,6 +75,7 @@ struct VMConfig { var noNet = false var stopGraceSeconds: Int = 10 var sharedDirs: [SharedDir] = [] + var disks: [DiskSpec] = [] // --disk / --disk-ro, in argv order var useDirectBoot: Bool = true // Try VZLinuxBootLoader first var stateDir: String = "" // checkpoint artifacts live here; empty disables checkpointing var restore: Bool = false // boot by restoring the saved state instead of a cold start @@ -89,6 +98,13 @@ struct VMConfig { case "--rootfs": i += 1 if i < args.count { config.rootfsPath = args[i] } + case "--disk", "--disk-ro": + // Repeatable, in any interleaving. Each one becomes its own + // virtio-blk device after the --rootfs/EFI disk, with serial + // disk counted over these flags only. + let readOnly = (args[i] == "--disk-ro") + i += 1 + if i < args.count { config.disks.append(DiskSpec(path: args[i], readOnly: readOnly)) } case "--cmdline": i += 1 if i < args.count { config.cmdline = args[i] } @@ -191,6 +207,10 @@ struct VMConfig { var machineIDPath: String { stateDir + "/machine-id" } var vmStatePath: String { stateDir + "/vm.vzstate" } var diskStatePath: String { stateDir + "/rootfs.img" } + // The checkpoint clone of the writable --disk with serial disk. + // Read-only disks are never cloned: nothing can have changed them. + func diskCloneName(_ n: Int) -> String { "disk\(n).img" } + func diskClonePath(_ n: Int) -> String { stateDir + "/" + diskCloneName(n) } var manifestPath: String { stateDir + "/latest.json" } var errorPath: String { stateDir + "/error" } } @@ -805,6 +825,19 @@ class VZManager: NSObject, VZVirtualMachineDelegate { vmConfig.storageDevices.append(VZVirtioBlockDeviceConfiguration(attachment: rootfsAttachment)) } + // --disk / --disk-ro, after the legacy disks, each with a stable + // serial so the guest does not depend on enumeration order. A + // read-only disk is enforced by the device, not by a mount option. + for (n, disk) in config.disks.enumerated() { + let attachment = try VZDiskImageStorageDeviceAttachment( + url: URL(fileURLWithPath: disk.path), readOnly: disk.readOnly) + let device = VZVirtioBlockDeviceConfiguration(attachment: attachment) + let serial = "disk\(n)" + try VZVirtioBlockDeviceConfiguration.validateBlockDeviceIdentifier(serial) + device.blockDeviceIdentifier = serial + vmConfig.storageDevices.append(device) + } + // Memory balloon — only without a state dir: Virtualization.framework // refuses to save/restore configurations that contain balloon devices, // and the runner never drives the balloon anyway. @@ -1054,14 +1087,46 @@ class VZManager: NSObject, VZVirtualMachineDelegate { print("VZRunner: checkpoint: disk clone failed (cp -c exit \(status))", to: &standardError) } } + // The --disk* devices: writable ones are cloned beside the + // machine state like the rootfs; read-only ones are recorded + // by path only, since the guest cannot have changed them. An + // entry whose clone failed names no image, the same signal + // diskImage gives for the rootfs. + var diskEntries: [[String: Any]] = [] + if saveError == nil { + for (n, disk) in self.config.disks.enumerated() { + var entry: [String: Any] = [ + "serial": "disk\(n)", + "path": disk.path, + "readOnly": disk.readOnly, + "image": "", + ] + if !disk.readOnly { + let clonePath = self.config.diskClonePath(n) + try? FileManager.default.removeItem(atPath: clonePath) + let status = (try? runShellQuietly("/bin/cp", ["-c", disk.path, clonePath])) ?? 1 + if status == 0 { + entry["image"] = self.config.diskCloneName(n) + } else { + print("VZRunner: checkpoint: disk\(n) clone failed (cp -c exit \(status))", to: &standardError) + } + } + diskEntries.append(entry) + } + } if saveError == nil { let elapsedMs = Int(Date().timeIntervalSince(started) * 1000) - let manifest: [String: Any] = [ + var manifest: [String: Any] = [ "stateFile": "vm.vzstate", "diskImage": diskCloned ? "rootfs.img" : "", "savedAt": ISO8601DateFormatter().string(from: started), "durationMs": elapsedMs, ] + // Absent without --disk*, so a single-rootfs manifest is + // unchanged. + if !self.config.disks.isEmpty { + manifest["disks"] = diskEntries + } if let data = try? JSONSerialization.data(withJSONObject: manifest, options: [.sortedKeys]) { try? data.write(to: URL(fileURLWithPath: self.config.manifestPath)) } @@ -1087,6 +1152,32 @@ class VZManager: NSObject, VZVirtualMachineDelegate { // restoreDisk puts the checkpointed rootfs clone back in place of the // live disk image. Must run before createVM attaches the file. func restoreDisk() throws { + try restoreRootfsDisk() + try restoreExtraDisks() + } + + // restoreExtraDisks puts back the clone of every writable --disk. A + // read-only disk was never cloned and is attached as given. A writable + // disk with no clone fails the restore: the machine state would rewind + // and that disk would not. + func restoreExtraDisks() throws { + for (n, disk) in config.disks.enumerated() where !disk.readOnly { + let clonePath = config.diskClonePath(n) + guard FileManager.default.fileExists(atPath: clonePath) else { + throw NSError(domain: "VZRunner", code: 3, + userInfo: [NSLocalizedDescriptionKey: "no disk\(n) image in checkpoint for \(disk.path); the disk cannot rewind with the machine state"]) + } + try? FileManager.default.removeItem(atPath: disk.path) + let status = try runShellQuietly("/bin/cp", ["-c", clonePath, disk.path]) + guard status == 0 else { + throw NSError(domain: "VZRunner", code: 3, + userInfo: [NSLocalizedDescriptionKey: "failed to restore disk\(n) from checkpoint (cp -c exit \(status))"]) + } + print("VZRunner: restore: disk\(n) restored from checkpoint", to: &standardError) + } + } + + func restoreRootfsDisk() throws { guard !config.rootfsPath.isEmpty else { return } guard FileManager.default.fileExists(atPath: config.diskStatePath) else { print("VZRunner: restore: no disk image in checkpoint, keeping current rootfs", to: &standardError) @@ -1119,6 +1210,9 @@ class VZManager: NSObject, VZVirtualMachineDelegate { print(" Kernel: \(config.kernelPath)", to: &standardError) if !config.initrdPath.isEmpty { print(" Initrd: \(config.initrdPath)", to: &standardError) } if !config.rootfsPath.isEmpty { print(" Rootfs: \(config.rootfsPath)", to: &standardError) } + for (n, disk) in config.disks.enumerated() { + print(" Disk disk\(n): \(disk.path)\(disk.readOnly ? " (ro)" : "")", to: &standardError) + } print(" Cmdline: \(config.cmdline)", to: &standardError) print(" Memory: \(config.memoryMB)MB, CPUs: \(config.cpuCount)", to: &standardError) fflush(stderr) @@ -1402,7 +1496,7 @@ struct VzRunner { } guard let config = VMConfig.parse(args) else { - print("Usage: vz-runner --kernel [--initrd ] [--rootfs ] [--cmdline ] [--mem MB] [--cpus N] [--efi] [--share ] [--mac

] [--net-fd ] [--stop-grace ] [--qmp socket] [--state-dir ] [--restore] [--gui] [--gui-title ] [--rosetta]", to: &standardError) + print("Usage: vz-runner --kernel [--initrd ] [--rootfs ] [--disk ]... [--disk-ro ]... [--cmdline ] [--mem MB] [--cpus N] [--efi] [--share ] [--mac
] [--net-fd ] [--stop-grace ] [--qmp socket] [--state-dir ] [--restore] [--gui] [--gui-title ] [--rosetta]", to: &standardError) exit(1) } From d6c4bb3eedb0c41ef27a44d45b5c4d9e2a73ba6c Mon Sep 17 00:00:00 2001 From: Anastassios Nanos Date: Fri, 9 Oct 2026 14:33:13 +0300 Subject: [PATCH 5/7] feat(run): Cache a read-only ext4 image per image digest ensureLowerRootfs builds /rootfs.ext4 from the store rootfs with the block path's mke2fs -d, debugfs ownership pass and repair, factored out of buildBlockRootfs. No journal, root owned by 0:0, sized from the block-rounded contents plus 20%, mode 0400. The build runs under an flock on rootfs.ext4.lock into a temporary file renamed into place, and is redone when the image is missing or older than the image's unpack-schema stamp. The per-instance block rootfs is built as before. Signed-off-by: Anastassios Nanos --- cmd/hull/blockrootfs.go | 27 ++- cmd/hull/lowerrootfs.go | 188 +++++++++++++++++++++ cmd/hull/lowerrootfs_test.go | 314 +++++++++++++++++++++++++++++++++++ 3 files changed, 524 insertions(+), 5 deletions(-) create mode 100644 cmd/hull/lowerrootfs.go create mode 100644 cmd/hull/lowerrootfs_test.go diff --git a/cmd/hull/blockrootfs.go b/cmd/hull/blockrootfs.go index a784fa99..ffe55b29 100644 --- a/cmd/hull/blockrootfs.go +++ b/cmd/hull/blockrootfs.go @@ -64,10 +64,8 @@ type blockInject struct { // (see ociclient.GuestAttr) and is restored here, straight into the finished // filesystem, where ext4's inodes can hold what APFS could not. func buildBlockRootfs(ctx context.Context, diskPath, rootfsDir string, injects []blockInject, sizeMB int) error { - mkCmd := exec.CommandContext(ctx, mke2fsBin, "-t", "ext4", "-d", rootfsDir, - "-L", "rootfs", "-m", "0", "-q", diskPath, fmt.Sprintf("%dM", sizeMB)) - if out, err := mkCmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to create ext4 image: %s: %w", string(out), err) + if err := mkfsFromDir(ctx, diskPath, rootfsDir, sizeMB); err != nil { + return err } // mke2fs creates the image file itself, under the process umask, and it @@ -87,10 +85,29 @@ func buildBlockRootfs(ctx context.Context, diskPath, rootfsDir string, injects [ if err := os.Chmod(diskPath, 0o600); err != nil { return fmt.Errorf("failed to restrict the block image: %w", err) } + return applyBlockFixups(ctx, diskPath, rootfsDir, injects, filepath.Dir(diskPath)) +} + +// mkfsFromDir runs mke2fs to write an ext4 image of rootfsDir at diskPath. +// extra goes before the device argument, for options only some callers want. +func mkfsFromDir(ctx context.Context, diskPath, rootfsDir string, sizeMB int, extra ...string) error { + args := []string{"-t", "ext4", "-d", rootfsDir, "-L", "rootfs", "-m", "0", "-q"} + args = append(args, extra...) + args = append(args, diskPath, fmt.Sprintf("%dM", sizeMB)) + if out, err := exec.CommandContext(ctx, mke2fsBin, args...).CombinedOutput(); err != nil { + return fmt.Errorf("failed to create ext4 image: %s: %w", string(out), err) + } + return nil +} +// applyBlockFixups writes injects into a finished image, gives every path the +// ownership the unpack recorded for it, and repairs the filesystem summaries +// debugfs leaves stale. Its scratch files go in a new directory under +// stageParent, or under os.TempDir() when stageParent is empty. +func applyBlockFixups(ctx context.Context, diskPath, rootfsDir string, injects []blockInject, stageParent string) error { // The injected files need to exist on the host for debugfs to copy them in. // Three small files beside the image, not a second copy of the rootfs. - stage, err := os.MkdirTemp(filepath.Dir(diskPath), "block-inject-") + stage, err := os.MkdirTemp(stageParent, "block-inject-") if err != nil { return fmt.Errorf("failed to stage block image files: %w", err) } diff --git a/cmd/hull/lowerrootfs.go b/cmd/hull/lowerrootfs.go new file mode 100644 index 00000000..aedd1883 --- /dev/null +++ b/cmd/hull/lowerrootfs.go @@ -0,0 +1,188 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build darwin + +package main + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "time" + + "golang.org/x/sys/unix" +) + +const ( + // lowerRootfsName is the read-only ext4 image of an image's rootfs, + // kept beside it in the store and shared by every instance of the image. + lowerRootfsName = "rootfs.ext4" + // unpackSchemaStamp is the file the unpack writes last into an image + // directory (store.WriteUnpackSchema). A lower older than it was built + // from a rootfs that has since been rewritten. + unpackSchemaStamp = "unpack-schema" + // ext4BlockSize is the block size mke2fs picks for these image sizes. + ext4BlockSize = 4096 +) + +// ensureLowerRootfs returns the read-only ext4 image of imageDir's rootfs, +// building it on first use. +// +// The image is built once per digest with the block path's tools (mke2fs -d, +// then the debugfs ownership pass and repair), into a temporary file renamed +// into place, under an flock on /rootfs.ext4.lock so concurrent runs +// of the same image build it once. It is rebuilt when it is missing or older +// than the image's unpack-schema stamp. It carries no per-instance content, +// has no journal, and is made read-only on the host as well. +func ensureLowerRootfs(ctx context.Context, imageDir string) (string, error) { + disk := filepath.Join(imageDir, lowerRootfsName) + if lowerRootfsFresh(disk, imageDir) { + return disk, nil + } + + unlock, err := lockFile(ctx, disk+".lock") + if err != nil { + return "", err + } + defer unlock() + + // Another run may have built it while this one waited for the lock. + if lowerRootfsFresh(disk, imageDir) { + return disk, nil + } + + rootfsDir := filepath.Join(imageDir, "rootfs") + sizeMB, inodes, err := lowerRootfsGeometry(rootfsDir) + if err != nil { + return "", fmt.Errorf("build read-only rootfs image %s: %w", disk, err) + } + + // A staging file left by a run that died is garbage; start over. + staging := disk + ".tmp" + _ = os.Remove(staging) + started := time.Now() + if err := buildLowerRootfs(ctx, staging, rootfsDir, sizeMB, inodes); err != nil { + _ = os.Remove(staging) + return "", fmt.Errorf("build read-only rootfs image %s: %w", disk, err) + } + if err := os.Rename(staging, disk); err != nil { + _ = os.Remove(staging) + return "", fmt.Errorf("install read-only rootfs image: %w", err) + } + log.Debugf("Built read-only rootfs image %s (%d MiB apparent) in %s", + disk, sizeMB, time.Since(started).Round(time.Millisecond)) + return disk, nil +} + +// buildLowerRootfs writes the read-only image at diskPath. +func buildLowerRootfs(ctx context.Context, diskPath, rootfsDir string, sizeMB int, inodes int64) error { + if err := mkfsFromDir(ctx, diskPath, rootfsDir, sizeMB, + "-O", "^has_journal", + "-N", fmt.Sprint(inodes), + "-E", "assume_storage_prezeroed=1,root_owner=0:0"); err != nil { + return err + } + if err := os.Chmod(diskPath, 0o600); err != nil { + return fmt.Errorf("failed to restrict the read-only rootfs image: %w", err) + } + // Scratch files go to the system temp dir, not the store image dir, so a + // run that dies here leaves nothing in the image. + if err := applyBlockFixups(ctx, diskPath, rootfsDir, nil, ""); err != nil { + return err + } + // Nothing writes it again: every VMM attaches it read-only. + if err := os.Chmod(diskPath, 0o400); err != nil { + return fmt.Errorf("failed to make the rootfs image read-only: %w", err) + } + return nil +} + +// lowerRootfsFresh reports whether disk exists and is not older than the +// image's unpack-schema stamp. An image with no stamp has nothing to be +// compared against, so an existing disk is used. +func lowerRootfsFresh(disk, imageDir string) bool { + info, err := os.Stat(disk) + if err != nil || !info.Mode().IsRegular() { + return false + } + stamp, err := os.Stat(filepath.Join(imageDir, unpackSchemaStamp)) + if err != nil { + return errors.Is(err, os.ErrNotExist) + } + return !info.ModTime().Before(stamp.ModTime()) +} + +// lowerRootfsGeometry sizes the read-only image from the rootfs it holds: +// every file's size rounded up to a block, one block per directory, symlink +// and other entry, plus 20% and 64 MiB of slack for the filesystem's own +// metadata. The file is sparse, so the slack costs no host space. The inode +// count is the entry count plus 20% and 4096, since an image of many small +// files runs out of inodes before blocks at mke2fs's default ratio. +func lowerRootfsGeometry(rootfsDir string) (sizeMB int, inodes int64, err error) { + var bytes, entries int64 + walkErr := filepath.WalkDir(rootfsDir, func(_ string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + entries++ + if d.Type().IsRegular() { + info, infoErr := d.Info() + if infoErr != nil { + return infoErr + } + bytes += (info.Size() + ext4BlockSize - 1) / ext4BlockSize * ext4BlockSize + } else { + bytes += ext4BlockSize + } + return nil + }) + if walkErr != nil { + return 0, 0, fmt.Errorf("size image rootfs %s: %w", rootfsDir, walkErr) + } + total := bytes + bytes/5 + 64<<20 + return int((total + 1<<20 - 1) >> 20), entries + entries/5 + 4096, nil +} + +// lockFile takes an exclusive flock on path, creating it if needed, and +// returns the function that releases it. It polls so a cancelled run stops +// waiting. +func lockFile(ctx context.Context, path string) (func(), error) { + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o600) + if err != nil { + return nil, fmt.Errorf("open lock %s: %w", path, err) + } + for { + err := unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB) + if err == nil { + return func() { + _ = unix.Flock(int(f.Fd()), unix.LOCK_UN) + _ = f.Close() + }, nil + } + if !errors.Is(err, unix.EWOULDBLOCK) && !errors.Is(err, unix.EINTR) { + _ = f.Close() + return nil, fmt.Errorf("lock %s: %w", path, err) + } + select { + case <-ctx.Done(): + _ = f.Close() + return nil, ctx.Err() + case <-time.After(50 * time.Millisecond): + } + } +} diff --git a/cmd/hull/lowerrootfs_test.go b/cmd/hull/lowerrootfs_test.go new file mode 100644 index 00000000..7d0286c1 --- /dev/null +++ b/cmd/hull/lowerrootfs_test.go @@ -0,0 +1,314 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build darwin + +package main + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "syscall" + "testing" + "time" +) + +const e2fsckBin = "/opt/homebrew/opt/e2fsprogs/sbin/e2fsck" + +func requireE2fsprogs(t *testing.T) { + t.Helper() + for _, bin := range []string{mke2fsBin, debugfsBin, e2fsckBin} { + if _, err := os.Stat(bin); err != nil { + t.Skipf("%s is not installed: %v", bin, err) + } + } +} + +// newImageDir lays out a store image directory: a rootfs with a setuid binary +// owned by root and a root:shadow file, both recorded the way the unpack +// records them, and the unpack-schema stamp. +func newImageDir(t *testing.T) string { + t.Helper() + imageDir := t.TempDir() + rootfs := filepath.Join(imageDir, "rootfs") + for _, dir := range []string{"usr/bin", "etc"} { + if err := os.MkdirAll(filepath.Join(rootfs, dir), 0o755); err != nil { + t.Fatal(err) + } + } + sudo := filepath.Join(rootfs, "usr/bin/sudo") + if err := os.WriteFile(sudo, []byte("#!/bin/sh\n"), 0o755); err != nil { + t.Fatal(err) + } + recordAttr(t, sudo, 0o104755, 0, 0) + if err := syscall.Chmod(sudo, 0o4755); err != nil { + t.Fatal(err) + } + shadow := filepath.Join(rootfs, "etc/shadow") + if err := os.WriteFile(shadow, []byte("root:*:1::::::\n"), 0o640); err != nil { + t.Fatal(err) + } + recordAttr(t, shadow, 0o100640, 0, 42) + if err := os.WriteFile(filepath.Join(imageDir, unpackSchemaStamp), []byte("3\n"), 0o600); err != nil { + t.Fatal(err) + } + return imageDir +} + +func debugfsStat(t *testing.T, disk, path string) string { + t.Helper() + out, err := exec.Command(debugfsBin, "-R", "stat "+path, disk).CombinedOutput() + if err != nil { + t.Fatalf("debugfs stat %s: %s: %v", path, out, err) + } + return string(out) +} + +// The lower is the image as the guest must see it: setuid kept, the recorded +// ownership instead of the host user's, a clean filesystem, and no journal. +func TestLowerRootfsCarriesImageOwnership(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("as root the host ownership already matches") + } + requireE2fsprogs(t) + imageDir := newImageDir(t) + + disk, err := ensureLowerRootfs(context.Background(), imageDir) + if err != nil { + t.Fatalf("ensureLowerRootfs: %v", err) + } + if want := filepath.Join(imageDir, lowerRootfsName); disk != want { + t.Fatalf("lower at %s, want %s", disk, want) + } + info, err := os.Stat(disk) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o400 { + t.Errorf("lower is %v, want read-only for the owner", info.Mode().Perm()) + } + if _, err := os.Stat(disk + ".tmp"); err == nil { + t.Error("the staging file was left behind") + } + if left, _ := filepath.Glob(filepath.Join(imageDir, "block-inject-*")); len(left) > 0 { + t.Errorf("debugfs scratch staged in the store image dir: %v", left) + } + + out := debugfsStat(t, disk, "/usr/bin/sudo") + if !strings.Contains(out, "Mode: 04755") || !strings.Contains(out, "User: 0") { + t.Errorf("sudo lost its setuid bit or its owner:\n%s", out) + } + out = debugfsStat(t, disk, "/etc/shadow") + if !strings.Contains(out, "Group: 42") { + t.Errorf("shadow lost its group:\n%s", out) + } + out = debugfsStat(t, disk, "/") + if !strings.Contains(out, "User: 0 Group: 0") { + t.Errorf("the root directory is not root's:\n%s", out) + } + + feats, err := exec.Command(debugfsBin, "-R", "features", disk).CombinedOutput() + if err != nil { + t.Fatalf("debugfs features: %s: %v", feats, err) + } + if strings.Contains(string(feats), "has_journal") { + t.Errorf("the read-only lower has a journal: %s", feats) + } + if fsck, err := exec.Command(e2fsckBin, "-fn", disk).CombinedOutput(); err != nil { + t.Errorf("e2fsck -fn on the lower: %s: %v", fsck, err) + } +} + +// A lower newer than the unpack stamp is reused; one older than it, or a +// missing one, is built again. +func TestLowerRootfsRebuildsOnlyWhenStale(t *testing.T) { + requireE2fsprogs(t) + imageDir := newImageDir(t) + ctx := context.Background() + + disk, err := ensureLowerRootfs(ctx, imageDir) + if err != nil { + t.Fatal(err) + } + first, err := os.Stat(disk) + if err != nil { + t.Fatal(err) + } + + if _, err := ensureLowerRootfs(ctx, imageDir); err != nil { + t.Fatal(err) + } + again, err := os.Stat(disk) + if err != nil { + t.Fatal(err) + } + if !os.SameFile(first, again) { + t.Error("a fresh lower was rebuilt") + } + + // A re-unpack rewrites the stamp after the lower was built. + stamp := filepath.Join(imageDir, unpackSchemaStamp) + future := time.Now().Add(time.Hour) + if err := os.Chtimes(stamp, future, future); err != nil { + t.Fatal(err) + } + if _, err := ensureLowerRootfs(ctx, imageDir); err != nil { + t.Fatal(err) + } + rebuilt, err := os.Stat(disk) + if err != nil { + t.Fatal(err) + } + if os.SameFile(first, rebuilt) { + t.Error("a lower older than the unpack stamp was reused") + } + + if err := os.Remove(disk); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(stamp, time.Now(), time.Now()); err != nil { + t.Fatal(err) + } + if _, err := ensureLowerRootfs(ctx, imageDir); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(disk); err != nil { + t.Errorf("a missing lower was not rebuilt: %v", err) + } +} + +// Runs of the same image that start together build the lower once and all +// get the same file. +func TestLowerRootfsConcurrentRunsShareOneBuild(t *testing.T) { + requireE2fsprogs(t) + imageDir := newImageDir(t) + + const runs = 4 + var wg sync.WaitGroup + infos := make([]os.FileInfo, runs) + errs := make([]error, runs) + for i := 0; i < runs; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + disk, err := ensureLowerRootfs(context.Background(), imageDir) + if err != nil { + errs[i] = err + return + } + infos[i], errs[i] = os.Stat(disk) + }(i) + } + wg.Wait() + for i := 0; i < runs; i++ { + if errs[i] != nil { + t.Fatalf("run %d: %v", i, errs[i]) + } + if !os.SameFile(infos[0], infos[i]) { + t.Errorf("run %d got a different lower than run 0", i) + } + } +} + +// A staging file left by a run that died is replaced, not trusted, and does +// not outlive the build. +func TestLowerRootfsReplacesACrashLeftover(t *testing.T) { + requireE2fsprogs(t) + imageDir := newImageDir(t) + staging := filepath.Join(imageDir, lowerRootfsName+".tmp") + if err := os.WriteFile(staging, []byte("not an ext4 image"), 0o400); err != nil { + t.Fatal(err) + } + + disk, err := ensureLowerRootfs(context.Background(), imageDir) + if err != nil { + t.Fatalf("ensureLowerRootfs over a leftover staging file: %v", err) + } + if _, err := os.Stat(staging); err == nil { + t.Error("the leftover staging file is still there") + } + if fsck, err := exec.Command(e2fsckBin, "-fn", disk).CombinedOutput(); err != nil { + t.Errorf("e2fsck -fn on the rebuilt lower: %s: %v", fsck, err) + } +} + +// A build that fails reports it, leaves no staging file, and releases the +// lock, so the next run fails the same way instead of hanging. +func TestLowerRootfsFailureLeavesNothingBehind(t *testing.T) { + requireE2fsprogs(t) + imageDir := t.TempDir() // no rootfs/ in it + staging := filepath.Join(imageDir, lowerRootfsName+".tmp") + + for i := 0; i < 2; i++ { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + _, err := ensureLowerRootfs(ctx, imageDir) + cancel() + if err == nil { + t.Fatalf("call %d: built a lower from a missing rootfs", i) + } + if errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("call %d: blocked on the lock: %v", i, err) + } + if !strings.Contains(err.Error(), "build read-only rootfs image") { + t.Errorf("call %d: error carries no context: %v", i, err) + } + if _, statErr := os.Stat(staging); statErr == nil { + t.Errorf("call %d: a staging file was left behind", i) + } + } + if _, err := os.Stat(filepath.Join(imageDir, lowerRootfsName)); err == nil { + t.Error("a failed build installed a lower") + } +} + +// A cancelled run stops waiting for a lock another build holds. +func TestLockFileHonoursCancellation(t *testing.T) { + path := filepath.Join(t.TempDir(), "x.lock") + unlock, err := lockFile(context.Background(), path) + if err != nil { + t.Fatal(err) + } + defer unlock() + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + if _, err := lockFile(ctx, path); err == nil { + t.Fatal("took a lock that is already held") + } +} + +// The size covers block-rounded file contents with room to spare, and the +// inode count covers every entry. +func TestLowerRootfsGeometry(t *testing.T) { + rootfs := t.TempDir() + for i := 0; i < 10; i++ { + if err := os.WriteFile(filepath.Join(rootfs, "f"+string(rune('a'+i))), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + sizeMB, inodes, err := lowerRootfsGeometry(rootfs) + if err != nil { + t.Fatal(err) + } + if sizeMB < 64 { + t.Errorf("size %d MiB leaves no room for metadata", sizeMB) + } + if inodes < 11 { + t.Errorf("%d inodes for 11 entries", inodes) + } +} From 492d9f17258912f33d7219f2a68143fc8e38e39a Mon Sep 17 00:00:00 2001 From: Anastassios Nanos Date: Fri, 9 Oct 2026 14:35:37 +0300 Subject: [PATCH 6/7] feat(run): Add HULL_ROOTFS_MODE=overlay-block for container boots On hvi and vz the container root becomes an overlay of two disks: the image's cached read-only ext4 as disk0 and the instance's sparse 16 GiB ext4 upper as disk1, passed as ExecArgs.BlockDevs in that order. - /urunc-rootfs-lower (block:disk0:ext4) and /urunc-rootfs-upper (block:disk1) are staged into the instance initrd. - No rootfs share is exported; home and project shares are unchanged. - rdinit stays /vz-init. - Checkpoint refuses an overlay-block instance by name. The mode refuses HULL_HVI_ROOTFS_UPPER and --rootfs-type block, and is ignored with a warning outside a container boot. With neither env set the boot is unchanged. Signed-off-by: Anastassios Nanos --- cmd/hull/checkpoint.go | 12 +++-- cmd/hull/rootfsupper.go | 62 ++++++++++++++++++++++++- cmd/hull/rootfsupper_test.go | 88 ++++++++++++++++++++++++++++++++++++ cmd/hull/run.go | 50 ++++++++++++++++++-- 4 files changed, 205 insertions(+), 7 deletions(-) create mode 100644 cmd/hull/rootfsupper_test.go diff --git a/cmd/hull/checkpoint.go b/cmd/hull/checkpoint.go index edcabd6c..8d59dea5 100644 --- a/cmd/hull/checkpoint.go +++ b/cmd/hull/checkpoint.go @@ -439,10 +439,16 @@ func requireNoDescriptorShare(cmdLine []string) error { } func requireBlockRootfs(cmdLine []string) error { - if !slices.Contains(cmdLine, "--rootfs") { - return errors.New("checkpoint/restore needs a block rootfs: start the instance with `run --rootfs-type block` (a virtiofs root goes stale across restore)") + if slices.Contains(cmdLine, "--rootfs") { + return nil } - return nil + // vz-runner's --disk flags come only from HULL_ROOTFS_MODE=overlay-block, + // which refuses --rootfs-type block, so the generic advice would point at + // a combination the mode rejects. + if slices.Contains(cmdLine, "--disk") || slices.Contains(cmdLine, "--disk-ro") { + return errors.New("an overlay-block root (HULL_ROOTFS_MODE=overlay-block) cannot be checkpointed in this cut; run with the variable unset and --rootfs-type block") + } + return errors.New("checkpoint/restore needs a block rootfs: start the instance with `run --rootfs-type block` (a virtiofs root goes stale across restore)") } // vzRunnerProcess reports whether pid's argv looks like vz-runner, guarding diff --git a/cmd/hull/rootfsupper.go b/cmd/hull/rootfsupper.go index e808eb05..16bebecb 100644 --- a/cmd/hull/rootfsupper.go +++ b/cmd/hull/rootfsupper.go @@ -1,12 +1,31 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build darwin + package main import ( + "context" "errors" "fmt" "os" "os/exec" "path/filepath" "strings" + + "github.com/urunc-dev/urunc/pkg/unikontainers/types" ) // rootfsUpperDiskSize is the apparent size of an ext4 upper layer. The file is @@ -14,12 +33,16 @@ import ( // can add on top of its image. const rootfsUpperDiskSize = "16G" +// rootfsModeOverlayBlock is the HULL_ROOTFS_MODE value that boots a container +// from a read-only ext4 lower and a per-instance ext4 upper, both as disks. +const rootfsModeOverlayBlock = "overlay-block" + // prepareRootfsUpper returns the instance's overlay upper layer, creating it on // first boot and reusing it on every later one. // // kind "virtiofs" is a directory hvi exports read-write; vz-init makes the // upper and work directories inside it. kind "block" is a sparse ext4 image -// hvi attaches as the guest's only disk. mke2fs is told the storage is already +// the VMM attaches as a disk. mke2fs is told the storage is already // zeroed, which a fresh sparse file is, so it writes only the metadata it needs // and the guest has no inode tables left to initialise. func prepareRootfsUpper(instanceDir, kind string) (string, error) { @@ -54,3 +77,40 @@ func prepareRootfsUpper(instanceDir, kind string) (string, error) { } return "", fmt.Errorf("HULL_HVI_ROOTFS_UPPER must be virtiofs or block (got %q)", kind) } + +// overlayBlockRootfs is the disk layout of HULL_ROOTFS_MODE=overlay-block. +type overlayBlockRootfs struct { + // disks go on the VMM's command line in this order, so the lower is + // serial disk0 and the upper disk1. + disks []types.BlockDevSpec + // lowerMarker and upperMarker are the contents of /urunc-rootfs-lower + // and /urunc-rootfs-upper in the instance's initrd. + lowerMarker string + upperMarker string +} + +// prepareOverlayBlockRootfs returns the overlay-block layout for an instance +// of the image whose store rootfs is imageRootfs: the image's cached read-only +// ext4 as the lower and the instance's own sparse ext4 as the upper. +func prepareOverlayBlockRootfs(ctx context.Context, imageRootfs, instanceDir string) (overlayBlockRootfs, error) { + if filepath.Base(imageRootfs) != "rootfs" { + return overlayBlockRootfs{}, fmt.Errorf("HULL_ROOTFS_MODE=%s needs a store image rootfs, got %s", + rootfsModeOverlayBlock, imageRootfs) + } + lower, err := ensureLowerRootfs(ctx, filepath.Dir(imageRootfs)) + if err != nil { + return overlayBlockRootfs{}, err + } + upper, err := prepareRootfsUpper(instanceDir, "block") + if err != nil { + return overlayBlockRootfs{}, err + } + return overlayBlockRootfs{ + disks: []types.BlockDevSpec{ + {Path: lower, ReadOnly: true}, + {Path: upper}, + }, + lowerMarker: "block:disk0:ext4", + upperMarker: "block:disk1", + }, nil +} diff --git a/cmd/hull/rootfsupper_test.go b/cmd/hull/rootfsupper_test.go new file mode 100644 index 00000000..8f5634c6 --- /dev/null +++ b/cmd/hull/rootfsupper_test.go @@ -0,0 +1,88 @@ +// Copyright (c) 2026, NOFire AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build darwin + +package main + +import ( + "context" + "path/filepath" + "strings" + "testing" +) + +// The guest finds its disks by serial, and the serial is the position on the +// VMM's command line: the image's read-only lower first, the instance's +// writable upper second, and markers that name them that way. +func TestOverlayBlockLayoutOrdersLowerThenUpper(t *testing.T) { + requireE2fsprogs(t) + imageDir := newImageDir(t) + instanceDir := t.TempDir() + + layout, err := prepareOverlayBlockRootfs(context.Background(), + filepath.Join(imageDir, "rootfs"), instanceDir) + if err != nil { + t.Fatal(err) + } + if len(layout.disks) != 2 { + t.Fatalf("got %d disks, want 2", len(layout.disks)) + } + lower, upper := layout.disks[0], layout.disks[1] + if lower.Path != filepath.Join(imageDir, lowerRootfsName) || !lower.ReadOnly { + t.Errorf("disk0 is %+v, want the image's cached lower, read-only", lower) + } + if upper.Path != filepath.Join(instanceDir, "rootfs-upper.ext4") || upper.ReadOnly { + t.Errorf("disk1 is %+v, want the instance's upper, writable", upper) + } + if layout.lowerMarker != "block:disk0:ext4" || layout.upperMarker != "block:disk1" { + t.Errorf("markers %q / %q do not name disk0 and disk1", layout.lowerMarker, layout.upperMarker) + } +} + +// Checkpoint refuses an overlay-block instance by name, rather than pointing +// at --rootfs-type block, which that mode refuses; the legacy cases keep +// their answers. +func TestRequireBlockRootfsNamesOverlayBlock(t *testing.T) { + cases := []struct { + name string + argv []string + wantErr string + }{ + {"block rootfs", []string{"vz-runner", "--rootfs", "/i/rootfs.ext4"}, ""}, + {"overlay-block", []string{"vz-runner", "--disk-ro", "/img/rootfs.ext4", "--disk", "/i/upper.ext4"}, "HULL_ROOTFS_MODE=overlay-block"}, + {"virtiofs root", []string{"vz-runner", "--share-ro", "/img/rootfs", "rootfs"}, "--rootfs-type block"}, + } + for _, c := range cases { + err := requireBlockRootfs(c.argv) + switch { + case c.wantErr == "" && err != nil: + t.Errorf("%s: refused: %v", c.name, err) + case c.wantErr != "" && (err == nil || !strings.Contains(err.Error(), c.wantErr)): + t.Errorf("%s: got %v, want an error naming %q", c.name, err, c.wantErr) + } + } + if err := requireBlockRootfs([]string{"vz-runner", "--disk", "/i/upper.ext4"}); err == nil || + strings.Contains(err.Error(), "start the instance with") { + t.Errorf("overlay-block got the generic advice: %v", err) + } +} + +// Only a store image rootfs has an image directory to cache the lower in. +func TestOverlayBlockLayoutRefusesANonStoreRootfs(t *testing.T) { + if _, err := prepareOverlayBlockRootfs(context.Background(), + filepath.Join(t.TempDir(), "elsewhere"), t.TempDir()); err == nil { + t.Fatal("accepted a rootfs outside an image directory") + } +} diff --git a/cmd/hull/run.go b/cmd/hull/run.go index 2fb47495..bcdefbbb 100644 --- a/cmd/hull/run.go +++ b/cmd/hull/run.go @@ -758,6 +758,29 @@ func runInstance(ctx context.Context, cmd *cli.Command) error { // kind ("virtiofs" or "block") and the directory or disk that holds it. containerRootfsUpper := "" containerRootfsUpperPath := "" + // HULL_ROOTFS_MODE=overlay-block: the container root is an overlay of a + // read-only ext4 image of the store rootfs (disk0) and the instance's + // own ext4 upper (disk1). Both reach the VMM as block devices, and + // vz-init finds them by serial from the /urunc-rootfs-lower and + // /urunc-rootfs-upper markers. + containerRootfsLower := "" + var containerBlockDevs []types.BlockDevSpec + rootfsMode := os.Getenv("HULL_ROOTFS_MODE") + switch rootfsMode { + case "", rootfsModeOverlayBlock: + default: + return fmt.Errorf("HULL_ROOTFS_MODE must be %s or unset (got %q)", rootfsModeOverlayBlock, rootfsMode) + } + overlayBlock := containerBoot && rootfsMode == rootfsModeOverlayBlock + if rootfsMode != "" && !containerBoot { + log.Warnf("HULL_ROOTFS_MODE=%s applies to container boots only; ignored for this image", rootfsMode) + } + if overlayBlock && os.Getenv("HULL_HVI_ROOTFS_UPPER") != "" { + return errors.New("HULL_ROOTFS_MODE=overlay-block and HULL_HVI_ROOTFS_UPPER both choose the root layout; set one") + } + if overlayBlock && rootfsTypeOverride == "block" { + return errors.New("HULL_ROOTFS_MODE=overlay-block does not combine with --rootfs-type block") + } mountRootfs := ociSpec.Annotations["com.urunc.unikernel.mountRootfs"] == "true" if containerBoot { @@ -822,7 +845,18 @@ func runInstance(ctx context.Context, cmd *cli.Command) error { if resolveErr != nil { return fmt.Errorf("resolve unpacked OCI rootfs: %w", resolveErr) } - if upper := os.Getenv("HULL_HVI_ROOTFS_UPPER"); vmmType == hypervisors.HviVmm && upper != "" { + if overlayBlock { + layout, err := prepareOverlayBlockRootfs(ctx, resolved, s.InstanceDir(instanceName)) + if err != nil { + return err + } + rootfsDir = resolved + containerMetadataRootfs = resolved + containerBlockDevs = layout.disks + containerRootfsLower = layout.lowerMarker + containerRootfsUpper = layout.upperMarker + containerRootfsUpperPath = layout.disks[1].Path + } else if upper := os.Getenv("HULL_HVI_ROOTFS_UPPER"); vmmType == hypervisors.HviVmm && upper != "" { // Overlay: the cached image is the read-only lower and every // guest write lands in the instance's own upper layer, so // nothing is copied before the VMM starts. @@ -1031,6 +1065,11 @@ exec %s "$@" return fmt.Errorf("select direct writable rootfs in boot initrd: %w", err) } } + if containerRootfsLower != "" { + if err := initrd.AddFileToInitrd(initrdPath, containerRootfsLower+"\n", "/urunc-rootfs-lower"); err != nil { + return fmt.Errorf("select overlay lower layer in boot initrd: %w", err) + } + } if containerRootfsUpper != "" { if err := initrd.AddFileToInitrd(initrdPath, containerRootfsUpper+"\n", "/urunc-rootfs-upper"); err != nil { return fmt.Errorf("select overlay upper layer in boot initrd: %w", err) @@ -1383,6 +1422,8 @@ exec %s "$@" // export (virtiofs on Vz, 9p on QEMU) mounted by the init wrapper. sharedfsParams := types.SharedfsParams{} switch { + case overlayBlock: + // The root comes from disks; there is no rootfs share to export. case containerBoot && rootfsDir != "": sharedfsParams = types.SharedfsParams{ Type: "virtiofs", Path: rootfsDir, Tag: "rootfs", ReadOnly: !containerRootfsDirect, @@ -1404,8 +1445,10 @@ exec %s "$@" log.Debugf("Shared directory: host=%s guest=%s tag=%s readOnly=%t", sh.host, sh.guest, sh.tag, sh.readOnly) } - // vz-init mounts the upper layer by kind: a writable tag named - // rootfs-upper, or the guest's only disk. + // HULL_HVI_ROOTFS_UPPER: vz-init mounts the upper layer by kind, a + // writable tag named rootfs-upper or the guest's only disk. The + // overlay-block kinds name serials instead and their disks are already in + // containerBlockDevs. blockDevPath := rootfsDiskImage switch containerRootfsUpper { case "virtiofs": @@ -1439,6 +1482,7 @@ exec %s "$@" InitrdPath: initrdPath, RootfsPath: rootfsDir, BlockDevPath: blockDevPath, + BlockDevs: containerBlockDevs, LogFile: logFile, AgentSockPath: s.InstanceAgentSocket(instanceName), GUI: gui, From 607cc453e66173fb3838fccfdd009d8b0fe390c3 Mon Sep 17 00:00:00 2001 From: Anastassios Nanos Date: Fri, 9 Oct 2026 14:36:14 +0300 Subject: [PATCH 7/7] docs: Document HULL_ROOTFS_MODE and HULL_HVI_ROOTFS_UPPER - cli.md lists both variables. - storage.md adds the overlay rows to the root filesystem table, the cached lower and the instance upper to the store layout, and how the lower is built and invalidated. - checkpoint-restore.md lists the vz-runner --disk clones. Signed-off-by: Anastassios Nanos --- docs/checkpoint-restore.md | 1 + docs/cli.md | 2 ++ docs/storage.md | 17 ++++++++++++++++- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/checkpoint-restore.md b/docs/checkpoint-restore.md index d4085fa5..1e9bfb09 100644 --- a/docs/checkpoint-restore.md +++ b/docs/checkpoint-restore.md @@ -46,6 +46,7 @@ Artifacts live in `/instances//checkpoint/`: | `machine-id` | persisted `VZGenericMachineIdentifier`. Restore requires the identical machine identity, so every Vz instance persists one from first boot | | `vm.vzstate` | machine + memory state (encrypted by the framework, bound to this Mac) | | `rootfs.img` | APFS copy-on-write clone of the block rootfs at the checkpoint moment | +| `disk.img` | clone of each writable `vz-runner --disk`. Read-only `--disk-ro` images are not cloned; `latest.json` lists every disk under `disks` with its serial, path and clone name | | `latest.json`| manifest, written last; its mtime marks checkpoint completion | - **Block-mode rootfs** (`run --rootfs-type block`) is required: the disk is diff --git a/docs/cli.md b/docs/cli.md index f360de7f..61b9c44e 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -294,6 +294,8 @@ See [telemetry.md](telemetry.md). | Variable | What it does | |---|---| | `HULL_TERMINAL_FILTER` | `off`, `0`, `none` or `false` turns off the filter hull puts between guest output and your terminal, which blocks OSC 52 clipboard reads, DCS passthrough and cursor-position queries. Anything else leaves it on | +| `HULL_ROOTFS_MODE` | `overlay-block` boots a container (hvi and vz) from two disks: a read-only ext4 of the image, built once per image at `/images//rootfs.ext4`, as the overlay lower (serial `disk0`), and a per-instance sparse 16 GiB ext4 as the upper (`disk1`). Unset keeps the default root. Any other value is refused, and outside a container boot it is ignored with a warning. It cannot be checkpointed. See [storage.md](storage.md#where-the-guests-root-filesystem-actually-is) | +| `HULL_HVI_ROOTFS_UPPER` | hvi container boot only. `virtiofs` or `block` keeps the image's virtio-fs share as a read-only overlay lower and gives the instance an upper: a directory exported as `rootfs-upper`, or a sparse 16 GiB ext4 disk. Unset keeps the APFS clone. Cannot be combined with `HULL_ROOTFS_MODE` | | `CI` | counts the session as non-interactive, so the consent prompt never shows | | `TERM` | forwarded into the guest for a pty `exec` | | `COMPOSE_FILE`, `COMPOSE_PROJECT_NAME`, `COMPOSE_PROFILES`, `COMPOSE_DISABLE_ENV_FILE` | compose equivalents of the flags above | diff --git a/docs/storage.md b/docs/storage.md index 8f72cba3..f9e6cc21 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -182,7 +182,9 @@ in the name only keeps two pulls' directories apart. │ ├── oci-config.json │ ├── rootfs/ the unpacked image │ ├── unpack-schema the layout version rootfs/ was written with -│ └── rootfs-entries how many entries rootfs/ held when published +│ ├── rootfs-entries how many entries rootfs/ held when published +│ ├── rootfs.ext4 read-only ext4 of rootfs/, HULL_ROOTFS_MODE=overlay-block only, 0400 +│ └── rootfs.ext4.lock flock serializing that build ├── assets/ generic boot assets, shared across instances │ ├── Image the kernel; bzImage on amd64 │ ├── container-initrd @@ -194,6 +196,8 @@ in the name only keeps two pulls' directories apart. ├── bundle/ config.json plus the rootfs symlink or image ├── log 0600, detached runs only ├── checkpoint/ machine-id, vm.vzstate, rootfs.img, latest.json + ├── rootfs-upper.ext4 overlay upper disk (overlay-block, or HULL_HVI_ROOTFS_UPPER=block) + ├── rootfs-upper/ overlay upper directory (HULL_HVI_ROOTFS_UPPER=virtiofs) └── (QMP socket, agent socket, staged boot files) ``` @@ -223,6 +227,8 @@ survive. | `hvi`, generic container boot | a same-volume APFS copy-on-write clone of the cached image rootfs, exported read-write | **Yes**, in that instance's directory, until `hull rm` | | `virtiofs` or `9pfs`, not generic container boot | a full `cp -c -a` APFS clone of the image rootfs inside the instance directory | **Yes**, until `hull rm` | | `block` | a per-instance ext4 image at `/bundle/rootfs.ext4`, mode 0600 | **Yes**, until `hull rm` | +| `HULL_ROOTFS_MODE=overlay-block` (`hvi` and `vz`, generic container boot) | overlayfs: the image's read-only ext4 `/rootfs.ext4` as the lower (disk serial `disk0`) and `/rootfs-upper.ext4` as the upper (`disk1`) | **Yes**, in the upper disk, until `hull rm` | +| `HULL_HVI_ROOTFS_UPPER=virtiofs` or `block` (`hvi`, generic container boot) | overlayfs: the shared image cache directory over read-only virtiofs as the lower, `/rootfs-upper/` or `/rootfs-upper.ext4` as the upper | **Yes**, in the upper, until `hull rm` | A freshly created instance holds no copy of the image at all: the bundle starts as a `config.json` plus a `rootfs` symlink into the image cache. What replaces @@ -244,6 +250,15 @@ nominal filesystem capacity. Building it reads the cached rootfs and never duplicates it: the per-instance files are injected into the finished image with `debugfs`. +The overlay-block lower is built the same way, once per image: `mke2fs -d` +from the cached rootfs, the recorded ownership applied with `debugfs`, no +journal, sized at the block-rounded contents plus 20%. The first run of an +image pays for the build; later runs reuse the file until a re-unpack makes +the image's unpack-schema stamp newer than it. The upper is created sparse on +first boot and costs only what the guest writes. `hull rmi` and `hull prune` +remove the lower with the image. `hull prune` counts the cached lower at its +allocated size; `hull images` does not include it. + ## Credentials on disk: read this before forwarding a secret `--env KEY` without a value inherits `KEY` from hull's own environment. It does