diff --git a/deploy/node/README.md b/deploy/node/README.md index 28ceb9616..4ab87636c 100644 --- a/deploy/node/README.md +++ b/deploy/node/README.md @@ -5,6 +5,7 @@ These modules are packaged as `node-install.pyz`, which Web serves at `/node-ins | Module | Role | | --- | --- | | `node_install.py`, `node_spec.py`, `node_generations.py` | Node installer and generation helper. `node_spec.py` is generated by `go run ./services/core/cmd/specification-contract -write` | +| `host_docker.py`, `host_microsandbox.py` | One module per Sandbox Provider: its host checks, Runtime import, node configuration and uninstall cleanup. `node_install.py` selects the module in its `HOSTS` table by the provider Core chose and names no provider elsewhere; a new node Provider adds a module and a table entry | | `distribution.py` | Verified artifacts, private temporary files, release metadata and Docker image identity | | `install_display.py`, `node_output.py` | Terminal output | | `provider_assets.py` | Node provider artifact names, generated by `go run ./services/core/cmd/provider-artifacts` | diff --git a/deploy/node/host_docker.py b/deploy/node/host_docker.py new file mode 100644 index 000000000..52489e03b --- /dev/null +++ b/deploy/node/host_docker.py @@ -0,0 +1,161 @@ +"""Docker host preparation for a node; node_install.py selects it through its HOSTS table. + +Every function takes the installer module first, as node_generations.py does, so this file needs no import of it.""" +import json +import os +from pathlib import Path +import re +import shutil +import stat + +import distribution + +NAME = "Docker" +COMMAND = ("docker", "--host", "unix:///var/run/docker.sock") +SOCKET = Path("/var/run/docker.sock") +# The group that owns the provider's device; the node's service user joins only this one. +GROUP = "docker" +UNIT_AFTER = "network-online.target docker.service" +# Unpacked Runtime copies the Provider's store keeps on the disk beyond the one the import unpacks. +EXTRA_UNPACKED_COPIES = 0 +# A root-owned, empty Docker configuration for the service user's docker calls, so no +# CLI plugin or credential helper the service user controls runs in the installer. +CHILD_CONFIG = Path("/run/oac-node-docker") + + +def network(installation_id): + return "oac-node-" + installation_id + + +def networks(installer): + return installer.checked(list(COMMAND) + ["network", "ls", "--format", "{{.Name}}"], "Cannot inspect Docker networks").splitlines() + + +def containers(installer, installation_id): + return installer.checked(list(COMMAND) + ["ps", "--all", "--filter", "label=io.oac.installation=" + installation_id, + "--format", "{{.Names}}"], "Cannot inspect Docker containers").split() + + +def preflight(installer): + installer.checked(list(COMMAND) + ["info", "--format", "{{.ServerVersion}}"], "Docker access through /var/run/docker.sock is required") + + +def host_check(installer): + """Check Docker without installing it; return the group that grants access and the engine's details.""" + if shutil.which("docker") is None: + raise installer.InstallError("Docker Engine is not installed. Install it (https://docs.docker.com/engine/install/), " + "then rerun this command." + installer.NOTHING_CHANGED) + try: + info = SOCKET.stat() + except OSError: + info = None + if info is None or not stat.S_ISSOCK(info.st_mode): + raise installer.InstallError("Docker is not running: run `sudo systemctl enable --now docker`, then rerun this command." + installer.NOTHING_CHANGED) + if info.st_gid == 0 or stat.S_IMODE(info.st_mode) & 0o060 != 0o060: + raise installer.InstallError("Nodes reach Docker at /var/run/docker.sock through its group, but the socket is not " + "group-accessible (rootless Docker is not supported)." + installer.NOTHING_CHANGED) + try: + details = json.loads(installer.checked(list(COMMAND) + ["info", "--format", "{{json .}}"], "Docker is not running")) + except (installer.InstallError, ValueError): + raise installer.InstallError("Docker is not running: run `sudo systemctl enable --now docker`, then rerun this command." + installer.NOTHING_CHANGED) from None + # docker info's JSON uses the Engine API names (CpuCfsQuota), not the Go field names. + if details.get("MemoryLimit") is not True or details.get("CpuCfsQuota") is not True: + raise installer.InstallError("Docker on this host does not enforce CPU and memory limits; use cgroup v2, then rerun this command." + installer.NOTHING_CHANGED) + return installer.device_group(GROUP, SOCKET, info.st_gid), details + + +def capacity(installer, details): + return details.get("NCPU", 0), details.get("MemTotal", 0) + + +def other_node(installer, args): + # A network that no container uses is a leftover (Docker was down during an uninstall); the install reuses it. + if network(args.installation_id) in networks(installer) and containers(installer, args.installation_id): + raise installer.InstallError("Another node for this installation already uses this Docker engine (network " + network(args.installation_id) + + "). Remove it on the Nodes page and uninstall it first." + installer.NOTHING_CHANGED) + + +def configure(installer, root, args): + if getattr(args, "checkpoint_root", None) is not None: + raise installer.InstallError("Checkpoint root applies only to microsandbox nodes") + + +def native_config(installer, root, args, manifest, runtime_image): + return {"host": "unix:///var/run/docker.sock", "image": runtime_image, "network": network(args.installation_id), + "seccomp_file": str(root / "runtime/seccomp.json"), "nested_sandbox": True} + + +def runtime_imported(installer, root, args, manifest): + return distribution.docker_image(manifest, "runtime", COMMAND) is not None + + +def prepare_runtime(installer, root, args, manifest): + image = distribution.ensure_docker_image( + manifest, "runtime", lambda: distribution.runtime_archive(manifest, root, getattr(args, "disposable", False)), list(COMMAND)) + installer.drop_runtime_archives(root) + name = network(args.installation_id) + if name not in networks(installer): + installer.checked(list(COMMAND) + ["network", "create", name], "Cannot create node Docker network") + return image + + +def verify_retained(installer, stored, runtime_image): + if stored.get("native", {}).get("image") != runtime_image: + raise installer.node_spec.SpecificationError("Retained Docker image differs; preserve the node and inspect its configuration") + + +def image_available(installer, value): + seccomp = Path(value["native"]["seccomp_file"]) + installer.existing_file(seccomp) + json.loads(seccomp.read_text()) + raw = installer.checked(list(COMMAND) + ["image", "inspect", value["native"]["image"], "--format", "{{.Id}} {{.Os}}/{{.Architecture}}"], "Cannot inspect pinned image") + return raw.strip() == value["native"]["image"] + " linux/amd64" + + +def prepare_host(installer): + """Create the root-owned empty directory the service user's docker calls read.""" + path = CHILD_CONFIG + if not path.exists() and not path.is_symlink(): + path.mkdir(mode=0o755) + os.chmod(path, 0o755) + info = path.lstat() + if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.geteuid() or stat.S_IMODE(info.st_mode) & 0o022: + raise installer.InstallError(str(path) + " must be a directory only root can write; inspect the host.") + + +def child_environment(): + return {"DOCKER_CONFIG": str(CHILD_CONFIG)} + + +def release(installer, installation_id): + """Remove the node's Docker network; never containers, volumes or images.""" + if shutil.which("docker") is None: + return + name = network(installation_id) + try: + if name not in networks(installer): + return + remaining = containers(installer, installation_id) + except installer.InstallError: + print("Docker is unavailable, so network " + name + " was not removed.") + return + if remaining: + print("Kept Docker network " + name + ": these containers still exist: " + ", ".join(remaining)) + return + installer.checked(list(COMMAND) + ["network", "rm", name], "Cannot remove Docker network " + name) + + +def check_removal(installer, installation_id): + pass + + +def after_remove(installer, installation_id, native, delete_store): + """Name the Runtime image the uninstall keeps.""" + image = native.get("image") + # The service user can write provider.json; print only what an image ID can be. + if isinstance(image, str) and re.fullmatch(r"sha256:[0-9a-f]{64}", image): + print("Kept the Runtime image " + image + "; remove it with `sudo docker image rm " + image + "` if no other node uses it.") + + +def kept_stores(installer): + return [] diff --git a/deploy/node/host_microsandbox.py b/deploy/node/host_microsandbox.py new file mode 100644 index 000000000..0c78c1696 --- /dev/null +++ b/deploy/node/host_microsandbox.py @@ -0,0 +1,231 @@ +"""microsandbox host preparation for a node; node_install.py selects it through its HOSTS table. + +Every function takes the installer module first, as node_generations.py does, so this file needs no import of it.""" +import hashlib +import json +import os +from pathlib import Path +import shutil +import socket +import stat +import subprocess +from urllib.parse import urlsplit +import uuid + +import distribution +import provider_assets + +NAME = "microsandbox" +ARTIFACTS = provider_assets.artifacts("microsandbox", ("runtime",)) +KVM = Path("/dev/kvm") +# The group that owns the provider's device; the node's service user joins only this one. +GROUP = "kvm" +UNIT_AFTER = "network-online.target" +# Unpacked Runtime copies the Provider's store keeps on the disk beyond the one the import unpacks. +EXTRA_UNPACKED_COPIES = 1 + + +def home(installer, installation_id): + directory = Path.home() / ".oac/m" / hashlib.sha256(installation_id.encode()).hexdigest()[:12] + if len(os.fsencode(directory)) > 48: + raise installer.InstallError("HOME is too long for microsandbox Unix socket paths; use a user with a shorter persistent home directory") + return directory + + +def checkpoint_root(args): + return getattr(args, "checkpoint_root", None) or Path.home() / ".oac/checkpoints" / hashlib.sha256(args.installation_id.encode()).hexdigest()[:12] + + +def prepare_checkpoint_root(installer, directory, *, initialize=True): + directory = Path(directory) + if not directory.is_absolute() or directory.resolve() != directory: + raise installer.InstallError("Checkpoint root must be a canonical absolute directory") + if not initialize and not directory.is_dir(): + raise installer.InstallError("Retained checkpoint root is missing") + installer.safe_directory(directory) + marker = directory / ".oac-checkpoint-store" + if not installer.existing_file(marker): + if not initialize: + raise installer.InstallError("Retained checkpoint store identity is missing") + if any(directory.iterdir()): + raise installer.InstallError("Checkpoint root contains unowned state") + try: + descriptor = os.open(marker, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + except FileExistsError: + pass + else: + with os.fdopen(descriptor, "w") as output: + output.write(str(uuid.uuid4()) + "\n") + output.flush() + os.fsync(output.fileno()) + installer.existing_file(marker) + value = marker.read_text() + try: + identity = uuid.UUID(value.rstrip("\n")) + except ValueError: + raise installer.InstallError("Invalid checkpoint store identity") from None + if identity.int == 0 or value != str(identity) + "\n": + raise installer.InstallError("Invalid checkpoint store identity") + with marker.open("rb") as source: + os.fsync(source.fileno()) + descriptor = os.open(directory, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def environment(installer, root, args): + return dict(os.environ, MSB_BACKEND="local", MSB_HOME=str(home(installer, args.installation_id)), + MSB_PATH=str(root / ARTIFACTS[1]), MSB_LIBKRUNFW_PATH=str(root / ARTIFACTS[2])) + + +def image_ready(installer, root, args, manifest): + inspect = [str(root / ARTIFACTS[1]), "image", "inspect", manifest["runtime_ref"], "--format", "json"] + try: + value = json.loads(installer.checked(inspect, "Runtime image is not installed", env=environment(installer, root, args))) + return (value.get("digest") == manifest["runtime_ref"].split("@", 1)[1] + and value.get("architecture") == "amd64" and value.get("os") == "linux") + except (installer.InstallError, ValueError, AttributeError): + return False + + +def preflight(installer): + if not os.access(KVM, os.R_OK | os.W_OK): + raise installer.InstallError("microsandbox requires read/write access to /dev/kvm") + + +def host_check(installer): + """Check KVM without installing it; return the group that grants access (None when everyone has it).""" + try: + info = KVM.stat() + except OSError: + info = None + if info is None or not stat.S_ISCHR(info.st_mode): + raise installer.InstallError("KVM is unavailable: enable hardware virtualization (or nested virtualization for this VM) and " + "load kvm_intel or kvm_amd, then rerun this command." + installer.NOTHING_CHANGED) + if stat.S_IMODE(info.st_mode) & 0o006 == 0o006: + return None, {} + if info.st_gid == 0 or stat.S_IMODE(info.st_mode) & 0o060 != 0o060: + raise installer.InstallError("/dev/kvm must be group-accessible, for example root:kvm with mode 0660 (your distribution's " + "KVM package sets this), then rerun this command." + installer.NOTHING_CHANGED) + return installer.device_group(GROUP, KVM, info.st_gid), {} + + +def capacity(installer, details): + memory = 0 + with open("/proc/meminfo") as stream: + for line in stream: + if line.startswith("MemTotal:"): + memory = int(line.split()[1]) * 1024 + return os.cpu_count() or 0, memory + + +def other_node(installer, args): + pass + + +def configure(installer, root, args): + retained_provider = installer.private_json(root / "provider.json") + if retained_provider: + retained_root = Path(retained_provider["native"]["checkpoint_root"]) + if getattr(args, "checkpoint_root", None) not in (None, retained_root): + raise installer.InstallError("Retained checkpoint root differs; preserve its state") + args.checkpoint_root = retained_root + runtime_home = home(installer, args.installation_id) + prepare_checkpoint_root(installer, checkpoint_root(args), initialize=retained_provider is None) + installer.safe_directory(runtime_home) + owner = runtime_home / "oac-installation.json" + if not owner.exists() and any(runtime_home.iterdir()): + raise installer.InstallError("Microsandbox home contains unowned state; refusing to adopt it") + installer.write_once(owner, installer.json_text({"installation_id": args.installation_id})) + + +def native_config(installer, root, args, manifest, runtime_image): + endpoint = urlsplit(args.core_url) + port = endpoint.port or (443 if endpoint.scheme == "https" else 80) + addresses = sorted({entry[4][0] for entry in socket.getaddrinfo(endpoint.hostname, port, type=socket.SOCK_STREAM)}) + core_rules = [{"action": "allow", "direction": "egress", "destination": address, "protocol": "tcp", "port": str(port)} for address in addresses] + return { + "helper_path": str(root / ARTIFACTS[0]), "runtime_path": str(root / ARTIFACTS[1]), "firmware_path": str(root / ARTIFACTS[2]), + "image": manifest["runtime_ref"], "runtime_sha256": manifest["microsandbox"]["runtime_sha256"], + "firmware_sha256": manifest["microsandbox"]["firmware_sha256"], + "runtime_home": str(home(installer, args.installation_id)), + "checkpoint_root": str(checkpoint_root(args)), + "network": {"default_egress": "deny", "default_ingress": "deny", "rules": core_rules + [ + {"action": "allow", "direction": "egress", "destination": "public"}, + {"action": "allow", "direction": "egress", "destination": "host", "protocol": "udp", "port": "53"}, + {"action": "allow", "direction": "egress", "destination": "host", "protocol": "tcp", "port": "53"}, + ]}, + } + + +def runtime_imported(installer, root, args, manifest): + return (root / ARTIFACTS[1]).is_file() and image_ready(installer, root, args, manifest) + + +def prepare_runtime(installer, root, args, manifest): + for name in ARTIFACTS: + result = subprocess.run(["ldd", str(root / name)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=30) + output = result.stdout.decode() + if "not found" in output or (result.returncode and "statically linked" not in output and "not a dynamic executable" not in output): + raise installer.InstallError("Install the microsandbox host shared-library prerequisites") + if not image_ready(installer, root, args, manifest): + archive = distribution.runtime_archive(manifest, root, getattr(args, "disposable", False)) + installer.checked([str(root / ARTIFACTS[1]), "image", "load", "--input", str(archive), "--tag", manifest["runtime_ref"], "--quiet"], + "Cannot import the microsandbox runtime image; check free disk space and host libraries", timeout=1800, + env=environment(installer, root, args)) + if not image_ready(installer, root, args, manifest): + raise installer.InstallError("Imported microsandbox runtime image identity or platform differs") + installer.drop_runtime_archives(root) + + +def verify_retained(installer, stored, runtime_image): + pass + + +def image_available(installer, value): + micro = value["native"] + for key in ("helper_path", "runtime_path", "firmware_path"): + if not installer.existing_file(Path(micro[key])): + return False + if (installer.file_digest(Path(micro["runtime_path"])) != micro["runtime_sha256"] + or installer.file_digest(Path(micro["firmware_path"])) != micro["firmware_sha256"]): + return False + env = dict(os.environ, MSB_BACKEND="local", MSB_HOME=micro["runtime_home"], MSB_PATH=micro["runtime_path"], MSB_LIBKRUNFW_PATH=micro["firmware_path"]) + image = json.loads(installer.checked([micro["runtime_path"], "image", "inspect", micro["image"], "--format", "json"], "Cannot inspect pinned image", env=env)) + return image.get("digest") == micro["image"].split("@", 1)[1] and image.get("architecture") == "amd64" and image.get("os") == "linux" + + +def prepare_host(installer): + pass + + +def child_environment(): + return {} + + +def release(installer, installation_id): + pass + + +def check_removal(installer, installation_id): + installer.no_links(home(installer, installation_id)) + + +def after_remove(installer, installation_id, native, delete_store): + """Delete the node's store once Core confirmed the removal; otherwise say how.""" + runtime_home = home(installer, installation_id) + if runtime_home.exists(): + installer.no_links(runtime_home) + if delete_store and installer.private_json(runtime_home / "oac-installation.json") == {"installation_id": installation_id}: + shutil.rmtree(runtime_home) + print("Deleted the microsandbox store " + str(runtime_home) + ".") + else: + sudo_mode = str(Path.home()) == str(installer.SERVICE_HOME) + print("Kept the microsandbox store " + str(runtime_home) + " with its images and any sandbox state. Delete it with `" + + ("sudo -u " + installer.SERVICE_USER + " rm -rf " if sudo_mode else "rm -rf ") + str(runtime_home) + "`.") + + +def kept_stores(installer): + return [str(installer.SERVICE_HOME / ".oac/m" / name) for name in installer.listdir_nofollow(installer.SERVICE_HOME, ".oac", "m")] diff --git a/deploy/node/node_generations.py b/deploy/node/node_generations.py index 911b653f8..6ba58aff8 100644 --- a/deploy/node/node_generations.py +++ b/deploy/node/node_generations.py @@ -37,7 +37,7 @@ def helper_archive(args, installer): archive = io.BytesIO() source_dir = Path(installer.__file__).parent with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_DEFLATED) as package: - for name in ("node_install.py", "node_spec.py", "distribution.py", "node_generations.py", "install_display.py", "node_output.py", "provider_assets.py"): + for name in ("node_install.py", "node_spec.py", "distribution.py", "node_generations.py", "install_display.py", "node_output.py", "provider_assets.py", "host_docker.py", "host_microsandbox.py"): package.write(source_dir / name, "__main__.py" if name == "node_install.py" else name) return archive.getvalue() @@ -122,22 +122,7 @@ def owned_root(args, installer): def image_available(value, installer): """Whether the installation's Runtime that this configuration names is still on the node.""" try: - if value["provider"] == "docker": - seccomp = Path(value["native"]["seccomp_file"]) - installer.existing_file(seccomp) - json.loads(seccomp.read_text()) - raw = installer.checked(list(installer.DOCKER) + ["image", "inspect", value["native"]["image"], "--format", "{{.Id}} {{.Os}}/{{.Architecture}}"], "Cannot inspect pinned image") - return raw.strip() == value["native"]["image"] + " linux/amd64" - micro = value["native"] - for key in ("helper_path", "runtime_path", "firmware_path"): - if not installer.existing_file(Path(micro[key])): - return False - if (installer.file_digest(Path(micro["runtime_path"])) != micro["runtime_sha256"] - or installer.file_digest(Path(micro["firmware_path"])) != micro["firmware_sha256"]): - return False - env = dict(os.environ, MSB_BACKEND="local", MSB_HOME=micro["runtime_home"], MSB_PATH=micro["runtime_path"], MSB_LIBKRUNFW_PATH=micro["firmware_path"]) - image = json.loads(installer.checked([micro["runtime_path"], "image", "inspect", micro["image"], "--format", "json"], "Cannot inspect pinned image", env=env)) - return image.get("digest") == micro["image"].split("@", 1)[1] and image.get("architecture") == "amd64" and image.get("os") == "linux" + return installer.HOSTS[value["provider"]].image_available(installer, value) except (installer.InstallError, OSError, ValueError): return False @@ -161,7 +146,7 @@ def prepare(args, installer): base = installer.private_json(root / "provider.json") value = retained_configs(root, installer).get(args.generation) if value is not None: - installer.node_spec.verify_provider(value, configuration, value.get("native", {}).get("image")) + installer.verify_retained(value, configuration, value.get("native", {}).get("image")) else: value = copy.deepcopy(base) value["generation"], value["specification"] = args.generation, configuration["specification"] diff --git a/deploy/node/node_install.py b/deploy/node/node_install.py index b32b3c1b1..a65a097af 100644 --- a/deploy/node/node_install.py +++ b/deploy/node/node_install.py @@ -31,6 +31,8 @@ import uuid import distribution +import host_docker +import host_microsandbox import provider_assets import node_spec import node_generations @@ -42,10 +44,10 @@ class InstallError(Exception): pass -MICRO = provider_assets.artifacts("microsandbox", ("runtime",)) -DOCKER = ("docker", "--host", "unix:///var/run/docker.sock") -DOCKER_SOCKET = Path("/var/run/docker.sock") -KVM = Path("/dev/kvm") +# The one place that names the Sandbox Providers: each module prepares its provider's host, and the +# flow below calls them by the provider Core selected. +HOSTS = {"docker": host_docker, "microsandbox": host_microsandbox} +SELF = sys.modules[__name__] # Sudo mode: run as root, the installer prepares the host itself. The node runs as # this dedicated service user under a root-owned system unit; root never runs a @@ -60,9 +62,6 @@ class InstallError(Exception): SYSTEMD_RUNNING = Path("/run/systemd/system") SELINUX_ENFORCE = Path("/sys/fs/selinux/enforce") SAFE_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" -# A root-owned, empty Docker configuration for the service user's docker calls, so no -# CLI plugin or credential helper the service user controls runs in the installer. -CHILD_DOCKER_CONFIG = Path("/run/oac-node-docker") LOCK_WAIT_SECONDS = 600 NOTHING_CHANGED = " Nothing was changed." DISK_FULL = "The disk is full. Free space under " + str(SERVICE_HOME) + ", then rerun the command." @@ -147,10 +146,7 @@ def preflight(provider): """Check host access after dropping to the node service account.""" if sys.version_info < (3, 9) or platform.system() != "Linux" or platform.machine() not in ("x86_64", "amd64") or os.getuid() == 0: raise InstallError("Run with Python 3.9+ as a non-root user on Linux amd64") - if provider == "docker": - checked(list(DOCKER) + ["info", "--format", "{{.ServerVersion}}"], "Docker access through /var/run/docker.sock is required") - elif not os.access(KVM, os.R_OK | os.W_OK): - raise InstallError("microsandbox requires read/write access to /dev/kvm") + HOSTS[provider].preflight(SELF) class NoRedirect(urllib.request.HTTPRedirectHandler): @@ -286,83 +282,6 @@ def download(source, name, root, expected, replace=False): os.chmod(target, 0o700) -def micro_home(installation_id): - directory = Path.home() / ".oac/m" / hashlib.sha256(installation_id.encode()).hexdigest()[:12] - if len(os.fsencode(directory)) > 48: - raise InstallError("HOME is too long for microsandbox Unix socket paths; use a user with a shorter persistent home directory") - return directory - - -def checkpoint_root(args): - return getattr(args, "checkpoint_root", None) or Path.home() / ".oac/checkpoints" / hashlib.sha256(args.installation_id.encode()).hexdigest()[:12] - - -def prepare_checkpoint_root(directory, *, initialize=True): - directory = Path(directory) - if not directory.is_absolute() or directory.resolve() != directory: - raise InstallError("Checkpoint root must be a canonical absolute directory") - if not initialize and not directory.is_dir(): - raise InstallError("Retained checkpoint root is missing") - safe_directory(directory) - marker = directory / ".oac-checkpoint-store" - if not existing_file(marker): - if not initialize: - raise InstallError("Retained checkpoint store identity is missing") - if any(directory.iterdir()): - raise InstallError("Checkpoint root contains unowned state") - try: - descriptor = os.open(marker, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) - except FileExistsError: - pass - else: - with os.fdopen(descriptor, "w") as output: - output.write(str(uuid.uuid4()) + "\n") - output.flush() - os.fsync(output.fileno()) - existing_file(marker) - value = marker.read_text() - try: - identity = uuid.UUID(value.rstrip("\n")) - except ValueError: - raise InstallError("Invalid checkpoint store identity") from None - if identity.int == 0 or value != str(identity) + "\n": - raise InstallError("Invalid checkpoint store identity") - with marker.open("rb") as source: - os.fsync(source.fileno()) - descriptor = os.open(directory, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) - try: - os.fsync(descriptor) - finally: - os.close(descriptor) - - -def provider_config(root, args, manifest, runtime_image): - result = {"installation_id": args.installation_id, "provider": args.provider, "core_url": args.core_url + "/api/v1", - "specification": args.configuration["specification"], "generation": args.configuration["generation"]} - if args.provider == "docker": - result["native"] = {"host": "unix:///var/run/docker.sock", "image": runtime_image, - "network": "oac-node-" + args.installation_id, - "seccomp_file": str(root / "runtime/seccomp.json"), "nested_sandbox": True} - else: - endpoint = urlsplit(args.core_url) - port = endpoint.port or (443 if endpoint.scheme == "https" else 80) - addresses = sorted({entry[4][0] for entry in socket.getaddrinfo(endpoint.hostname, port, type=socket.SOCK_STREAM)}) - core_rules = [{"action": "allow", "direction": "egress", "destination": address, "protocol": "tcp", "port": str(port)} for address in addresses] - result["native"] = { - "helper_path": str(root / MICRO[0]), "runtime_path": str(root / MICRO[1]), "firmware_path": str(root / MICRO[2]), - "image": manifest["runtime_ref"], "runtime_sha256": manifest["microsandbox"]["runtime_sha256"], - "firmware_sha256": manifest["microsandbox"]["firmware_sha256"], - "runtime_home": str(micro_home(args.installation_id)), - "checkpoint_root": str(checkpoint_root(args)), - "network": {"default_egress": "deny", "default_ingress": "deny", "rules": core_rules + [ - {"action": "allow", "direction": "egress", "destination": "public"}, - {"action": "allow", "direction": "egress", "destination": "host", "protocol": "udp", "port": "53"}, - {"action": "allow", "direction": "egress", "destination": "host", "protocol": "tcp", "port": "53"}, - ]}, - } - return result - - def drop_runtime_archives(root): """The imported image is the cache; the archives only hold disk.""" for name in ("images/runtime.tar", "images/runtime.tar.gz"): @@ -370,46 +289,16 @@ def drop_runtime_archives(root): distribution.partial_path(root / name).unlink(missing_ok=True) -def micro_env(root, args): - return dict(os.environ, MSB_BACKEND="local", MSB_HOME=str(micro_home(args.installation_id)), - MSB_PATH=str(root / MICRO[1]), MSB_LIBKRUNFW_PATH=str(root / MICRO[2])) - - -def micro_image_ready(root, args, manifest): - inspect = [str(root / MICRO[1]), "image", "inspect", manifest["runtime_ref"], "--format", "json"] - try: - value = json.loads(checked(inspect, "Runtime image is not installed", env=micro_env(root, args))) - return (value.get("digest") == manifest["runtime_ref"].split("@", 1)[1] - and value.get("architecture") == "amd64" and value.get("os") == "linux") - except (InstallError, ValueError, AttributeError): - return False +def provider_config(root, args, manifest, runtime_image): + result = {"installation_id": args.installation_id, "provider": args.provider, "core_url": args.core_url + "/api/v1", + "specification": args.configuration["specification"], "generation": args.configuration["generation"]} + result["native"] = HOSTS[args.provider].native_config(SELF, root, args, manifest, runtime_image) + return result -def prepare_runtime(root, args, manifest): - if args.provider == "docker": - docker = ["docker", "--host", "unix:///var/run/docker.sock"] - image = distribution.ensure_docker_image( - manifest, "runtime", lambda: distribution.runtime_archive(manifest, root, getattr(args, "disposable", False)), docker) - drop_runtime_archives(root) - network = "oac-node-" + args.installation_id - networks = checked(docker + ["network", "ls", "--format", "{{.Name}}"], "Cannot inspect Docker networks").splitlines() - if network not in networks: - checked(docker + ["network", "create", network], "Cannot create node Docker network") - return image - else: - for name in MICRO: - result = subprocess.run(["ldd", str(root / name)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=30) - output = result.stdout.decode() - if "not found" in output or (result.returncode and "statically linked" not in output and "not a dynamic executable" not in output): - raise InstallError("Install the microsandbox host shared-library prerequisites") - if not micro_image_ready(root, args, manifest): - archive = distribution.runtime_archive(manifest, root, getattr(args, "disposable", False)) - checked([str(root / MICRO[1]), "image", "load", "--input", str(archive), "--tag", manifest["runtime_ref"], "--quiet"], - "Cannot import the microsandbox runtime image; check free disk space and host libraries", timeout=1800, - env=micro_env(root, args)) - if not micro_image_ready(root, args, manifest): - raise InstallError("Imported microsandbox runtime image identity or platform differs") - drop_runtime_archives(root) +def verify_retained(stored, configuration, runtime_image): + node_spec.verify_provider(stored, configuration) + HOSTS[configuration["provider"]].verify_retained(SELF, stored, runtime_image) def unit_name(installation_id): @@ -433,22 +322,7 @@ def configure_node(root, args, token): discard_unregistered(root) args.provider = args.configuration["provider"] preflight(args.provider) - if args.provider != "microsandbox" and getattr(args, "checkpoint_root", None) is not None: - raise InstallError("Checkpoint root applies only to microsandbox nodes") - if args.provider == "microsandbox": - retained_provider = private_json(root / "provider.json") - if retained_provider: - retained_root = Path(retained_provider["native"]["checkpoint_root"]) - if getattr(args, "checkpoint_root", None) not in (None, retained_root): - raise InstallError("Retained checkpoint root differs; preserve its state") - args.checkpoint_root = retained_root - runtime_home = micro_home(args.installation_id) - prepare_checkpoint_root(checkpoint_root(args), initialize=retained_provider is None) - safe_directory(runtime_home) - owner = runtime_home / "oac-installation.json" - if not owner.exists() and any(runtime_home.iterdir()): - raise InstallError("Microsandbox home contains unowned state; refusing to adopt it") - write_once(owner, json_text({"installation_id": args.installation_id})) + HOSTS[args.provider].configure(SELF, root, args) @contextlib.contextmanager @@ -473,7 +347,7 @@ def require_disk(root, args, manifest, names): """Refuse before downloading when the node account's disk cannot hold what is still missing. The Runtime archive and its unpacked copy exist only until the image is imported; a - microsandbox store then keeps about one more unpacked image under the same disk.""" + Provider's store may then keep more unpacked copies under the same disk.""" needed = 0 for name in names: if name != "runtime/seccomp.json": @@ -481,19 +355,15 @@ def require_disk(root, args, manifest, names): target = root / name if not (target.is_file() and distribution.matches(target, entry)): needed += entry["size"] - distribution.resumable(distribution.partial_path(target), entry["size"]) - if args.provider == "docker": - imported = distribution.docker_image(manifest, "runtime", DOCKER) is not None - else: - imported = (root / MICRO[1]).is_file() and micro_image_ready(root, args, manifest) - if not imported: + host = HOSTS[args.provider] + if not host.runtime_imported(SELF, root, args, manifest): archive = distribution.artifact(manifest, "images/runtime.tar.gz") packed = root / "images/runtime.tar.gz" if not packed.is_file(): needed += archive["size"] - distribution.resumable(distribution.partial_path(packed), archive["size"]) if not (root / "images/runtime.tar").is_file(): needed += archive["unpacked_size"] - if args.provider == "microsandbox": - needed += archive["unpacked_size"] + needed += host.EXTRA_UNPACKED_COPIES * archive["unpacked_size"] free = shutil.disk_usage(root).free if free < needed: raise InstallError("Not enough disk space under %s: this installation needs about %.1f GB free and %.1f GB is available. " @@ -524,20 +394,20 @@ def register_node(root, args, token, helper_archive=None, *, secret_path): existing_file(target) distribution.obtain_artifact(manifest, name, target, args.disposable) os.chmod(target, 0o700) - node_generations.install_helper(root, args, sys.modules[__name__], helper_archive) + node_generations.install_helper(root, args, SELF, helper_archive) safe_directory(root / "state/node") lease_identity = {"installation_id": args.installation_id, "generation": args.configuration["generation"], "specification_digest": args.configuration["specification_digest"]} - with node_generations.collection_lease(root, args.configuration["generation"], sys.modules[__name__], lease_identity, + with node_generations.collection_lease(root, args.configuration["generation"], SELF, lease_identity, initialize=not (root / "provider.json").exists()): pass install_display.step("Checking the sandbox runtime") - runtime_image = prepare_runtime(root, args, manifest) + runtime_image = HOSTS[args.provider].prepare_runtime(SELF, root, args, manifest) # Retain the original network policy when recovering a partial installation. if not existing_file(root / "provider.json"): write_once(root / "provider.json", json_text(provider_config(root, args, manifest, runtime_image))) else: - node_spec.verify_provider(json.loads((root / "provider.json").read_text()), args.configuration, runtime_image) + verify_retained(json.loads((root / "provider.json").read_text()), args.configuration, runtime_image) if not existing_file(marker): # The one-time credential is never passed through process arguments or service environments. with os.fdopen(os.open(secret_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "w") as secret: @@ -746,7 +616,9 @@ def service_child(account, function, arguments, parent, mask, read_ends, output_ proxies = proxy_settings(os.environ) os.environ.clear() os.environ.update(HOME=account.pw_dir, USER=account.pw_name, LOGNAME=account.pw_name, PATH=SAFE_PATH, - LANG="C.UTF-8", DOCKER_CONFIG=str(CHILD_DOCKER_CONFIG)) + LANG="C.UTF-8") + for host in HOSTS.values(): + os.environ.update(host.child_environment()) os.environ.update(proxies) os.chdir(account.pw_dir) function(*arguments) @@ -814,17 +686,6 @@ def write_service_environment(path, content): return True -def child_docker_config(): - """Create the root-owned empty directory the service user's docker calls read.""" - path = CHILD_DOCKER_CONFIG - if not path.exists() and not path.is_symlink(): - path.mkdir(mode=0o755) - os.chmod(path, 0o755) - info = path.lstat() - if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.geteuid() or stat.S_IMODE(info.st_mode) & 0o022: - raise InstallError(str(path) + " must be a directory only root can write; inspect the host.") - - @contextlib.contextmanager def host_lock(): """Serialize root-owned installation and removal, including account changes.""" @@ -940,83 +801,29 @@ def ours(account, recorded_uid=None): if (account is None or account.pw_dir != str(SERVICE_HOME) or not account.pw_shell.endswith(("nologin", "false")) or recorded_uid not in (None, account.pw_uid) or account.pw_uid == 0 or account.pw_gid == 0): return False - allowed = {account.pw_gid} | {gid for gid in map(group_id, DEVICE_GROUPS.values()) if gid is not None} + allowed = {account.pw_gid} | {gid for gid in (group_id(host.GROUP) for host in HOSTS.values()) if gid is not None} return set(os.getgrouplist(account.pw_name, account.pw_gid)) <= allowed -def host_capacity(provider, resources, docker_info): - if provider == "docker": - cpus, memory = docker_info.get("NCPU", 0), docker_info.get("MemTotal", 0) - else: - cpus, memory = os.cpu_count() or 0, 0 - with open("/proc/meminfo") as stream: - for line in stream: - if line.startswith("MemTotal:"): - memory = int(line.split()[1]) * 1024 +def host_capacity(provider, resources, details): + cpus, memory = HOSTS[provider].capacity(SELF, details) if cpus < resources["cpus"] or memory < resources["memory_mib"] * 1024 * 1024: raise InstallError("This host has %d CPUs and %d MiB of memory; each sandbox needs %d CPUs and %d MiB." % (cpus, memory // (1024 * 1024), resources["cpus"], resources["memory_mib"]) + NOTHING_CHANGED) -# The service user joins only the group that owns the provider's device, and only -# these names; a device owned by a privileged group (disk, sudo, wheel, adm) is refused. -DEVICE_GROUPS = {"docker": "docker", "microsandbox": "kvm"} - - -def provider_group(provider): - """Check Docker or KVM without installing either; return the group that grants access.""" - if provider == "docker": - if shutil.which("docker") is None: - raise InstallError("Docker Engine is not installed. Install it (https://docs.docker.com/engine/install/), " - "then rerun this command." + NOTHING_CHANGED) - try: - info = DOCKER_SOCKET.stat() - except OSError: - info = None - if info is None or not stat.S_ISSOCK(info.st_mode): - raise InstallError("Docker is not running: run `sudo systemctl enable --now docker`, then rerun this command." + NOTHING_CHANGED) - if info.st_gid == 0 or stat.S_IMODE(info.st_mode) & 0o060 != 0o060: - raise InstallError("Nodes reach Docker at /var/run/docker.sock through its group, but the socket is not " - "group-accessible (rootless Docker is not supported)." + NOTHING_CHANGED) - try: - details = json.loads(checked(list(DOCKER) + ["info", "--format", "{{json .}}"], "Docker is not running")) - except (InstallError, ValueError): - raise InstallError("Docker is not running: run `sudo systemctl enable --now docker`, then rerun this command." + NOTHING_CHANGED) from None - # docker info's JSON uses the Engine API names (CpuCfsQuota), not the Go field names. - if details.get("MemoryLimit") is not True or details.get("CpuCfsQuota") is not True: - raise InstallError("Docker on this host does not enforce CPU and memory limits; use cgroup v2, then rerun this command." + NOTHING_CHANGED) - return device_group(provider, DOCKER_SOCKET, info.st_gid), details - try: - info = KVM.stat() - except OSError: - info = None - if info is None or not stat.S_ISCHR(info.st_mode): - raise InstallError("KVM is unavailable: enable hardware virtualization (or nested virtualization for this VM) and " - "load kvm_intel or kvm_amd, then rerun this command." + NOTHING_CHANGED) - if stat.S_IMODE(info.st_mode) & 0o006 == 0o006: - return None, {} - if info.st_gid == 0 or stat.S_IMODE(info.st_mode) & 0o060 != 0o060: - raise InstallError("/dev/kvm must be group-accessible, for example root:kvm with mode 0660 (your distribution's " - "KVM package sets this), then rerun this command." + NOTHING_CHANGED) - return device_group(provider, KVM, info.st_gid), {} - - -def device_group(provider, device, gid): +def device_group(expected, device, gid): + """The name of the group that owns the Provider's device; the service user joins only the Provider's own.""" try: name = grp.getgrgid(gid).gr_name except KeyError: name = str(gid) - if name != DEVICE_GROUPS[provider]: + if name != expected: raise InstallError(str(device) + " belongs to the group " + name + "; the node's service user joins only the " - + DEVICE_GROUPS[provider] + " group. Give the device that group, then rerun this command." + NOTHING_CHANGED) + + expected + " group. Give the device that group, then rerun this command." + NOTHING_CHANGED) return name -def installation_containers(installation_id): - return checked(list(DOCKER) + ["ps", "--all", "--filter", "label=io.oac.installation=" + installation_id, - "--format", "{{.Names}}"], "Cannot inspect Docker containers").split() - - def other_node(args, provider): """Refuse a second sudo-mode Core on this host, or a second node for this installation. @@ -1037,12 +844,7 @@ def other_node(args, provider): if home.startswith("/") and listdir_nofollow(home, ".oac", "nodes", args.installation_id): raise InstallError("Existing node state for this installation belongs to " + sudo_user + ". Preserve it and arrange cleanup separately before installing a system node." + NOTHING_CHANGED) - if provider == "docker": - networks = checked(list(DOCKER) + ["network", "ls", "--format", "{{.Name}}"], "Cannot inspect Docker networks").splitlines() - # A network that no container uses is a leftover (Docker was down during an uninstall); the install reuses it. - if "oac-node-" + args.installation_id in networks and installation_containers(args.installation_id): - raise InstallError("Another node for this installation already uses this Docker engine (network oac-node-" - + args.installation_id + "). Remove it on the Nodes page and uninstall it first." + NOTHING_CHANGED) + HOSTS[provider].other_node(SELF, args) def account_plan(): @@ -1103,10 +905,9 @@ def prepare_account(account, record, group): def system_unit(root, provider, environment): def quote(value): return '"' + str(value).replace("\\", "\\\\").replace('"', '\\"').replace("%", "%%") + '"' - after = "network-online.target docker.service" if provider == "docker" else "network-online.target" # Root owns this file; the service user can change only its own node files. The # service runs with the account's own primary group. - return ("[Unit]\nDescription=OpenAgentCore sandbox node " + root.name + "\nWants=network-online.target\nAfter=" + after + return ("[Unit]\nDescription=OpenAgentCore sandbox node " + root.name + "\nWants=network-online.target\nAfter=" + HOSTS[provider].UNIT_AFTER + "\nStartLimitIntervalSec=0\n\n[Service]\nType=exec\nUser=" + SERVICE_USER + "\nExecStart=:" + quote(root / provider_assets.artifacts(provider, ("node",))[0]) + " run --config " + quote(root / "provider.json") + " --state-dir " + quote(root / "state/node") + "\nWorkingDirectory=" + str(root).replace("%", "%%") @@ -1129,7 +930,7 @@ def node_record(installation_id): if record is not None: expected = {"installation_id": installation_id, "node_root": str(SERVICE_HOME / ".oac/nodes" / installation_id), "unit": str(SYSTEM_UNITS / unit_name(installation_id))} - if any(record.get(key) != value for key, value in expected.items()) or record.get("provider") not in DEVICE_GROUPS: + if any(record.get(key) != value for key, value in expected.items()) or record.get("provider") not in HOSTS: raise InstallError(str(SYSTEM_RECORDS / (installation_id + ".json")) + " is not this installer's record; preserve " "it and inspect the host." + NOTHING_CHANGED) recorded_origin(record.get("core_url"), str(SYSTEM_RECORDS / (installation_id + ".json"))) @@ -1166,7 +967,7 @@ def install_system(args, token): raise InstallError("This host's node uses " + record["core_url"] + ", but this command uses " + args.core_url + ". Remove the node on the Nodes page, uninstall it, then run a new command." + NOTHING_CHANGED) install_display.step("Checking host requirements") - group, details = provider_group(provider) + group, details = HOSTS[provider].host_check(SELF) if record is None: other_node(args, provider) host_capacity(provider, configuration["specification"]["resources"], details) @@ -1174,11 +975,12 @@ def install_system(args, token): if node_record(args.installation_id) != recorded or (record is None and [i for i in node_records() if i != args.installation_id]): raise InstallError("Another node installation changed this host meanwhile; rerun the command." + NOTHING_CHANGED) account, account_record = account_plan() - helper_archive = node_generations.helper_archive(args, sys.modules[__name__]) + helper_archive = node_generations.helper_archive(args, SELF) # Every check has passed; from here on the host changes. install_display.step("Preparing the node service account") account = prepare_account(account, account_record, group) - child_docker_config() + for host in HOSTS.values(): + host.prepare_host(SELF) root = SERVICE_HOME / ".oac/nodes" / args.installation_id unit = SYSTEM_UNITS / unit_name(args.installation_id) entry = {"format": 1, "installation_id": args.installation_id, "provider": provider, @@ -1197,7 +999,7 @@ def install_system(args, token): checked(["systemctl", "is-active", "--quiet", unit.name], "Node service is unavailable; inspect sudo journalctl -u " + unit.name) install_display.step("Waiting for Core connection and provider readiness") run_as(account, wait_ready, root, args) - node_output.summary(root, args, unit.name, SERVICE_USER) + node_output.summary(root, args, unit.name, SERVICE_USER, HOSTS[args.provider].NAME) # Uninstall ------------------------------------------------------------------- @@ -1236,7 +1038,8 @@ def confirm_removed(root, core_url, force): Runs as the node's own user. The Core address comes from the caller: in sudo mode the root-owned record, never a file the service user can write.""" no_links(root) - no_links(micro_home(root.name)) # Refused here, before anything is removed, so "Nothing was changed" holds. + for host in HOSTS.values(): # Refused here, before anything is removed, so "Nothing was changed" holds. + host.check_removal(SELF, root.name) identity_file = root / "state/node/identity.json" if not has_node_state(root): return @@ -1268,49 +1071,19 @@ def confirm_removed(root, core_url, force): "if this Core no longer exists." + NOTHING_CHANGED) -def release_docker_network(installation_id): - """Remove the node's Docker network; never containers, volumes or images.""" - if shutil.which("docker") is None: - return - network = "oac-node-" + installation_id - try: - if network not in checked(list(DOCKER) + ["network", "ls", "--format", "{{.Name}}"], "Docker unavailable").splitlines(): - return - remaining = installation_containers(installation_id) - except InstallError: - print("Docker is unavailable, so network " + network + " was not removed.") - return - if remaining: - print("Kept Docker network " + network + ": these containers still exist: " + ", ".join(remaining)) - return - checked(list(DOCKER) + ["network", "rm", network], "Cannot remove Docker network " + network) - - def remove_node_files(root, installation_id, confirmed): - """Delete the node's state directory and, once Core confirmed the removal, its microsandbox store; keep the Runtime image. + """Delete the node's state directory and, once Core confirmed the removal, the Provider's store; keep the Runtime image. A node without registered state was never confirmed, so its store stays. Runs as the node's own user, so a link it planted can never reach another user's files.""" no_links(root) - provider = private_json(root / "provider.json") or {} - image = (provider.get("native") or {}).get("image") - runtime_home = micro_home(installation_id) + native = (private_json(root / "provider.json") or {}).get("native") or {} delete_store = confirmed and has_node_state(root) if root.exists(): shutil.rmtree(root) - # The service user can write provider.json; print only what an image ID can be. - if isinstance(image, str) and re.fullmatch(r"sha256:[0-9a-f]{64}", image): - print("Kept the Runtime image " + image + "; remove it with `sudo docker image rm " + image + "` if no other node uses it.") - if runtime_home.exists(): - no_links(runtime_home) - if delete_store and private_json(runtime_home / "oac-installation.json") == {"installation_id": installation_id}: - shutil.rmtree(runtime_home) - print("Deleted the microsandbox store " + str(runtime_home) + ".") - else: - sudo_mode = str(Path.home()) == str(SERVICE_HOME) - print("Kept the microsandbox store " + str(runtime_home) + " with its images and any sandbox state. Delete it with `" - + ("sudo -u " + SERVICE_USER + " rm -rf " if sudo_mode else "rm -rf ") + str(runtime_home) + "`.") + for host in HOSTS.values(): + host.after_remove(SELF, installation_id, native, delete_store) def uninstall_system(args): @@ -1341,7 +1114,8 @@ def uninstall_system(args): if core_url is None and not args.force: raise InstallError("This node's record is missing, so Core cannot be asked; rerun with --force only if " "Core no longer exists." + NOTHING_CHANGED) - child_docker_config() + for host in HOSTS.values(): + host.prepare_host(SELF) run_as(account, confirm_removed, root, core_url, args.force) if unit.exists(): checked(["systemctl", "disable", "--now", unit.name], "Cannot stop the node service " + unit.name) @@ -1350,7 +1124,8 @@ def uninstall_system(args): # A removed node's service ends failed (exit 78); drop that state with the unit. with contextlib.suppress(InstallError): checked(["systemctl", "reset-failed", unit.name], "Cannot reset " + unit.name) - release_docker_network(args.installation_id) + for host in HOSTS.values(): + host.release(SELF, args.installation_id) # --force skips Core's confirmation, so it never deletes the store. run_as(account, remove_node_files, root, args.installation_id, not args.force) (SYSTEM_RECORDS / (args.installation_id + ".json")).unlink(missing_ok=True) @@ -1370,12 +1145,13 @@ def release_account(account, record): print("The " + SERVICE_USER + " account differs from the one this installer recorded; it was left alone.") return if record.get("created"): - stores = [str(SERVICE_HOME / ".oac/m" / name) for name in listdir_nofollow(SERVICE_HOME, ".oac", "m")] - if stores: - # The names come from the service user's home; show them only as plain text. - print("Kept the " + SERVICE_USER + " user while microsandbox stores remain: " + plain(", ".join(stores), sys.stdout.encoding) - + ". Remove them with `sudo -u " + SERVICE_USER + " rm -rf `, then rerun this uninstall command.") - return + for host in HOSTS.values(): + stores = host.kept_stores(SELF) + if stores: + # The names come from the service user's home; show them only as plain text. + print("Kept the " + SERVICE_USER + " user while " + host.NAME + " stores remain: " + plain(", ".join(stores), sys.stdout.encoding) + + ". Remove them with `sudo -u " + SERVICE_USER + " rm -rf `, then rerun this uninstall command.") + return checked(["userdel", SERVICE_USER], "Cannot remove the " + SERVICE_USER + " user; stop its processes and rerun the uninstall command") # No process can run as the deleted user, so nothing races this removal. if SERVICE_HOME.is_dir() and not SERVICE_HOME.is_symlink(): @@ -1457,7 +1233,7 @@ def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--source-url", type=origin) parser.add_argument("--core-url", type=origin) - parser.add_argument("--provider", choices=("docker", "microsandbox"), help="Optional assertion; Core owns provider selection") + parser.add_argument("--provider", choices=tuple(HOSTS), help="Optional assertion; Core owns provider selection") parser.add_argument("--installation-id", required=True) parser.add_argument("--checkpoint-root", type=Path, help="Private microsandbox checkpoint directory, shared across compatible nodes when configured") parser.add_argument("--enrollment-token-stdin", action="store_true", help="Read the one-time enrollment token from standard input") @@ -1483,7 +1259,7 @@ def main(argv=None): if args.generation_action: if args.generation is None or not 1 <= args.generation <= 9223372036854775807 or not re.fullmatch(r"[0-9a-f]{64}", args.specification_digest or ""): parser.error("Invalid generation authorization") - (node_generations.prepare if args.generation_action == "prepare" else node_generations.collect)(args, sys.modules[__name__]) + (node_generations.prepare if args.generation_action == "prepare" else node_generations.collect)(args, SELF) return if os.geteuid() != 0: raise InstallError("Node installation and removal require root. Run this command with sudo.") diff --git a/deploy/node/node_output.py b/deploy/node/node_output.py index aed6a4ee4..032725be6 100644 --- a/deploy/node/node_output.py +++ b/deploy/node/node_output.py @@ -5,11 +5,11 @@ from install_display import color, heading, paragraph -def summary(root, args, unit, account): +def summary(root, args, unit, account, provider): print("\n" + color("Node installation complete.", "32")) heading("Status") print(" Core connection: connected") - print(" Sandbox Provider: " + {"docker": "Docker", "microsandbox": "microsandbox"}[args.provider] + " (ready)") + print(" Sandbox Provider: " + provider + " (ready)") heading("Node") print(" Core: " + args.core_url) print(" Runs as: " + account) diff --git a/deploy/node/node_spec.py b/deploy/node/node_spec.py index dfb9ef4ce..809f020fb 100644 --- a/deploy/node/node_spec.py +++ b/deploy/node/node_spec.py @@ -77,7 +77,7 @@ def validate(data, args): raise SpecificationError(PUBLIC_URL_CHANGED) try: provider, spec = data["provider"], data["specification"] - if (provider not in ("docker", "microsandbox") or data["installation_id"] != args.installation_id + if (_CONTRACT["providers"].get(provider, {}).get("mode") != "nodes" or data["installation_id"] != args.installation_id or data["core_url"] != args.core_url or type(data["generation"]) is not int or data["generation"] < 1 or getattr(args, "provider", None) not in (None, provider) or set(spec) - {"workspace"} != {"resources"}): @@ -148,12 +148,10 @@ def fetch(args, token, retained, open_request, allow_enrollment=False, generatio return data -def verify_provider(stored, configuration, runtime_image): +def verify_provider(stored, configuration): spec, provider = configuration["specification"], configuration["provider"] if (stored.get("specification") != spec or stored.get("provider") != provider or stored.get("installation_id") != configuration["installation_id"] or stored.get("generation") != configuration["generation"] or stored.get("core_url") != configuration["core_url"] + "/api/v1"): raise SpecificationError("Retained node configuration differs from Core; preserve its state") - if provider == "docker" and stored.get("native", {}).get("image") != runtime_image: - raise SpecificationError("Retained Docker image differs; preserve the node and inspect its configuration") diff --git a/deploy/node/test_node_helper_transfer.py b/deploy/node/test_node_helper_transfer.py index 920a89067..5e0193831 100644 --- a/deploy/node/test_node_helper_transfer.py +++ b/deploy/node/test_node_helper_transfer.py @@ -31,7 +31,7 @@ def test_captured_archive_survives_removal_of_private_source(self): self.assertEqual(helper.read_bytes(), captured) self.assertEqual(stat.S_IMODE(helper.stat().st_mode), 0o600) with zipfile.ZipFile(helper) as archive: - self.assertEqual(set(archive.namelist()), {"__main__.py", "node_spec.py", "distribution.py", "node_generations.py", "install_display.py", "node_output.py", "provider_assets.py"}) + self.assertEqual(set(archive.namelist()), {"__main__.py", "node_spec.py", "distribution.py", "node_generations.py", "install_display.py", "node_output.py", "provider_assets.py", "host_docker.py", "host_microsandbox.py"}) @unittest.skipUnless(hasattr(os, "fork") and os.geteuid() == 0, "requires a disposable Linux root test environment") def test_real_service_uid_cannot_read_private_download_but_can_install_snapshot(self): diff --git a/deploy/node/test_node_install.py b/deploy/node/test_node_install.py index 36e30ff36..0c026946f 100644 --- a/deploy/node/test_node_install.py +++ b/deploy/node/test_node_install.py @@ -23,6 +23,8 @@ from types import SimpleNamespace from unittest import mock +import host_docker +import host_microsandbox import node_install as installer import node_spec @@ -85,7 +87,7 @@ def setUp(self): mock.patch.object(installer, "wait_ready"), mock.patch.object(installer, "open_request", side_effect=self.configuration_response), mock.patch.object(installer.distribution.urllib.request, "build_opener", return_value=mock.Mock(open=self.artifact_response)), - mock.patch.object(installer, "micro_home", return_value=self.home / "m"), + mock.patch.object(host_microsandbox, "home", return_value=self.home / "m"), mock.patch.object(installer, "fetch", side_effect=lambda source, name: io.BytesIO(self.payloads[name])), mock.patch.object(installer, "checked", side_effect=self.checked), mock.patch.object(installer.distribution, "docker_command", side_effect=self.docker_command), @@ -122,8 +124,8 @@ def refresh_manifest(self): self.manifest["artifacts"] = {name: {"filename": name.replace("/", "-") + "-" + self.manifest["source_commit"], "size": len(raw), "sha256": hashlib.sha256(raw).hexdigest()} for name, raw in self.payloads.items() if name.startswith("native/") or name == "images/runtime.tar.gz"} - self.manifest["microsandbox"] = {"runtime_sha256": hashlib.sha256(self.payloads[installer.MICRO[1]]).hexdigest(), - "firmware_sha256": hashlib.sha256(self.payloads[installer.MICRO[2]]).hexdigest()} + self.manifest["microsandbox"] = {"runtime_sha256": hashlib.sha256(self.payloads[host_microsandbox.ARTIFACTS[1]]).hexdigest(), + "firmware_sha256": hashlib.sha256(self.payloads[host_microsandbox.ARTIFACTS[2]]).hexdigest()} self.manifest["artifacts"]["images/runtime.tar.gz"].update(unpacked_sha256=hashlib.sha256(b"runtime archive").hexdigest(), unpacked_size=len(b"runtime archive")) self.payloads["manifest.json"] = json.dumps(self.manifest).encode() self.payloads["SHA256SUMS"] = "".join(hashlib.sha256(raw).hexdigest() + " " + name + "\n" @@ -188,7 +190,7 @@ def test_docker_installs_matched_payload_registers_and_starts_persistent_service self.assertEqual(config["native"]["image"], self.manifest["images"]["runtime"]) self.assertEqual(config["core_url"], self.args.core_url + "/api/v1") self.assertEqual(config["installation_id"], self.args.installation_id) - self.assertFalse((self.root / installer.MICRO[0]).exists()) + self.assertFalse((self.root / host_microsandbox.ARTIFACTS[0]).exists()) unit = (system / "units" / ("oac-node-" + self.args.installation_id + ".service")).read_text() self.assertIn(" run --config ", unit) self.assertIn("KillMode=process", unit) @@ -415,7 +417,7 @@ def test_warm_image_skips_archive_download_and_import_before_registration(self): self.args.provider = provider self.image_present = True with mock.patch.object(installer.distribution, "runtime_archive") as archive: - installer.prepare_runtime(self.root, self.args, self.manifest) + installer.HOSTS[provider].prepare_runtime(installer, self.root, self.args, self.manifest) archive.assert_not_called() self.assertFalse(any("load" in call for call, _ in self.calls)) @@ -466,11 +468,11 @@ def run_as(account, function, *arguments): self.service_steps.append("wait_ready" if function is installer.wait_ready else function.__name__) with mock.patch.object(installer.Path, "home", return_value=Path(account.pw_dir)): function(*arguments) - for patch in (mock.patch.multiple(installer, SERVICE_HOME=service_home, SERVICE_UID=os.getuid(), SYSTEM_RECORDS=system / "etc", + for patch in (mock.patch.multiple(host_docker, CHILD_CONFIG=system / "docker-config", SOCKET=system / "docker.sock"), + mock.patch.multiple(installer, SERVICE_HOME=service_home, SERVICE_UID=os.getuid(), SYSTEM_RECORDS=system / "etc", SYSTEM_UNITS=system / "units", SYSTEM_LOCKS=system / "run", - CHILD_DOCKER_CONFIG=system / "docker-config", SYSTEMD_RUNNING=system / "systemd", SELINUX_ENFORCE=system / "selinux", - DOCKER_SOCKET=system / "docker.sock", run_as=run_as, + run_as=run_as, service_account=lambda: self.account), mock.patch.object(installer.shutil, "which", side_effect=lambda tool: None if tool == "docker" and not self.docker_installed else "/usr/bin/" + tool), mock.patch.object(installer.grp, "getgrnam", side_effect=lambda name: SimpleNamespace(gr_mem=["oac-node"] if self.joined else [])), @@ -495,7 +497,7 @@ def test_sudo_summary_commands_retain_permissions_of_invoking_account(self): with self.subTest(uid=uid), mock.patch.dict(os.environ, {"SUDO_UID": uid}): stream = io.StringIO() with contextlib.redirect_stdout(stream): - installer.node_output.summary(self.root, self.args, "oac-node-example.service", "oac-node") + installer.node_output.summary(self.root, self.args, "oac-node-example.service", "oac-node", "Docker") self.assertIn(" Status: " + prefix + "systemctl status oac-node-example.service", stream.getvalue()) self.assertIn(" Logs: " + prefix + "journalctl -u oac-node-example.service -f", stream.getvalue()) @@ -826,7 +828,7 @@ def probe(): os.readlink("/proc/self/fd/0"), os.environ["DOCKER_CONFIG"], tty)) output = io.StringIO() self.service_step(probe, output) - self.assertIn("session-leader=True stdin=/dev/null docker-config=" + str(installer.CHILD_DOCKER_CONFIG) + " tty=unavailable", + self.assertIn("session-leader=True stdin=/dev/null docker-config=" + str(host_docker.CHILD_CONFIG) + " tty=unavailable", output.getvalue()) def test_service_user_output_reaches_the_terminal_as_plain_text(self): @@ -1083,35 +1085,35 @@ def test_preflight_rejects_missing_kvm_before_downloads(self): def test_microsandbox_short_home_is_stable_and_rejects_long_user_home(self): with mock.patch.object(installer.Path, "home", return_value=Path("/home/node")): - first = installer.micro_home("94be54a1-138c-4f30-bc87-b13686272dbe") - self.assertEqual(first, installer.micro_home("94be54a1-138c-4f30-bc87-b13686272dbe")) + first = host_microsandbox.home(installer, "94be54a1-138c-4f30-bc87-b13686272dbe") + self.assertEqual(first, host_microsandbox.home(installer, "94be54a1-138c-4f30-bc87-b13686272dbe")) self.assertLessEqual(len(os.fsencode(first)), 48) for length in (28, 29): with mock.patch.object(installer.Path, "home", return_value=Path("/" + "h" * (length - 1))): if length == 28: - self.assertEqual(len(os.fsencode(installer.micro_home("94be54a1-138c-4f30-bc87-b13686272dbe"))), 48) + self.assertEqual(len(os.fsencode(host_microsandbox.home(installer, "94be54a1-138c-4f30-bc87-b13686272dbe"))), 48) else: with self.assertRaisesRegex(installer.InstallError, "HOME is too long"): - installer.micro_home("94be54a1-138c-4f30-bc87-b13686272dbe") + host_microsandbox.home(installer, "94be54a1-138c-4f30-bc87-b13686272dbe") with mock.patch.object(installer.Path, "home", return_value=Path("/home/" + "long" * 20)): with self.assertRaisesRegex(installer.InstallError, "HOME is too long"): - installer.micro_home("94be54a1-138c-4f30-bc87-b13686272dbe") + host_microsandbox.home(installer, "94be54a1-138c-4f30-bc87-b13686272dbe") class CheckpointRootTests(unittest.TestCase): def test_identity_is_private_stable_and_not_recreated(self): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) / "checkpoints" - installer.prepare_checkpoint_root(root) + host_microsandbox.prepare_checkpoint_root(installer, root) marker = root / ".oac-checkpoint-store" first = marker.read_bytes() - installer.prepare_checkpoint_root(root) + host_microsandbox.prepare_checkpoint_root(installer, root) self.assertEqual(first, marker.read_bytes()) self.assertEqual(stat.S_IMODE(root.stat().st_mode), 0o700) self.assertEqual(stat.S_IMODE(marker.stat().st_mode), 0o600) marker.write_text("incomplete") with self.assertRaises(installer.InstallError): - installer.prepare_checkpoint_root(root) + host_microsandbox.prepare_checkpoint_root(installer, root) def test_foreign_contents_and_symlink_are_not_adopted(self): with tempfile.TemporaryDirectory() as temporary: @@ -1119,12 +1121,12 @@ def test_foreign_contents_and_symlink_are_not_adopted(self): root.mkdir() (root / "unowned").write_bytes(b"data") with self.assertRaisesRegex(installer.InstallError, "unowned"): - installer.prepare_checkpoint_root(root) + host_microsandbox.prepare_checkpoint_root(installer, root) (root / "unowned").unlink() marker = root / ".oac-checkpoint-store" marker.symlink_to(Path(temporary) / "missing") with self.assertRaises(installer.InstallError): - installer.prepare_checkpoint_root(root) + host_microsandbox.prepare_checkpoint_root(installer, root) if __name__ == "__main__": diff --git a/scripts/build-core-distribution.sh b/scripts/build-core-distribution.sh index a2a3c5249..50a920b7a 100755 --- a/scripts/build-core-distribution.sh +++ b/scripts/build-core-distribution.sh @@ -104,7 +104,7 @@ if [[ "$(go env GOVERSION)" != "$required_go" ]]; then exit 1 fi go run ./services/core/cmd/provider-artifacts -for file in install_display.py node_output.py node_install.py provider_assets.py node_spec.py node_generations.py node_payload.py distribution.py; do +for file in install_display.py node_output.py node_install.py host_docker.py host_microsandbox.py provider_assets.py node_spec.py node_generations.py node_payload.py distribution.py; do cp "deploy/node/$file" "$bundle/$file" done python3 scripts/core-distribution-manifest.py bootstraps "$bundle" "$source_epoch" "$revision" diff --git a/scripts/core-distribution-manifest.py b/scripts/core-distribution-manifest.py index fa9ad6de0..05b985336 100644 --- a/scripts/core-distribution-manifest.py +++ b/scripts/core-distribution-manifest.py @@ -325,7 +325,7 @@ def bootstraps(bundle, epoch, revision): bundle = pathlib.Path(bundle) with tempfile.TemporaryDirectory(dir=bundle.parent) as directory: modules = (("node_install.py", "__main__.py"), ("distribution.py", "distribution.py"), - *((name, name) for name in ("node_spec.py", "node_generations.py", "install_display.py", "node_output.py", "provider_assets.py"))) + *((name, name) for name in ("node_spec.py", "node_generations.py", "install_display.py", "node_output.py", "provider_assets.py", "host_docker.py", "host_microsandbox.py"))) for original, packaged in modules: target = pathlib.Path(directory) / packaged shutil.copyfile(bundle / original, target) diff --git a/scripts/core-distribution-manifest.test.py b/scripts/core-distribution-manifest.test.py index c2f3cf679..b488e9530 100644 --- a/scripts/core-distribution-manifest.test.py +++ b/scripts/core-distribution-manifest.test.py @@ -312,7 +312,7 @@ def test_release_location_and_explicit_offline_requirements(self): self.assertEqual(json.loads((self.bundle / "manifest.json").read_text())["artifact_base_url"], "") def test_bootstraps_include_shared_downloader_and_are_reproducible(self): - for name in ("node_install.py", "node_generations.py", "install_display.py", "node_output.py", "provider_assets.py", "distribution.py", "node_spec.py"): + for name in ("node_install.py", "node_generations.py", "install_display.py", "node_output.py", "provider_assets.py", "host_docker.py", "host_microsandbox.py", "distribution.py", "node_spec.py"): (self.bundle / name).write_text("# " + name + "\n") distribution.bootstraps(self.bundle, "1700000000", "a" * 40) first = (self.bundle / "node-install.pyz").read_bytes() @@ -320,7 +320,7 @@ def test_bootstraps_include_shared_downloader_and_are_reproducible(self): self.assertEqual(first, (self.bundle / "node-install.pyz").read_bytes()) self.assertFalse((self.bundle / "oac.pyz").exists()) with zipfile.ZipFile(self.bundle / "node-install.pyz") as contents: - self.assertEqual(set(contents.namelist()), {"__main__.py", "node_spec.py", "distribution.py", "node_generations.py", "install_display.py", "node_output.py", "provider_assets.py"}) + self.assertEqual(set(contents.namelist()), {"__main__.py", "node_spec.py", "distribution.py", "node_generations.py", "install_display.py", "node_output.py", "provider_assets.py", "host_docker.py", "host_microsandbox.py"}) self.assertEqual(contents.read("__main__.py"), (self.bundle / "node_install.py").read_bytes())