From 72569c84bcd669304841db47717c6ad73553a3d4 Mon Sep 17 00:00:00 2001 From: Zijian Zhang <35801754+futrime@users.noreply.github.com> Date: Mon, 12 Oct 2026 22:56:35 +0800 Subject: [PATCH] feat(web): daemon registry API under /api/daemons Keep the ssh daemons a page may switch between in settings.yaml under `daemons`, checked as an ssh runtime is: Local first and implicit, then each one added by hand or imported from the ssh config. Each carries how it stands from `site.remotes` where the site has one. Co-Authored-By: Claude Opus 5.5 --- docs/reference/web.md | 5 + src/hmz/web/daemons.py | 294 ++++++++++++++++++++++++++ src/hmz/web/site.py | 2 + tests/integration/test_web_daemons.py | 124 +++++++++++ tests/unit/web/test_daemons.py | 219 +++++++++++++++++++ 5 files changed, 644 insertions(+) create mode 100644 src/hmz/web/daemons.py create mode 100644 tests/integration/test_web_daemons.py create mode 100644 tests/unit/web/test_daemons.py diff --git a/docs/reference/web.md b/docs/reference/web.md index e4311393..d5a8e61e 100644 --- a/docs/reference/web.md +++ b/docs/reference/web.md @@ -274,6 +274,11 @@ Every `error` but the last is the runtime's own sentence, e.g. | `POST /api/runtimes/check` | `backend`, `name` | What it said: `reached`, `said`, `home`, `cpus`, `memory`, `gpus`, `usable`, `gpu_memory`, `runtimes`, `version`, `short`, `nodes` | | `GET /api/runtimes/hosts` | | `hosts` your ssh config names (`alias`, `host`, `user`, `port`, `identity_files`, `proxy_jump`, `saved`) | | `POST /api/runtimes/import` | `names`; `update` (bool) | As `GET /api/runtimes` | +| `GET /api/daemons` | | `daemons`: `Local` first, then each ssh daemon (`name`, `label`, `host`, `user`, `port`, `identity_file`, `proxy_jump`, `options` as `[keyword, value]` pairs, `alias`, `config`) with `state`, `detail` and `version` | +| `POST /api/daemons` | `name`, `fields`; `replace` (bool) | As `GET` | +| `POST /api/daemons/remove` | `name` | As `GET` | +| `GET /api/daemons/hosts` | | `hosts` your ssh config names that are not daemons yet (`alias`, `name`, `host`, `user`, `port`, `identity_files`, `proxy_jump`) | +| `POST /api/daemons/import` | `aliases` | As `GET`, and `added`: the daemons made, skipping any already there | An account is listed by the names of the variables it sets (`sets`), never by a value. diff --git a/src/hmz/web/daemons.py b/src/hmz/web/daemons.py new file mode 100644 index 00000000..5d867160 --- /dev/null +++ b/src/hmz/web/daemons.py @@ -0,0 +1,294 @@ +"""The daemons a page may switch between: this machine's, and each ssh host written down. + +`Local` is always there and never written down. Every other daemon is an ssh host, kept in +humanize's one settings file under `daemons` with what `ssh` is told to reach it -- the same +fields an ssh runtime has, checked as one is, but for its workdir and fallback -- typed in a +part at a time or brought in from the ssh config. How each stands is the remote reach's to +say, where this site has one; without one, Local is up and every other daemon is idle. +""" + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING, Any, cast + +from .routing import Refusal, routes + +if TYPE_CHECKING: + from collections.abc import Callable + + from .routing import Asked + +__all__ = ["LOCAL", "ROUTES", "found", "listed", "saved"] + +#: The daemon of this machine, which is always there and never written down. +LOCAL = "local" + +#: The key of the settings file the daemons are kept under. +_KEY = "daemons" + +#: What a daemon's name is spelled with: never starting with a dot, so never `.` or `..`. +_NAMED = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*") + +#: What a daemon holds beside its name, as `ssh` is told it. +_FIELDS = ( + "host", + "user", + "port", + "identity_file", + "proxy_jump", + "options", + "alias", + "config", +) + +#: What a write of one takes. +_TAKES = frozenset({"name", "fields", "replace"}) + + +def _checked(name: str, fields: dict[str, Any]) -> dict[str, Any]: + """One daemon as it is written down, checked as an ssh runtime is. + + Args: + name: What it is called. + fields: The rest of it, `options` as `[[KEYWORD, VALUE], ...]` or by keyword. + + Returns: + It, `options` as pairs. + + Raises: + ValueError: If it cannot be one. + """ + from hmz.coganchor.machines.store import SSH, SSHRuntime, new + + if not _NAMED.fullmatch(name) or name.lower() == LOCAL: + raise ValueError( + f"A daemon cannot be called {name!r}: say a letter or digit, then letters, " + f"digits, dots, dashes and underscores, and not {LOCAL}." + ) + if extra := sorted(set(fields) - set(_FIELDS)): + raise ValueError(f"A daemon takes no {', '.join(extra)}.") + given = dict(fields) + options: object = given.get("options", []) + if isinstance(options, list): + pairs = cast("list[object]", options) + if not all( + isinstance(one, list) and len(cast("list[object]", one)) == 2 # noqa: PLR2004 + for one in pairs + ): + raise ValueError("Say options as pairs of a keyword and its value.") + given["options"] = { + str(key): str(value) for key, value in cast("list[list[object]]", pairs) + } + one = cast("SSHRuntime", new(SSH, name, **given)) + return { + "name": one.name, + **{field: getattr(one, field) for field in _FIELDS if field != "options"}, + "options": [[key, value] for key, value in one.options.items()], + } + + +def _valid(held: dict[str, Any]) -> list[dict[str, Any]]: + """Every ssh daemon a reading of the settings file holds, in order, each checked. + + One that cannot be one -- written wrong by hand, or under a name taken before it -- is + left out, and so is gone the next time the daemons are written. + """ + there: object = held.get(_KEY) + kept: list[dict[str, Any]] = [] + for one in cast("list[object]", there) if isinstance(there, list) else []: + fields = dict(cast("dict[str, Any]", one)) if isinstance(one, dict) else {} + name = fields.pop("name", None) + if not isinstance(name, str): + continue + try: + entry = _checked(name, fields) + except ValueError: + continue + if all(entry["name"] != each["name"] for each in kept): + kept.append(entry) + return kept + + +def saved() -> list[dict[str, Any]]: + """Every ssh daemon written down, in order; one that cannot be one is left out.""" + from hmz.coganchor import settings + + return _valid(settings.read()) + + +def found(name: str) -> dict[str, Any] | None: + """The ssh daemon written down under a name, or None where there is none.""" + return next((one for one in saved() if one["name"] == name), None) + + +def listed(site: Any) -> dict[str, Any]: + """Local, then every ssh daemon, each with how it stands.""" + from hmz.coganchor import __version__ + + remotes = getattr(site, "remotes", None) + local: dict[str, Any] = { + "name": LOCAL, + **dict.fromkeys(_FIELDS, ""), + "port": 0, + "options": [], + "label": "Local", + "state": "up", + "detail": "", + "version": __version__, + } + daemons = [local] + for one in saved(): + stands = ( + dict(remotes.state(one["name"])) + if remotes is not None + else {"state": "idle", "detail": "", "version": ""} + ) + daemons.append(one | {"label": one["name"]} | stands) + return {"daemons": daemons} + + +def _changes(change: Callable[[list[dict[str, Any]]], None]) -> None: + """Makes one change to the daemons written down, and to nothing else in the file. + + Raises: + Refusal: If the file cannot be written. + """ + from hmz.coganchor import settings + + def changing(held: dict[str, Any]) -> None: + daemons = _valid(held) + change(daemons) + held[_KEY] = daemons + + try: + settings.changes(changing) + except OSError as why: + raise Refusal(f"The daemons could not be written: {why}", 500) from why + + +def _write(asked: Asked) -> dict[str, Any]: + if extra := sorted(set(asked.body) - _TAKES): + raise Refusal(f"A daemon is written with no {', '.join(extra)}.") + name = asked.text("name") + replacing = asked.body.get("replace", False) + if not isinstance(replacing, bool): + raise Refusal("Say replace as true or false.") + try: + entry = _checked(name, asked.named("fields")) + except ValueError as why: + raise Refusal(str(why)) from why + + def change(daemons: list[dict[str, Any]]) -> None: + at = next((i for i, one in enumerate(daemons) if one.get("name") == name), -1) + if at < 0: + daemons.append(entry) + elif replacing: + daemons[at] = entry + else: + raise Refusal(f"There is a daemon called {name} already.", 409) + + _changes(change) + return listed(asked.site) + + +def _remove(asked: Asked) -> dict[str, Any]: + if extra := sorted(set(asked.body) - {"name"}): + raise Refusal(f"A daemon is removed with no {', '.join(extra)}.") + name = asked.text("name") + if name.lower() == LOCAL: + raise Refusal("Local is always there, and cannot be removed.") + + def change(daemons: list[dict[str, Any]]) -> None: + kept = [one for one in daemons if one.get("name") != name] + if len(kept) == len(daemons): + raise Refusal(f"There is no daemon called {name}.", 404) + daemons[:] = kept + + _changes(change) + return listed(asked.site) + + +def _named(alias: str) -> str: + """What a host of the ssh config is called as a daemon: anything no name holds a dash.""" + return re.sub(r"[^A-Za-z0-9._-]", "-", alias).lstrip("._-") + + +def _offered(aliases: list[str]) -> dict[str, dict[str, Any]]: + """Each alias that can be brought in as a daemon not written down yet, as it would be. + + Of two aliases that would be called the same, the first is offered. + """ + daemons = saved() + names = {one["name"] for one in daemons} + taken = {one["alias"] for one in daemons} + offered: dict[str, dict[str, Any]] = {} + for alias in aliases: + if alias in taken: + continue + try: + one = _checked(_named(alias), {"alias": alias}) + except ValueError: + continue + if one["name"] not in names: + names.add(one["name"]) + offered[alias] = one + return offered + + +def _hosts(_asked: Asked) -> dict[str, Any]: + import dataclasses + + from hmz.coganchor.machines import sshconfig + + try: + return { + "hosts": [ + dataclasses.asdict(sshconfig.resolve(alias)) | {"name": one["name"]} + for alias, one in _offered(sshconfig.aliases()).items() + ] + } + except OSError as why: + raise Refusal(f"The ssh config could not be read: {why}", 503) from why + + +def _imports(asked: Asked) -> dict[str, Any]: + from hmz.coganchor.machines import sshconfig + + if extra := sorted(set(asked.body) - {"aliases"}): + raise Refusal(f"An import takes no {', '.join(extra)}.") + wanted = asked.texts("aliases") + if not wanted: + raise Refusal("Say aliases as the hosts to bring in, by their Host.") + there = sshconfig.aliases() + if missing := [one for one in wanted if one not in there]: + raise Refusal(f"Your ssh config has no host {', '.join(missing)}.", 404) + for alias in wanted: + try: + _checked(_named(alias), {"alias": alias}) + except ValueError as why: + raise Refusal(f"{alias} cannot be a daemon: {why}") from why + added: list[str] = [] + + def change(daemons: list[dict[str, Any]]) -> None: + names = {one["name"] for one in daemons} + taken = {one["alias"] for one in daemons} + for alias in dict.fromkeys(wanted): + one = _checked(_named(alias), {"alias": alias}) + if alias not in taken and one["name"] not in names: + daemons.append(one) + names.add(one["name"]) + added.append(one["name"]) + + _changes(change) + return listed(asked.site) | {"added": added} + + +#: What a page reads of the daemons, and does to them. +ROUTES = routes( + ("GET", "/api/daemons", lambda asked: listed(asked.site)), + ("POST", "/api/daemons", _write), + ("POST", "/api/daemons/remove", _remove), + ("GET", "/api/daemons/hosts", _hosts), + ("POST", "/api/daemons/import", _imports), +) diff --git a/src/hmz/web/site.py b/src/hmz/web/site.py index e6204457..4aad7b1a 100644 --- a/src/hmz/web/site.py +++ b/src/hmz/web/site.py @@ -39,6 +39,7 @@ from . import ( accounts, + daemons, fallbacks, flows, held, @@ -68,6 +69,7 @@ *accounts.ROUTES, *fallbacks.ROUTES, *runtimes.ROUTES, + *daemons.ROUTES, *paths.ROUTES, *places.ROUTES, *verses.ROUTES, diff --git a/tests/integration/test_web_daemons.py b/tests/integration/test_web_daemons.py new file mode 100644 index 00000000..e71b55ce --- /dev/null +++ b/tests/integration/test_web_daemons.py @@ -0,0 +1,124 @@ +"""`hmz web`'s daemons: written down, brought in from the ssh config, and removed.""" + +from __future__ import annotations + +import shutil +from pathlib import Path +from typing import Any + +import pytest +import yaml + +from hmz import home +from tests.integration.doubles_web import Browser, served + +pytestmark = pytest.mark.skipif(shutil.which("ssh") is None, reason="needs ssh -G") + + +@pytest.fixture +def ssh_config() -> Path: + """An ssh config, in the test's own home, naming two hosts.""" + at = Path.home() / ".ssh" / "config" + at.parent.mkdir(parents=True, exist_ok=True) + at.write_text( + "Host gpu\n HostName gpu.example\n User me\n Port 2222\n" + "Host box\n HostName box.example\n" + ) + return at + + +def _settings() -> dict[str, Any]: + return yaml.safe_load((home() / "settings.yaml").read_text()) + + +def test_a_daemon_is_added_edited_and_removed(ssh_config: Path) -> None: + with served() as site: + browser = Browser(site) + browser.signs_in() + + assert [ + one["label"] for one in browser.get("/api/daemons").json()["daemons"] + ] == ["Local"] + added = browser.post( + "/api/daemons", + { + "name": "lab", + "fields": {"host": "lab.example", "options": [["Compression", "yes"]]}, + }, + ) + assert added.status == 200, added.body + assert [(one["name"], one["state"]) for one in added.json()["daemons"]] == [ + ("local", "up"), + ("lab", "idle"), + ] + assert ( + browser.post( + "/api/daemons", {"name": "lab", "fields": {"host": "b"}} + ).status + == 409 + ) + assert ( + browser.post( + "/api/daemons", + {"name": "lab", "fields": {"host": "b"}, "replace": True}, + ).status + == 200 + ) + refused = browser.post( + "/api/daemons", {"name": "local", "fields": {"host": "b"}} + ) + assert refused.status == 400 + assert refused.json()["error"] + assert browser.post("/api/daemons", {"name": "x", "nope": 1}).status == 400 + + assert _settings()["daemons"] == [ + { + "name": "lab", + "host": "b", + "user": "", + "port": 0, + "identity_file": "", + "proxy_jump": "", + "alias": "", + "config": "", + "options": [], + } + ] + + removed = browser.post("/api/daemons/remove", {"name": "lab"}) + assert [one["name"] for one in removed.json()["daemons"]] == ["local"] + assert browser.post("/api/daemons/remove", {"name": "local"}).status == 400 + + +def test_hosts_of_the_ssh_config_are_offered_and_imported(ssh_config: Path) -> None: + with served() as site: + browser = Browser(site) + browser.signs_in() + + hosts = browser.get("/api/daemons/hosts").json()["hosts"] + # Which hosts are named is read off the config in this home; what each comes to + # is `ssh -G`'s, which reads the config of the account's own home instead. + assert [(one["alias"], one["name"]) for one in hosts] == [ + ("gpu", "gpu"), + ("box", "box"), + ] + imported = browser.post("/api/daemons/import", {"aliases": ["gpu"]}) + assert imported.json()["added"] == ["gpu"] + again = browser.post("/api/daemons/import", {"aliases": ["gpu", "box"]}) + assert again.json()["added"] == ["box"] + assert [one["name"] for one in again.json()["daemons"]] == [ + "local", + "gpu", + "box", + ] + assert browser.get("/api/daemons/hosts").json()["hosts"] == [] + assert ( + browser.post("/api/daemons/import", {"aliases": ["elsewhere"]}).status + == 404 + ) + assert [one["alias"] for one in _settings()["daemons"]] == ["gpu", "box"] + + +def test_a_stranger_is_told_nothing_of_the_daemons(ssh_config: Path) -> None: + with served() as site: + assert Browser(site).get("/api/daemons").status == 401 diff --git a/tests/unit/web/test_daemons.py b/tests/unit/web/test_daemons.py new file mode 100644 index 00000000..5ae9634c --- /dev/null +++ b/tests/unit/web/test_daemons.py @@ -0,0 +1,219 @@ +"""`hmz.web.daemons`: the daemons a page may switch between, written down and brought in.""" + +from __future__ import annotations + +import copy +from types import SimpleNamespace +from typing import TYPE_CHECKING, Any + +import pytest + +from hmz.web.daemons import LOCAL, ROUTES, found, listed, saved +from hmz.web.routing import Asked, Refusal + +if TYPE_CHECKING: + from collections.abc import Callable + + +@pytest.fixture +def held(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: + """The settings file, as a dict that `read` and `changes` read and write.""" + from hmz.coganchor import settings + + file: dict[str, Any] = {"other": {"kept": True}} + + def changes(change: Callable[[dict[str, Any]], None]) -> dict[str, Any]: + working = copy.deepcopy(file) + change(working) + file.clear() + file.update(working) + return file + + monkeypatch.setattr(settings, "read", lambda: copy.deepcopy(file)) + monkeypatch.setattr(settings, "changes", changes) + return file + + +@pytest.fixture +def config(monkeypatch: pytest.MonkeyPatch) -> list[str]: + """The hosts the ssh config names, each resolving to `.example`.""" + from hmz.coganchor.machines import sshconfig + + named = ["gpu", "box.one", "local", "a+b", "a-b", "__", "x$y"] + + def resolve(alias: str, *_: object, **__: object) -> sshconfig.SSHHost: + return sshconfig.SSHHost( + alias=alias, host=f"{alias}.example", user="me", port=22 + ) + + def aliases(*_: object) -> list[str]: + return list(named) + + monkeypatch.setattr(sshconfig, "aliases", aliases) + monkeypatch.setattr(sshconfig, "resolve", resolve) + return named + + +def _asks(method: str, path: str, body: dict[str, Any] | None = None) -> Any: + route = next(one for one in ROUTES if one.match(method, path) is not None) + return route.answers(Asked(site=SimpleNamespace(), body=body or {})) + + +def test_local_is_first_and_up_with_nothing_written_down(held: dict[str, Any]) -> None: + daemons = listed(SimpleNamespace())["daemons"] + + assert [(one["name"], one["label"], one["state"]) for one in daemons] == [ + (LOCAL, "Local", "up") + ] + + +def test_a_daemon_is_written_with_its_options_as_pairs(held: dict[str, Any]) -> None: + said = _asks( + "POST", + "/api/daemons", + { + "name": "gpu", + "fields": { + "host": "gpu.example", + "port": 2222, + "options": [["Compression", "yes"]], + }, + }, + ) + + assert [one["name"] for one in said["daemons"]] == [LOCAL, "gpu"] + assert said["daemons"][1]["state"] == "idle" + assert held["other"] == {"kept": True} + assert held["daemons"] == [ + { + "name": "gpu", + "host": "gpu.example", + "user": "", + "port": 2222, + "identity_file": "", + "proxy_jump": "", + "alias": "", + "config": "", + "options": [["Compression", "yes"]], + } + ] + assert found("gpu") == held["daemons"][0] + assert found("nope") is None + + +def test_one_there_already_is_replaced_only_when_asked(held: dict[str, Any]) -> None: + _asks("POST", "/api/daemons", {"name": "gpu", "fields": {"host": "a"}}) + + with pytest.raises(Refusal) as refused: + _asks("POST", "/api/daemons", {"name": "gpu", "fields": {"host": "b"}}) + assert refused.value.status == 409 + _asks( + "POST", + "/api/daemons", + {"name": "gpu", "fields": {"host": "b"}, "replace": True}, + ) + + assert [one["host"] for one in saved()] == ["b"] + + +@pytest.mark.parametrize( + "body", + [ + {"name": "local", "fields": {"host": "a"}}, + {"name": "Local", "fields": {"host": "a"}}, + {"name": "a b", "fields": {"host": "a"}}, + {"name": "..", "fields": {"host": "a"}}, + {"name": "gpu", "fields": {}}, + {"name": "gpu", "fields": {"host": "a", "workdir": "/w"}}, + {"name": "gpu", "fields": {"host": "a"}, "secret": "x"}, + {"name": "gpu", "fields": {"host": "a", "port": "22"}}, + {"name": "gpu", "fields": {"host": "a", "options": [["Port", "22"]]}}, + ], +) +def test_what_cannot_be_a_daemon_is_refused( + held: dict[str, Any], body: dict[str, Any] +) -> None: + with pytest.raises(Refusal): + _asks("POST", "/api/daemons", body) + + assert "daemons" not in held + + +def test_a_daemon_is_removed_and_local_never_is(held: dict[str, Any]) -> None: + _asks("POST", "/api/daemons", {"name": "gpu", "fields": {"host": "a"}}) + + with pytest.raises(Refusal): + _asks("POST", "/api/daemons/remove", {"name": "local"}) + with pytest.raises(Refusal) as missing: + _asks("POST", "/api/daemons/remove", {"name": "nope"}) + assert missing.value.status == 404 + said = _asks("POST", "/api/daemons/remove", {"name": "gpu"}) + + assert [one["name"] for one in said["daemons"]] == [LOCAL] + assert held["daemons"] == [] + + +def test_one_written_wrong_by_hand_is_left_out(held: dict[str, Any]) -> None: + held["daemons"] = [ + "nonsense", + {"name": "local", "host": "a"}, + {"name": "ok", "host": "a"}, + {"name": "ok", "host": "b"}, + {"host": "nameless"}, + ] + + assert [(one["name"], one["host"]) for one in saved()] == [("ok", "a")] + + +def test_each_stands_as_the_remote_reach_says(held: dict[str, Any]) -> None: + held["daemons"] = [{"name": "gpu", "host": "a"}] + + def state(name: str) -> dict[str, str]: + return {"state": "failed", "detail": f"{name} down", "version": ""} + + remotes = SimpleNamespace(state=state) + + daemons = listed(SimpleNamespace(remotes=remotes))["daemons"] + + assert daemons[0]["state"] == "up" + assert (daemons[1]["state"], daemons[1]["detail"]) == ("failed", "gpu down") + + +def test_hosts_not_yet_added_are_offered( + held: dict[str, Any], config: list[str] +) -> None: + held["daemons"] = [{"name": "g", "alias": "gpu"}] + + hosts = _asks("GET", "/api/daemons/hosts")["hosts"] + + # Of two called the same, the first; never one no daemon could be. + assert [(one["alias"], one["name"], one["host"]) for one in hosts] == [ + ("box.one", "box.one", "box.one.example"), + ("a+b", "a-b", "a+b.example"), + ] + + +def test_hosts_are_imported_by_alias_skipping_any_already_there( + held: dict[str, Any], config: list[str] +) -> None: + held["daemons"] = [{"name": "gpu", "host": "a"}] + + said = _asks("POST", "/api/daemons/import", {"aliases": ["gpu", "box.one"]}) + + assert said["added"] == ["box.one"] + assert [(one["name"], one["alias"], one["host"]) for one in saved()] == [ + ("gpu", "", "a"), + ("box.one", "box.one", ""), + ] + + +@pytest.mark.parametrize( + "body", [{"aliases": []}, {"aliases": ["elsewhere"]}, {"aliases": ["local"]}] +) +def test_an_import_of_nothing_or_of_a_host_it_cannot_be_is_refused( + held: dict[str, Any], config: list[str], body: dict[str, Any] +) -> None: + with pytest.raises(Refusal): + _asks("POST", "/api/daemons/import", body) + + assert "daemons" not in held