From 4ca46cc6e60bf40eb0ffd54e623e8c523193b565 Mon Sep 17 00:00:00 2001 From: Vishnu Therayil Sasikumar Date: Mon, 25 May 2026 17:48:21 -0700 Subject: [PATCH 01/21] Updates to include gateware build --- synapse/cli/build.py | 125 +++++++-- synapse/cli/gateware.py | 194 +++++++++++++ synapse/cli/peripherals.py | 540 ++++++++++++++++++++++++++++++------- 3 files changed, 740 insertions(+), 119 deletions(-) create mode 100644 synapse/cli/gateware.py diff --git a/synapse/cli/build.py b/synapse/cli/build.py index cdda8aef..f560f122 100644 --- a/synapse/cli/build.py +++ b/synapse/cli/build.py @@ -3,6 +3,7 @@ import glob import json import os +import re import shutil import subprocess import tempfile @@ -71,42 +72,111 @@ def ensure_docker() -> bool: return False -def build_docker_image(app_dir: str, app_name: str | None = None) -> str: - """(Re)build the cross-compile SDK Docker image and return its tag.""" +_ARG_HOST_UID_RE = re.compile(r"^ARG HOST_UID\b", re.MULTILINE) + + +def _resolve_host_uid() -> int: + """Return the invoking user's UID, falling back to 1000 on non-POSIX hosts.""" + try: + return os.getuid() + except AttributeError: + console.print( + "[yellow]Warning:[/yellow] os.getuid() unavailable on this host; " + "falling back to HOST_UID=1000." + ) + return 1000 + + +def _dockerfile_needs_host_uid(dockerfile_path: str) -> bool: + """Return True iff *dockerfile_path* declares ``ARG HOST_UID`` at line start.""" + with open(dockerfile_path, "r", encoding="utf-8") as fp: + return bool(_ARG_HOST_UID_RE.search(fp.read())) + + +def build_docker_image(app_dir: str, app_name: str | None = None) -> dict[str, str]: + """(Re)build the cross-compile SDK Docker image(s) and return role -> tag. + + Discovers every ``*.Dockerfile`` directly under ``/Dockerfiles/`` + and builds each as ``-:latest-`` where ``role`` is + the filename stem (e.g. ``gateware.Dockerfile`` -> ``gateware``). Returns + a dict mapping role -> image tag. + + Back-compat: if ``/Dockerfiles/`` does not exist and + ``/Dockerfile`` does, builds the single legacy image tagged + ``:latest-`` (no role suffix) and returns + ``{"driver": ""}``. + + If ``Dockerfiles/`` exists but is empty, or if neither path exists, + raises :class:`FileNotFoundError`. + + For each Dockerfile whose contents contain a line matching + ``^ARG HOST_UID\\b`` the build additionally receives + ``--build-arg HOST_UID=``. On non-POSIX hosts (where + ``os.getuid()`` is unavailable) the value falls back to ``1000``. + """ if app_name is None: app_name = os.path.basename(app_dir) arch_suffix = detect_arch() # "arm64" or "amd64" - # Look for a Dockerfile at the top level of the app directory - dockerfile_path = os.path.join(app_dir, "Dockerfile") - - if not os.path.exists(dockerfile_path): + dockerfiles_dir = os.path.join(app_dir, "Dockerfiles") + legacy_dockerfile = os.path.join(app_dir, "Dockerfile") + + # Discovery: Dockerfiles/ wins over the legacy root ./Dockerfile. + discovered: list[tuple[str, str]] = [] + is_legacy = False + if os.path.isdir(dockerfiles_dir): + for entry in sorted(os.listdir(dockerfiles_dir)): + if entry.endswith(".Dockerfile"): + role = entry[: -len(".Dockerfile")] + discovered.append((role, os.path.join(dockerfiles_dir, entry))) + elif os.path.exists(legacy_dockerfile): + discovered.append(("driver", legacy_dockerfile)) + is_legacy = True + + if not discovered: raise FileNotFoundError( - f"Expected Dockerfile not found at {dockerfile_path}. " - "Ensure your application provides the required Dockerfile." + f"Expected Dockerfile not found at {legacy_dockerfile} or any " + f"*.Dockerfile under {dockerfiles_dir}. Ensure your application " + "provides the required Dockerfile." ) - image_tag = f"{app_name}:latest-{arch_suffix}" + tags: dict[str, str] = {} + for role, dockerfile_path in discovered: + image_tag = ( + f"{app_name}:latest-{arch_suffix}" + if is_legacy + else f"{app_name}-{role}:latest-{arch_suffix}" + ) - console.print(f"[yellow]Building Docker image [bold]{image_tag}[/bold]...[/yellow]") - subprocess.run( - [ - "docker", - "build", - "-t", - image_tag, - "-f", - dockerfile_path, - ".", - ], - check=True, - cwd=app_dir, - ) + build_args: list[str] = [] + if _dockerfile_needs_host_uid(dockerfile_path): + host_uid = _resolve_host_uid() + build_args.extend(["--build-arg", f"HOST_UID={host_uid}"]) + + console.print( + f"[yellow]Building Docker image [bold]{image_tag}[/bold]...[/yellow]" + ) + subprocess.run( + [ + "docker", + "build", + "-t", + image_tag, + "-f", + dockerfile_path, + *build_args, + ".", + ], + check=True, + cwd=app_dir, + ) - console.print(f"[green]Successfully built Docker image {image_tag}[/green]") - return image_tag + console.print(f"[green]Successfully built Docker image {image_tag}[/green]") + tags[role] = image_tag + + return tags def build_app( @@ -127,12 +197,9 @@ def build_app( console.print("[yellow]Binary not found, attempting to build...[/yellow]") - arch_suffix = detect_arch() - image_tag = f"{os.path.basename(app_dir)}:latest-{arch_suffix}" - # Build (or rebuild) the Docker image – this function is idempotent. try: - image_tag = build_docker_image(app_dir, app_name) + image_tag = build_docker_image(app_dir, app_name)["driver"] except (subprocess.CalledProcessError, FileNotFoundError) as exc: console.print( f"[bold red]Error:[/bold red] Failed to build Docker image: {exc}" diff --git a/synapse/cli/gateware.py b/synapse/cli/gateware.py new file mode 100644 index 00000000..7c4e89ef --- /dev/null +++ b/synapse/cli/gateware.py @@ -0,0 +1,194 @@ +"""Gateware build helpers for the ``synapsectl peripherals`` CLI. + +This module exposes the LM_LICENSE_FILE helper used by the gateware docker +invocation, the :func:`run_gateware_build` runner that wraps +``axon-peripheral-sdk build`` inside the gateware container, and the +:func:`_gateware_passthrough` dispatcher used by +``synapsectl peripherals gateware [args...]``. +""" + +from __future__ import annotations + +import glob +import os +import re +import subprocess +import sys +from pathlib import Path +from typing import Mapping, Sequence + +from rich.console import Console + +console = Console() + + +_NON_POSIX_MSG = ( + "synapsectl peripherals gateware requires a POSIX host (Linux or macOS): " + "os.getuid() / os.getgid() are needed to set the container's --user flag " + "so files written under the bind-mount belong to you. On Windows, invoke " + "axon-peripheral-sdk directly inside WSL or a Linux container." +) + + +# Module-level constant for floating-license detection. Matches both +# single-server (``port@host``) and colon-joined multi-server FlexLM +# redundancy strings (``port1@host1:port2@host2``). Rejects anything +# containing ``/`` so file paths fall through to the path branch even +# when they contain ``@`` (e.g. ``/home/user@work/license.dat``). +# ``\Z`` (not ``$``) anchors the end strictly — ``$`` would match before a +# trailing newline and let pathological values like ``"27000@host\n"`` slip +# through into the container as an LM_LICENSE_FILE env var. +_PORT_AT_HOST_RE = re.compile(r"\A[^/\s]+@[^/\s]+\Z") + +_LICENSE_UNSET_MSG = ( + "LM_LICENSE_FILE is not set. Set it to a license file path " + "(e.g. /etc/lattice/license.dat) or a port@host floating-license " + "spec (e.g. 7788@licenseserver)." +) + +_CONTAINER_LICENSE_PATH = "/opt/lattice/license.dat" + + +class LicenseUnsetError(RuntimeError): + """Raised when ``LM_LICENSE_FILE`` is unset or empty.""" + + +def build_license_docker_args( + env: Mapping[str, str] = os.environ, +) -> list[str]: + """Return the ``docker run`` flags that forward the Radiant license. + + Three modes: + + - **File path** (default branch): resolved with + ``Path(value).expanduser().resolve(strict=True)`` and bind-mounted + read-only into the container at ``/opt/lattice/license.dat``. + - **Floating** (``port@host`` or ``port1@host1:port2@host2``): the + value is forwarded verbatim via ``-e LM_LICENSE_FILE=`` + with no bind-mount. + - **Unset / empty**: raises :class:`LicenseUnsetError`. + + The helper reads only from ``env`` and never falls back to + ``os.environ`` when the key is missing from the supplied mapping. + """ + value = env.get("LM_LICENSE_FILE", "") + if not value: + raise LicenseUnsetError(_LICENSE_UNSET_MSG) + + if _PORT_AT_HOST_RE.match(value): + return ["-e", f"LM_LICENSE_FILE={value}"] + + resolved = Path(value).expanduser().resolve(strict=True) + return [ + "-v", + f"{resolved}:{_CONTAINER_LICENSE_PATH}:ro", + "-e", + f"LM_LICENSE_FILE={_CONTAINER_LICENSE_PATH}", + ] + + +_SDK_BUILD_CMD = ( + "axon-peripheral-sdk build --project src/gateware --pdc devkit --impl impl_1" +) + + +def run_gateware_build( + peripheral_dir: str, + image_tag: str, + env: Mapping[str, str] = os.environ, +) -> str: + """Invoke ``axon-peripheral-sdk build`` inside the gateware container. + + Returns the absolute path to the newest ``sdk_*.bit`` emitted under + ``/src/gateware/build/bitstreams/``. + + Raises: + LicenseUnsetError: if ``LM_LICENSE_FILE`` is unset (propagated from + :func:`build_license_docker_args`). + subprocess.CalledProcessError: if the container's build exits non-zero. + FileNotFoundError: if the build succeeds but no bitstream is emitted. + """ + license_args = build_license_docker_args(env) + + abs_peripheral_dir = os.path.abspath(peripheral_dir) + argv = [ + "docker", + "run", + "--rm", + "--user", + "dev", + "-v", + f"{abs_peripheral_dir}:/home/workspace", + "-w", + "/home/workspace", + *license_args, + image_tag, + "/bin/bash", + "-lc", + _SDK_BUILD_CMD, + ] + subprocess.run(argv, check=True) + + bit_glob = os.path.join( + abs_peripheral_dir, "src", "gateware", "build", "bitstreams", "sdk_*.bit" + ) + matches = glob.glob(bit_glob) + if not matches: + raise FileNotFoundError( + "axon-peripheral-sdk build completed but no sdk_*.bit was emitted " + "under src/gateware/build/bitstreams/" + ) + + matches.sort(key=os.path.getmtime, reverse=True) + chosen = matches[0] + if len(matches) > 1: + console.print( + f"[yellow]Multiple bitstreams matched; selected newest: {chosen}[/yellow]" + ) + return chosen + + +def _gateware_passthrough( + argv: Sequence[str], + peripheral_dir: str, + license_args: Sequence[str], + gateware_image_tag: str, +) -> int: + """Forward ``argv`` verbatim to ``axon-peripheral-sdk`` inside the container. + + Builds the docker-run command in argv-list form (``shell=False``) so the + SDK sees its arguments byte-for-byte — no shell concatenation, no + ``shlex.quote`` escaping. Returns the SDK's exit code; the caller is + responsible for translating it into a ``sys.exit``. + + POSIX-only: ``os.getuid()`` / ``os.getgid()`` are required to construct the + ``--user`` argument. On a non-POSIX host (Python-on-Windows) those + attributes are missing and we exit with a clear error rather than silently + falling back to a hard-coded UID — Docker-for-Windows bind-mount UID + semantics are messy enough that a wrong default would cause confusing + file-ownership bugs. + """ + try: + host_uid = os.getuid() + host_gid = os.getgid() + except AttributeError: + sys.exit(_NON_POSIX_MSG) + + cmd = [ + "docker", + "run", + "--rm", + "-v", + f"{os.path.abspath(peripheral_dir)}:/home/workspace", + "-w", + "/home/workspace", + "--user", + f"{host_uid}:{host_gid}", + *license_args, + gateware_image_tag, + "axon-peripheral-sdk", + *argv, + ] + # check=False: surface the SDK's exit code rather than raising on non-zero. + result = subprocess.run(cmd, check=False) + return result.returncode diff --git a/synapse/cli/peripherals.py b/synapse/cli/peripherals.py index 68250365..4756306e 100644 --- a/synapse/cli/peripherals.py +++ b/synapse/cli/peripherals.py @@ -20,13 +20,16 @@ import os import shutil import subprocess +import sys import tempfile +from pathlib import Path from typing import Optional from rich import box from rich.console import Console from rich.panel import Panel +from synapse.cli import gateware from synapse.cli.build import ( build_docker_image, detect_arch, @@ -35,6 +38,7 @@ validate_manifest, ) from synapse.cli.deploy import deploy_package +from synapse.cli.gateware import LicenseUnsetError console = Console() @@ -72,7 +76,22 @@ def add_commands(subparsers: argparse._SubParsersAction): default=False, help="Clean build directories before compiling", ) - build_parser.set_defaults(func=build_cmd) + build_half_group = build_parser.add_mutually_exclusive_group() + build_half_group.add_argument( + "--driver", + dest="half", + action="store_const", + const="driver", + help="Build/package only the driver .so (skips the gateware container).", + ) + build_half_group.add_argument( + "--gateware", + dest="half", + action="store_const", + const="gateware", + help="Build/package only the FPGA .bit (skips cmake/vcpkg).", + ) + build_parser.set_defaults(func=build_cmd, half="both") deploy_parser = peripherals_subparsers.add_parser( "deploy", @@ -94,7 +113,44 @@ def add_commands(subparsers: argparse._SubParsersAction): default=None, help="Path to a pre-built .deb to deploy (skips local build and package steps)", ) - deploy_parser.set_defaults(func=deploy_cmd) + deploy_half_group = deploy_parser.add_mutually_exclusive_group() + deploy_half_group.add_argument( + "--driver", + dest="half", + action="store_const", + const="driver", + help="Build/deploy only the driver .so (skips the gateware container).", + ) + deploy_half_group.add_argument( + "--gateware", + dest="half", + action="store_const", + const="gateware", + help="Build/deploy only the FPGA .bit (skips cmake/vcpkg).", + ) + deploy_parser.set_defaults(func=deploy_cmd, half="both") + + # `peripherals gateware [args...]` — pass-through dispatcher to + # axon-peripheral-sdk inside the gateware container. argparse.REMAINDER + # captures the entire tail verbatim so the SDK is the sole source of + # truth for verbs and flags; synapsectl does NOT gate on a known-verb + # list. peripheral_dir is intentionally NOT a positional here -- REMAINDER + # would swallow it -- the dispatcher uses os.getcwd() instead. + gateware_parser = peripherals_subparsers.add_parser( + "gateware", + help="Pass arguments through to axon-peripheral-sdk inside the gateware container.", + description=( + "Forwards the verb and arguments verbatim to axon-peripheral-sdk " + "inside the gateware container. Run `synapsectl peripherals gateware " + " --help` for SDK-side help." + ), + ) + gateware_parser.add_argument( + "argv", + nargs=argparse.REMAINDER, + help="SDK verb and its arguments (forwarded verbatim).", + ) + gateware_parser.set_defaults(func=gateware_cmd) # --------------------------------------------------------------------------- @@ -116,6 +172,26 @@ def _expected_so_filename(manifest: dict) -> str: return f"{manifest['name']}.so" +def _expected_bit_filename(manifest: dict) -> str: + """Return the basename of the .bit this plugin produces. + + Reads manifest.install.gateware_target if present (e.g. + "/usr/lib/scifi/gateware/via.bit" → "via.bit"), + otherwise falls back to the .so stem (e.g. + "via.so" → "via.bit"), which itself falls back + to ".bit" when install.target is also absent. + + An empty-string gateware_target is treated as "not set" and falls + through to the .so-stem fallback. + """ + install = manifest.get("install") or {} + target = install.get("gateware_target") + if target: + return os.path.basename(target) + so_stem = os.path.splitext(_expected_so_filename(manifest))[0] + return f"{so_stem}.bit" + + # --------------------------------------------------------------------------- # Build .so # --------------------------------------------------------------------------- @@ -130,7 +206,7 @@ def build_peripheral_so( so_path = os.path.join(peripheral_dir, "build/aarch64", so_filename) try: - image_tag = build_docker_image(peripheral_dir, plugin_name) + image_tag = build_docker_image(peripheral_dir, plugin_name)["driver"] except (subprocess.CalledProcessError, FileNotFoundError) as exc: console.print( f"[bold red]Error:[/bold red] Failed to build Docker image: {exc}" @@ -140,10 +216,14 @@ def build_peripheral_so( if clean: console.print("[yellow]Cleaning build directories...[/yellow]") clean_cmd = [ - "docker", "run", "--rm", - "-v", f"{os.path.abspath(peripheral_dir)}:/home/workspace", + "docker", + "run", + "--rm", + "-v", + f"{os.path.abspath(peripheral_dir)}:/home/workspace", image_tag, - "/bin/bash", "-c", + "/bin/bash", + "-c", "cd /home/workspace && rm -rf build/ || true", ] try: @@ -153,10 +233,14 @@ def build_peripheral_so( console.print("[blue]Installing dependencies (vcpkg)...[/blue]") vcpkg_cmd = [ - "docker", "run", "--rm", - "-v", f"{os.path.abspath(peripheral_dir)}:/home/workspace", + "docker", + "run", + "--rm", + "-v", + f"{os.path.abspath(peripheral_dir)}:/home/workspace", image_tag, - "/bin/bash", "-c", + "/bin/bash", + "-c", "cd /home/workspace && " "if [ -f vcpkg.json ]; then " '${VCPKG_ROOT}/vcpkg install --triplet arm64-linux-dynamic-release --x-install-root "$PWD/build/host/vcpkg_installed"; ' @@ -188,10 +272,14 @@ def build_peripheral_so( "fi" ) build_cmd_args = [ - "docker", "run", "--rm", - "-v", f"{os.path.abspath(peripheral_dir)}:/home/workspace", + "docker", + "run", + "--rm", + "-v", + f"{os.path.abspath(peripheral_dir)}:/home/workspace", image_tag, - "/bin/bash", "-c", + "/bin/bash", + "-c", build_cmd_str, ] try: @@ -213,18 +301,25 @@ def build_peripheral_so( try: found = subprocess.run( [ - "find", peripheral_dir, "-type", "f", "-name", so_filename, - "-not", "-path", "*/.*", + "find", + peripheral_dir, + "-type", + "f", + "-name", + so_filename, + "-not", + "-path", + "*/.*", ], - capture_output=True, text=True, check=False, + capture_output=True, + text=True, + check=False, ).stdout.strip() if found: located = found.split("\n")[0] os.makedirs(os.path.dirname(so_path), exist_ok=True) shutil.copy(located, so_path) - console.print( - f"[green]Copied {located} → {so_path}[/green]" - ) + console.print(f"[green]Copied {located} → {so_path}[/green]") return True except Exception: pass @@ -241,69 +336,141 @@ def build_peripheral_so( def build_peripheral_deb( - peripheral_dir: str, plugin_name: str, so_filename: str, version: str = "0.1.0" + peripheral_dir: str, + manifest: dict, + *, + bit_path: Optional[str] = None, + so_path: Optional[str] = None, + version: str = "0.1.0", ) -> bool: - """Stage plugin .so + SDK runtime library, then run fpm to produce a .deb. + """Stage plugin artifacts + SDK runtime, then run fpm to produce a .deb. - Layout inside the .deb: - /usr/lib/scifi/plugins/ ← the plugin itself - /usr/lib/libscifi-peripheral-sdk.so.* ← extracted from the builder image + Layout inside the .deb (entries present per the supplied paths): + /usr/lib/scifi/plugins/ ← when so_path is provided + /usr/lib/scifi/gateware/ ← when bit_path is provided + /usr/lib/libscifi-peripheral-sdk.so.* ← only when so_path is provided Section is set to `synapse-peripherals` so scifi-server's DeployApp gate accepts it (sibling accept-list entry next to `synapse-apps`). """ + if so_path is None and bit_path is None: + console.print( + "[bold red]Error:[/bold red] build_peripheral_deb requires at least one of " + "so_path or bit_path." + ) + return False + + plugin_name = manifest["name"] + so_filename = _expected_so_filename(manifest) + bit_filename = _expected_bit_filename(manifest) + staging_dir = tempfile.mkdtemp(prefix="synapse-peripheral-package-") try: - so_path = os.path.join(peripheral_dir, "build/aarch64", so_filename) - if not os.path.exists(so_path): - console.print( - f"[bold red]Error:[/bold red] Plugin .so not found at {so_path}" - ) - return False - # 1. Stage the plugin .so at /usr/lib/scifi/plugins/.so - plugin_dst = os.path.join(staging_dir, "usr", "lib", "scifi", "plugins") - os.makedirs(plugin_dst, exist_ok=True) - shutil.copy2(so_path, os.path.join(plugin_dst, so_filename)) + if so_path is not None: + if not os.path.exists(so_path): + console.print( + f"[bold red]Error:[/bold red] Plugin .so not found at {so_path}" + ) + return False + plugin_dst = os.path.join(staging_dir, "usr", "lib", "scifi", "plugins") + os.makedirs(plugin_dst, exist_ok=True) + shutil.copy2(so_path, os.path.join(plugin_dst, so_filename)) + + # 1b. Stage the gateware .bit at /usr/lib/scifi/gateware/.bit + if bit_path is not None: + if not os.path.exists(bit_path): + console.print( + f"[bold red]Error:[/bold red] Gateware .bit not found at {bit_path}" + ) + return False + gw_dst = os.path.join(staging_dir, "usr", "lib", "scifi", "gateware") + os.makedirs(gw_dst, exist_ok=True) + shutil.copy2(bit_path, os.path.join(gw_dst, bit_filename)) # 2. Stage libscifi-peripheral-sdk.so* from the builder image at /usr/lib. - # The SDK ships there via `apt-get install scifi-peripheral-sdk` inside - # the builder Dockerfile, so it's the same source the linker resolved - # against at build time — guaranteeing ABI alignment for the plugin. - sdk_dst = os.path.join(staging_dir, "usr", "lib") - os.makedirs(sdk_dst, exist_ok=True) - - arch_suffix = detect_arch() - image_tag = f"{plugin_name}:latest-{arch_suffix}" - platform_opt = "linux/arm64" if arch_suffix == "arm64" else "linux/amd64" - - console.print( - f"[yellow]Extracting SDK runtime from Docker image [bold]{image_tag}[/bold]...[/yellow]" - ) - extract_cmd = [ - "docker", "run", "--rm", - "--platform", platform_opt, - "-v", f"{sdk_dst}:/out", - image_tag, - "/bin/bash", "-c", - r"find /usr/lib -maxdepth 1 -name 'libscifi-peripheral-sdk.so*' -exec cp -a {} /out/ \;", - ] - try: - subprocess.run(extract_cmd, check=True) - except subprocess.CalledProcessError as exc: - console.print( - f"[bold red]Error:[/bold red] Failed to extract SDK runtime: {exc}" - ) - return False - - # 3. Sanity check — make sure the extraction actually copied something. - sdk_files = [f for f in os.listdir(sdk_dst) if f.startswith("libscifi-peripheral-sdk.so")] - if not sdk_files: - console.print( - "[bold red]Error:[/bold red] SDK runtime libraries not found in builder image. " - "Make sure your Dockerfile installs scifi-peripheral-sdk." + # Only when the driver half is part of this .deb — a gateware-only + # package has no need for the C++ runtime. The SDK ships via + # `apt-get install scifi-peripheral-sdk` inside the builder Dockerfile, + # so it's the same source the linker resolved against at build time — + # guaranteeing ABI alignment for the plugin. + if so_path is not None: + sdk_dst = os.path.join(staging_dir, "usr", "lib") + os.makedirs(sdk_dst, exist_ok=True) + + # Prefer libs already produced on disk next to the .so (the driver + # builder may stage them there). Fall back to extracting from the + # builder image only if none are present locally. + local_libs_dir = os.path.join(peripheral_dir, "build", "aarch64") + local_libs = ( + [ + f + for f in os.listdir(local_libs_dir) + if f.startswith("libscifi-peripheral-sdk.so") + ] + if os.path.isdir(local_libs_dir) + else [] ) - return False + if local_libs: + for fname in local_libs: + shutil.copy2( + os.path.join(local_libs_dir, fname), + os.path.join(sdk_dst, fname), + ) + else: + try: + image_tag = build_docker_image(peripheral_dir, plugin_name)[ + "driver" + ] + except ( + subprocess.CalledProcessError, + FileNotFoundError, + KeyError, + ) as exc: + console.print( + f"[bold red]Error:[/bold red] Failed to resolve driver image tag: {exc}" + ) + return False + arch_suffix = detect_arch() + platform_opt = ( + "linux/arm64" if arch_suffix == "arm64" else "linux/amd64" + ) + + console.print( + f"[yellow]Extracting SDK runtime from Docker image [bold]{image_tag}[/bold]...[/yellow]" + ) + extract_cmd = [ + "docker", + "run", + "--rm", + "--platform", + platform_opt, + "-v", + f"{sdk_dst}:/out", + image_tag, + "/bin/bash", + "-c", + r"find /usr/lib -maxdepth 1 -name 'libscifi-peripheral-sdk.so*' -exec cp -a {} /out/ \;", + ] + try: + subprocess.run(extract_cmd, check=True) + except subprocess.CalledProcessError as exc: + console.print( + f"[bold red]Error:[/bold red] Failed to extract SDK runtime: {exc}" + ) + return False + + sdk_files = [ + f + for f in os.listdir(sdk_dst) + if f.startswith("libscifi-peripheral-sdk.so") + ] + if not sdk_files: + console.print( + "[bold red]Error:[/bold red] SDK runtime libraries not found in builder image. " + "Make sure your Dockerfile installs scifi-peripheral-sdk." + ) + return False # 4. Postinstall: nudge the user to restart scifi-server. # Restarting automatically could interrupt an active recording session, @@ -324,18 +491,28 @@ def build_peripheral_deb( fpm_args = [ "fpm", - "-s", "dir", - "-t", "deb", - "-n", plugin_name, + "-s", + "dir", + "-t", + "deb", + "-n", + plugin_name, "-f", - "-v", version, - "-C", "/pkg", + "-v", + version, + "-C", + "/pkg", "--deb-no-default-config-files", - "--vendor", "Science Corporation", - "--description", "Synapse peripheral plugin", - "--architecture", "arm64", - "--category", SECTION_LABEL, - "--after-install", "/pkg/postinstall.sh", + "--vendor", + "Science Corporation", + "--description", + "Synapse peripheral plugin", + "--architecture", + "arm64", + "--category", + SECTION_LABEL, + "--after-install", + "/pkg/postinstall.sh", ".", ] @@ -343,11 +520,17 @@ def build_peripheral_deb( f"[yellow]Packaging plugin .deb (Docker image: {FPM_IMAGE}) ...[/yellow]" ) docker_fpm_cmd = [ - "docker", "run", "--rm", - "--platform", "linux/amd64", - "-v", f"{staging_dir}:/pkg", - "-v", f"{dist_dir}:/out", - "-w", "/out", + "docker", + "run", + "--rm", + "--platform", + "linux/amd64", + "-v", + f"{staging_dir}:/pkg", + "-v", + f"{dist_dir}:/out", + "-w", + "/out", FPM_IMAGE, ] + fpm_args @@ -379,6 +562,56 @@ def build_peripheral_deb( # /tmp eventually cleans itself. +# --------------------------------------------------------------------------- +# Gateware half helpers +# --------------------------------------------------------------------------- + + +def _clean_gateware_tree(peripheral_dir: str, gateware_image_tag: str) -> None: + """Wipe ``/src/gateware/build/`` via a docker run. + + Mirrors the driver-side clean in :func:`build_peripheral_so`: it runs the + rm inside the gateware container so the host user does not need to chown + files written as the in-container ``dev`` user. + """ + console.print("[yellow]Cleaning gateware build directory...[/yellow]") + clean_cmd = [ + "docker", + "run", + "--rm", + "-v", + f"{os.path.abspath(peripheral_dir)}:/home/workspace", + gateware_image_tag, + "/bin/bash", + "-c", + "cd /home/workspace && rm -rf src/gateware/build || true", + ] + try: + subprocess.run(clean_cmd, check=True, cwd=peripheral_dir) + except subprocess.CalledProcessError: + console.print("[yellow]Warning: gateware clean failed; continuing.[/yellow]") + + +def _run_gateware_half(peripheral_dir: str, plugin_name: str) -> Optional[str]: + """Run the gateware build half; return the emitted ``.bit`` path or None.""" + try: + image_tag = build_docker_image(peripheral_dir, plugin_name)["gateware"] + except (subprocess.CalledProcessError, FileNotFoundError, KeyError) as exc: + console.print( + f"[bold red]Error:[/bold red] Failed to build gateware Docker image: {exc}" + ) + return None + + try: + return gateware.run_gateware_build(peripheral_dir, image_tag) + except LicenseUnsetError as exc: + console.print(f"[bold red]Error:[/bold red] {exc}") + return None + except (subprocess.CalledProcessError, FileNotFoundError) as exc: + console.print(f"[bold red]Error:[/bold red] Gateware build failed: {exc}") + return None + + # --------------------------------------------------------------------------- # `peripherals build` # --------------------------------------------------------------------------- @@ -399,16 +632,49 @@ def build_cmd(args) -> None: plugin_name = manifest["name"] version = manifest.get("version", "0.1.0") so_filename = _expected_so_filename(manifest) + half = getattr(args, "half", "both") + do_driver = half in ("driver", "both") + do_gateware = half in ("gateware", "both") console.print( f"[bold]Building peripheral plugin:[/bold] [yellow]{plugin_name}[/yellow] " f"(artifact: [cyan]{so_filename}[/cyan])" ) - if not build_peripheral_so(peripheral_dir, plugin_name, so_filename, clean=args.clean): - return + if do_gateware and args.clean: + try: + gateware_image_tag = build_docker_image(peripheral_dir, plugin_name)[ + "gateware" + ] + except (subprocess.CalledProcessError, FileNotFoundError, KeyError) as exc: + console.print( + f"[bold red]Error:[/bold red] Failed to build gateware Docker image: {exc}" + ) + return + _clean_gateware_tree(peripheral_dir, gateware_image_tag) - if not build_peripheral_deb(peripheral_dir, plugin_name, so_filename, version=version): + so_path: Optional[str] = None + bit_path: Optional[str] = None + + if do_driver: + if not build_peripheral_so( + peripheral_dir, plugin_name, so_filename, clean=args.clean + ): + return + so_path = os.path.join(peripheral_dir, "build/aarch64", so_filename) + + if do_gateware: + bit_path = _run_gateware_half(peripheral_dir, plugin_name) + if bit_path is None: + return + + if not build_peripheral_deb( + peripheral_dir, + manifest, + so_path=so_path, + bit_path=bit_path, + version=version, + ): return deb_path = find_deb_package(os.path.join(peripheral_dir, "dist")) @@ -441,8 +707,15 @@ def deploy_cmd(args) -> None: accept-list. No new RPC, no new install plumbing. """ + half = getattr(args, "half", "both") + # --package short-circuit: skip build, deploy the supplied .deb directly. if args.package: + if half != "both": + console.print( + f"[yellow]Warning: --{half} ignored when --package is provided; " + f"deploying the supplied .deb as-is.[/yellow]" + ) deb_package: Optional[str] = os.path.abspath(args.package) if not os.path.exists(deb_package): console.print( @@ -464,14 +737,33 @@ def deploy_cmd(args) -> None: plugin_name = manifest["name"] version = manifest.get("version", "0.1.0") so_filename = _expected_so_filename(manifest) + do_driver = half in ("driver", "both") + do_gateware = half in ("gateware", "both") console.print( f"[bold]Deploying peripheral plugin:[/bold] [yellow]{plugin_name}[/yellow]" ) - if not build_peripheral_so(peripheral_dir, plugin_name, so_filename): - return - if not build_peripheral_deb(peripheral_dir, plugin_name, so_filename, version=version): + so_path: Optional[str] = None + bit_path: Optional[str] = None + + if do_driver: + if not build_peripheral_so(peripheral_dir, plugin_name, so_filename): + return + so_path = os.path.join(peripheral_dir, "build/aarch64", so_filename) + + if do_gateware: + bit_path = _run_gateware_half(peripheral_dir, plugin_name) + if bit_path is None: + return + + if not build_peripheral_deb( + peripheral_dir, + manifest, + so_path=so_path, + bit_path=bit_path, + version=version, + ): return deb_package = find_deb_package(os.path.join(peripheral_dir, "dist")) @@ -486,3 +778,71 @@ def deploy_cmd(args) -> None: return deploy_package(args.uri, deb_package) + + +# --------------------------------------------------------------------------- +# `peripherals gateware [args...]` pass-through dispatcher +# --------------------------------------------------------------------------- + + +def gateware_cmd(args) -> None: + """Handle ``synapsectl peripherals gateware [args...]``. + + Forwards ``args.argv`` (captured by ``argparse.REMAINDER``) verbatim to + ``axon-peripheral-sdk`` inside the gateware container. The handler + always terminates via ``sys.exit`` so the SDK's exit code propagates + cleanly up to the shell. + + Order of operations (mirrors the plan's AC-13 spec): + + 1. Resolve LM_LICENSE_FILE -> docker flags. Unset license short-circuits + before any docker invocation. + 2. Resolve the peripheral dir to ``os.getcwd()``. REMAINDER captures + every token after ``gateware``, so a positional ``peripheral_dir`` + cannot coexist with the pass-through; cwd is the only sensible default. + 3. Require ``Dockerfiles/gateware.Dockerfile`` so the user gets a clear + error before the docker build attempts a missing context. + 4. Build / fetch the gateware image tag via :func:`build_docker_image`. + 5. Delegate to :func:`gateware._gateware_passthrough` and ``sys.exit`` on + its return code. + """ + try: + license_args = gateware.build_license_docker_args(os.environ) + except LicenseUnsetError as exc: + console.print(f"[bold red]Error:[/bold red] {exc}") + sys.exit(1) + + peripheral_dir = os.path.abspath(os.getcwd()) + + dockerfile = Path(peripheral_dir) / "Dockerfiles" / "gateware.Dockerfile" + if not dockerfile.exists(): + console.print( + "[bold red]Error:[/bold red] No gateware Dockerfile found at " + f"{dockerfile}. The `gateware` subcommand requires " + "Dockerfiles/gateware.Dockerfile in the peripheral plugin directory." + ) + sys.exit(1) + + try: + tags = build_docker_image(peripheral_dir) + except (subprocess.CalledProcessError, FileNotFoundError) as exc: + console.print( + f"[bold red]Error:[/bold red] Failed to build Docker image: {exc}" + ) + sys.exit(1) + + if "gateware" not in tags: + console.print( + "[bold red]Error:[/bold red] build_docker_image returned no 'gateware' " + "tag; cannot run the gateware pass-through." + ) + sys.exit(1) + + sys.exit( + gateware._gateware_passthrough( + argv=list(args.argv), + peripheral_dir=peripheral_dir, + license_args=license_args, + gateware_image_tag=tags["gateware"], + ) + ) From ad8c90cf13ec3aec14794cae658796f54e343d4c Mon Sep 17 00:00:00 2001 From: Vishnu Therayil Sasikumar Date: Tue, 26 May 2026 10:56:15 -0700 Subject: [PATCH 02/21] Gets Host MAC for Lattice Radiant License --- synapse/cli/build.py | 28 +++++++++++++++++++++++++++- synapse/cli/gateware.py | 34 +++++++++++++++++++++++++++++++--- synapse/cli/peripherals.py | 34 +++++++++++++++++++--------------- 3 files changed, 77 insertions(+), 19 deletions(-) diff --git a/synapse/cli/build.py b/synapse/cli/build.py index f560f122..70d4c499 100644 --- a/synapse/cli/build.py +++ b/synapse/cli/build.py @@ -93,7 +93,11 @@ def _dockerfile_needs_host_uid(dockerfile_path: str) -> bool: return bool(_ARG_HOST_UID_RE.search(fp.read())) -def build_docker_image(app_dir: str, app_name: str | None = None) -> dict[str, str]: +def build_docker_image( + app_dir: str, + app_name: str | None = None, + roles: list[str] | None = None, +) -> dict[str, str]: """(Re)build the cross-compile SDK Docker image(s) and return role -> tag. Discovers every ``*.Dockerfile`` directly under ``/Dockerfiles/`` @@ -113,6 +117,17 @@ def build_docker_image(app_dir: str, app_name: str | None = None) -> dict[str, s ``^ARG HOST_UID\\b`` the build additionally receives ``--build-arg HOST_UID=``. On non-POSIX hosts (where ``os.getuid()`` is unavailable) the value falls back to ``1000``. + + Args: + app_dir, app_name: as before. + roles: when provided, restricts the build to Dockerfiles whose role + (filename stem) is in this list. If a requested role is not found + on disk, raises FileNotFoundError. When None (default), builds + every discovered Dockerfile. The back-compat legacy single- + ``./Dockerfile`` path is treated as role "driver"; pass + ``roles=["driver"]`` (or None) to consume it; passing + ``roles=["gateware"]`` against a legacy repo with no + ``Dockerfiles/`` raises FileNotFoundError. """ if app_name is None: @@ -142,6 +157,17 @@ def build_docker_image(app_dir: str, app_name: str | None = None) -> dict[str, s "provides the required Dockerfile." ) + if roles is not None: + requested = set(roles) + available = {role for role, _ in discovered} + missing = requested - available + if missing: + raise FileNotFoundError( + f"Requested roles {sorted(missing)} not found in {dockerfiles_dir}. " + f"Available roles: {sorted(available)}." + ) + discovered = [(role, path) for role, path in discovered if role in requested] + tags: dict[str, str] = {} for role, dockerfile_path in discovered: image_tag = ( diff --git a/synapse/cli/gateware.py b/synapse/cli/gateware.py index 7c4e89ef..26e8241f 100644 --- a/synapse/cli/gateware.py +++ b/synapse/cli/gateware.py @@ -14,6 +14,7 @@ import re import subprocess import sys +import uuid from pathlib import Path from typing import Mapping, Sequence @@ -53,6 +54,26 @@ class LicenseUnsetError(RuntimeError): """Raised when ``LM_LICENSE_FILE`` is unset or empty.""" +def _host_mac_address() -> str | None: + """Return the host's primary MAC as ``xx:xx:xx:xx:xx:xx``, or ``None``. + + Lattice node-locked licenses are bound to the host's MAC. When we run + Radiant inside a container, FlexLM sees the container's virtual eth0 + MAC (auto-generated, different from the host) and rejects the license. + Passing ``--mac-address`` to ``docker run`` forces the container's eth0 + onto the host's MAC so the license validates. + + Uses :func:`uuid.getnode`, which falls back to a random multicast MAC + when no real hardware address is available; we detect that case via + the multicast bit and return ``None`` so the caller can skip the + ``--mac-address`` flag rather than passing a useless random value. + """ + node = uuid.getnode() + if (node >> 40) & 0x01: + return None + return ":".join(f"{(node >> (8 * i)) & 0xFF:02x}" for i in range(5, -1, -1)) + + def build_license_docker_args( env: Mapping[str, str] = os.environ, ) -> list[str]: @@ -62,10 +83,13 @@ def build_license_docker_args( - **File path** (default branch): resolved with ``Path(value).expanduser().resolve(strict=True)`` and bind-mounted - read-only into the container at ``/opt/lattice/license.dat``. + read-only into the container at ``/opt/lattice/license.dat``. The + host's MAC is also forwarded via ``--mac-address`` (when detectable) + so the container's eth0 matches the node-locked license's HOSTID. - **Floating** (``port@host`` or ``port1@host1:port2@host2``): the value is forwarded verbatim via ``-e LM_LICENSE_FILE=`` - with no bind-mount. + with no bind-mount or MAC override — a license server checks out + tokens by network, hostid is irrelevant. - **Unset / empty**: raises :class:`LicenseUnsetError`. The helper reads only from ``env`` and never falls back to @@ -79,12 +103,16 @@ def build_license_docker_args( return ["-e", f"LM_LICENSE_FILE={value}"] resolved = Path(value).expanduser().resolve(strict=True) - return [ + args = [ "-v", f"{resolved}:{_CONTAINER_LICENSE_PATH}:ro", "-e", f"LM_LICENSE_FILE={_CONTAINER_LICENSE_PATH}", ] + mac = _host_mac_address() + if mac is not None: + args.extend(["--mac-address", mac]) + return args _SDK_BUILD_CMD = ( diff --git a/synapse/cli/peripherals.py b/synapse/cli/peripherals.py index 4756306e..620c2e09 100644 --- a/synapse/cli/peripherals.py +++ b/synapse/cli/peripherals.py @@ -206,10 +206,12 @@ def build_peripheral_so( so_path = os.path.join(peripheral_dir, "build/aarch64", so_filename) try: - image_tag = build_docker_image(peripheral_dir, plugin_name)["driver"] + image_tag = build_docker_image( + peripheral_dir, "axon-peripheral", roles=["driver"] + )["driver"] except (subprocess.CalledProcessError, FileNotFoundError) as exc: console.print( - f"[bold red]Error:[/bold red] Failed to build Docker image: {exc}" + f"[bold red]Error:[/bold red] Failed to build driver Docker image: {exc}" ) return False @@ -419,16 +421,16 @@ def build_peripheral_deb( ) else: try: - image_tag = build_docker_image(peripheral_dir, plugin_name)[ - "driver" - ] + image_tag = build_docker_image( + peripheral_dir, "axon-peripheral", roles=["driver"] + )["driver"] except ( subprocess.CalledProcessError, FileNotFoundError, KeyError, ) as exc: console.print( - f"[bold red]Error:[/bold red] Failed to resolve driver image tag: {exc}" + f"[bold red]Error:[/bold red] Failed to build driver Docker image: {exc}" ) return False arch_suffix = detect_arch() @@ -592,10 +594,12 @@ def _clean_gateware_tree(peripheral_dir: str, gateware_image_tag: str) -> None: console.print("[yellow]Warning: gateware clean failed; continuing.[/yellow]") -def _run_gateware_half(peripheral_dir: str, plugin_name: str) -> Optional[str]: +def _run_gateware_half(peripheral_dir: str) -> Optional[str]: """Run the gateware build half; return the emitted ``.bit`` path or None.""" try: - image_tag = build_docker_image(peripheral_dir, plugin_name)["gateware"] + image_tag = build_docker_image( + peripheral_dir, "axon-peripheral", roles=["gateware"] + )["gateware"] except (subprocess.CalledProcessError, FileNotFoundError, KeyError) as exc: console.print( f"[bold red]Error:[/bold red] Failed to build gateware Docker image: {exc}" @@ -643,9 +647,9 @@ def build_cmd(args) -> None: if do_gateware and args.clean: try: - gateware_image_tag = build_docker_image(peripheral_dir, plugin_name)[ - "gateware" - ] + gateware_image_tag = build_docker_image( + peripheral_dir, "axon-peripheral", roles=["gateware"] + )["gateware"] except (subprocess.CalledProcessError, FileNotFoundError, KeyError) as exc: console.print( f"[bold red]Error:[/bold red] Failed to build gateware Docker image: {exc}" @@ -664,7 +668,7 @@ def build_cmd(args) -> None: so_path = os.path.join(peripheral_dir, "build/aarch64", so_filename) if do_gateware: - bit_path = _run_gateware_half(peripheral_dir, plugin_name) + bit_path = _run_gateware_half(peripheral_dir) if bit_path is None: return @@ -753,7 +757,7 @@ def deploy_cmd(args) -> None: so_path = os.path.join(peripheral_dir, "build/aarch64", so_filename) if do_gateware: - bit_path = _run_gateware_half(peripheral_dir, plugin_name) + bit_path = _run_gateware_half(peripheral_dir) if bit_path is None: return @@ -824,10 +828,10 @@ def gateware_cmd(args) -> None: sys.exit(1) try: - tags = build_docker_image(peripheral_dir) + tags = build_docker_image(peripheral_dir, "axon-peripheral", roles=["gateware"]) except (subprocess.CalledProcessError, FileNotFoundError) as exc: console.print( - f"[bold red]Error:[/bold red] Failed to build Docker image: {exc}" + f"[bold red]Error:[/bold red] Failed to build gateware Docker image: {exc}" ) sys.exit(1) From 057a3368522911abde41b6563d815d33bc881cc9 Mon Sep 17 00:00:00 2001 From: Vishnu Therayil Sasikumar Date: Tue, 2 Jun 2026 14:23:27 -0700 Subject: [PATCH 03/21] feat(peripherals): make build/deploy half-selectors subcommands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the mutually-exclusive --driver/--gateware flags on `synapsectl peripherals build` and `... deploy` with explicit driver/gateware/both subcommands: synapsectl peripherals build driver|gateware|both [dir] [--clean] synapsectl peripherals deploy driver|gateware|both [dir] [--package P] A new `_add_half_subcommands` helper wires the three leaf subparsers, each setting `half` via set_defaults — the value build_cmd/deploy_cmd already branch on, so the handlers are unchanged. A bare `build`/`deploy` prints help; an unknown target gives argparse's "invalid choice". The `peripherals gateware ` SDK pass-through is untouched. Also fixes pre-existing lint surfaced while editing: - validate_manifest return type dict|bool -> dict|Literal[False] (it never returns True), clearing spurious Literal[True] type errors at every call site. - staging-time maintainer scripts chmod 0o755 -> 0o644 (fpm embeds the contents and dpkg sets the exec bit at install; the staging mode never ships). Adds the peripheral-CLI unit test suite under synapse/tests/cli/. --- synapse/cli/build.py | 14 +- synapse/cli/peripherals.py | 117 +- synapse/tests/cli/__init__.py | 0 synapse/tests/cli/conftest.py | 43 + .../tests/cli/test_gateware_passthrough.py | 815 +++++++++++++ synapse/tests/cli/test_gateware_runner.py | 256 +++++ synapse/tests/cli/test_half_selectors.py | 1009 +++++++++++++++++ synapse/tests/cli/test_license_mode.py | 268 +++++ 8 files changed, 2460 insertions(+), 62 deletions(-) create mode 100644 synapse/tests/cli/__init__.py create mode 100644 synapse/tests/cli/conftest.py create mode 100644 synapse/tests/cli/test_gateware_passthrough.py create mode 100644 synapse/tests/cli/test_gateware_runner.py create mode 100644 synapse/tests/cli/test_half_selectors.py create mode 100644 synapse/tests/cli/test_license_mode.py diff --git a/synapse/cli/build.py b/synapse/cli/build.py index 70d4c499..baaef5f9 100644 --- a/synapse/cli/build.py +++ b/synapse/cli/build.py @@ -7,7 +7,7 @@ import shutil import subprocess import tempfile -from typing import Any +from typing import Any, Literal from rich import box from rich.console import Console @@ -16,7 +16,7 @@ console = Console() -def validate_manifest(manifest_path: str) -> dict[str, Any] | bool: +def validate_manifest(manifest_path: str) -> dict[str, Any] | Literal[False]: """Return the parsed ``manifest.json`` dictionary or ``False`` on error.""" try: @@ -410,10 +410,14 @@ def build_deb_package(app_dir: str, app_name: str, version: str = "0.1.0") -> bo lifecycle_scripts_tmp: list[str] = [] + # 0o644 suffices for all three: fpm embeds each file's *contents* as a + # .deb maintainer script (--after-install / --before-remove / + # --after-remove below), and dpkg makes maintainer scripts executable + # itself at install time. The staging files' exec bits never ship. postinstall_path = os.path.join(staging_dir, "postinstall.sh") with open(postinstall_path, "w", encoding="utf-8") as fp: fp.write("#!/bin/bash\nset -e\nsystemctl daemon-reload\n") - os.chmod(postinstall_path, 0o755) + os.chmod(postinstall_path, 0o644) lifecycle_scripts_tmp.append(postinstall_path) preremove_path = os.path.join(staging_dir, "preremove.sh") @@ -421,13 +425,13 @@ def build_deb_package(app_dir: str, app_name: str, version: str = "0.1.0") -> bo fp.write( f"#!/bin/bash\nset -e\nsystemctl stop {app_name} || true\nsystemctl disable {app_name} || true\n" ) - os.chmod(preremove_path, 0o755) + os.chmod(preremove_path, 0o644) lifecycle_scripts_tmp.append(preremove_path) postremove_path = os.path.join(staging_dir, "postremove.sh") with open(postremove_path, "w", encoding="utf-8") as fp: fp.write("#!/bin/bash\nset -e\nsystemctl daemon-reload\n") - os.chmod(postremove_path, 0o755) + os.chmod(postremove_path, 0o644) lifecycle_scripts_tmp.append(postremove_path) lib_dst_dir = os.path.join(staging_dir, "opt", "scifi", "lib") diff --git a/synapse/cli/peripherals.py b/synapse/cli/peripherals.py index 620c2e09..0155ee43 100644 --- a/synapse/cli/peripherals.py +++ b/synapse/cli/peripherals.py @@ -51,6 +51,35 @@ # --------------------------------------------------------------------------- +def _add_half_subcommands(parent_parser, *, func, action_label, extra_args): + """Wire driver/gateware/both leaf subcommands under *parent_parser*. + + Each leaf carries a ``peripheral_dir`` positional plus whatever per-command + options *extra_args* installs, and sets ``half`` to its own name so the + shared handler (*func*) branches on ``args.half`` exactly as it did under + the old half-selector flags. A bare parent command (no leaf chosen) prints + its own help. *action_label* is the verb phrase used in each leaf's help + line ("Build/package", "Build/deploy"). + """ + targets = { + "driver": "only the driver .so (skips the gateware container)", + "gateware": "only the FPGA .bit (skips cmake/vcpkg)", + "both": "both the driver .so and the FPGA .bit", + } + half_subparsers = parent_parser.add_subparsers(title="Target") + for half, what in targets.items(): + leaf = half_subparsers.add_parser(half, help=f"{action_label} {what}.") + leaf.add_argument( + "peripheral_dir", + nargs="?", + default=".", + help="Path to the peripheral plugin directory (defaults to cwd)", + ) + extra_args(leaf) + leaf.set_defaults(func=func, half=half) + parent_parser.set_defaults(func=lambda _: parent_parser.print_help()) + + def add_commands(subparsers: argparse._SubParsersAction): """Add the peripherals command group to the CLI.""" peripherals_parser = subparsers.add_parser( @@ -60,38 +89,25 @@ def add_commands(subparsers: argparse._SubParsersAction): title="Peripheral Commands" ) + # `build` / `deploy` each expose driver/gateware/both as subcommands rather + # than half-selector flags. Each leaf sets `half` to its own name, which is + # exactly the value build_cmd/deploy_cmd already branch on, so the handlers + # need no change. A bare `build`/`deploy` (no leaf chosen) prints its help. build_parser = peripherals_subparsers.add_parser( "build", - help="Cross-compile a peripheral plugin into a .so and package it as a .deb", - ) - build_parser.add_argument( - "peripheral_dir", - nargs="?", - default=".", - help="Path to the peripheral plugin directory (defaults to cwd)", - ) - build_parser.add_argument( - "--clean", - action="store_true", - default=False, - help="Clean build directories before compiling", + help="Cross-compile a peripheral plugin into a .so/.bit and package it as a .deb", ) - build_half_group = build_parser.add_mutually_exclusive_group() - build_half_group.add_argument( - "--driver", - dest="half", - action="store_const", - const="driver", - help="Build/package only the driver .so (skips the gateware container).", - ) - build_half_group.add_argument( - "--gateware", - dest="half", - action="store_const", - const="gateware", - help="Build/package only the FPGA .bit (skips cmake/vcpkg).", + _add_half_subcommands( + build_parser, + func=build_cmd, + action_label="Build/package", + extra_args=lambda leaf: leaf.add_argument( + "--clean", + action="store_true", + default=False, + help="Clean build directories before compiling", + ), ) - build_parser.set_defaults(func=build_cmd, half="both") deploy_parser = peripherals_subparsers.add_parser( "deploy", @@ -100,35 +116,18 @@ def add_commands(subparsers: argparse._SubParsersAction): "Builds first unless --package is provided." ), ) - deploy_parser.add_argument( - "peripheral_dir", - nargs="?", - default=".", - help="Path to the peripheral plugin directory (defaults to cwd)", - ) - deploy_parser.add_argument( - "--package", - "-p", - type=str, - default=None, - help="Path to a pre-built .deb to deploy (skips local build and package steps)", - ) - deploy_half_group = deploy_parser.add_mutually_exclusive_group() - deploy_half_group.add_argument( - "--driver", - dest="half", - action="store_const", - const="driver", - help="Build/deploy only the driver .so (skips the gateware container).", - ) - deploy_half_group.add_argument( - "--gateware", - dest="half", - action="store_const", - const="gateware", - help="Build/deploy only the FPGA .bit (skips cmake/vcpkg).", + _add_half_subcommands( + deploy_parser, + func=deploy_cmd, + action_label="Build/deploy", + extra_args=lambda leaf: leaf.add_argument( + "--package", + "-p", + type=str, + default=None, + help="Path to a pre-built .deb to deploy (skips local build and package steps)", + ), ) - deploy_parser.set_defaults(func=deploy_cmd, half="both") # `peripherals gateware [args...]` — pass-through dispatcher to # axon-peripheral-sdk inside the gateware container. argparse.REMAINDER @@ -485,7 +484,11 @@ def build_peripheral_deb( "echo 'Peripheral plugin installed. Restart scifi-server to load it.'\n" "exit 0\n" ) - os.chmod(postinstall_path, 0o755) + # 0o644 is sufficient: fpm embeds this file's *contents* as the .deb's + # postinst maintainer script (via --after-install), and dpkg makes + # maintainer scripts executable itself at install time. The staging + # file's own exec bit never reaches the package. + os.chmod(postinstall_path, 0o644) # 5. Run fpm inside the cdrx/fpm-ubuntu image (matches apps' packaging path). dist_dir = os.path.join(peripheral_dir, "dist") diff --git a/synapse/tests/cli/__init__.py b/synapse/tests/cli/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/synapse/tests/cli/conftest.py b/synapse/tests/cli/conftest.py new file mode 100644 index 00000000..28447373 --- /dev/null +++ b/synapse/tests/cli/conftest.py @@ -0,0 +1,43 @@ +"""Workaround for pre-existing import bug in synapse.cli. + +`synapse.cli.__init__` eagerly imports `synapse.cli.__main__`, which imports +`synapse.cli.settings`, which fails with +`ImportError: cannot import name 'SettingDescriptor' from 'synapse.api.device_pb2'`. + +To let us import `synapse.cli.build` / `synapse.cli.peripherals` / +`synapse.cli.gateware` in unit tests, we pre-register stub modules for +`synapse.cli.settings` and `synapse.cli.__main__` in `sys.modules` BEFORE the +test collector first touches `synapse.cli`. pytest loads this conftest.py +before collecting any test in this directory. +""" + +from __future__ import annotations + +import sys +import types + + +def _install_cli_import_stubs() -> None: + # Stub synapse.cli.settings so the real module's broken import is skipped. + if "synapse.cli.settings" not in sys.modules: + stub_settings = types.ModuleType("synapse.cli.settings") + + def _add_commands(_subparsers): # pragma: no cover - never invoked in tests + return None + + stub_settings.add_commands = _add_commands # type: ignore[attr-defined] + sys.modules["synapse.cli.settings"] = stub_settings + + # Stub synapse.cli.__main__ so synapse.cli's __init__ doesn't drag in the + # whole CLI surface (which transitively imports settings the normal way). + if "synapse.cli.__main__" not in sys.modules: + stub_main = types.ModuleType("synapse.cli.__main__") + + def _main(): # pragma: no cover - never invoked in tests + return None + + stub_main.main = _main # type: ignore[attr-defined] + sys.modules["synapse.cli.__main__"] = stub_main + + +_install_cli_import_stubs() diff --git a/synapse/tests/cli/test_gateware_passthrough.py b/synapse/tests/cli/test_gateware_passthrough.py new file mode 100644 index 00000000..2cb3ec6d --- /dev/null +++ b/synapse/tests/cli/test_gateware_passthrough.py @@ -0,0 +1,815 @@ +"""AC-14 → AC-13 (sub-phase 4.6). + +Tests the ``synapsectl peripherals gateware [args...]`` pass-through +dispatcher introduced by AC-13. The dispatcher MUST forward argv verbatim +to ``axon-peripheral-sdk`` inside the gateware container with NO +synapsectl-side argv parsing, NO shell concatenation, and NO +``shlex.quote``-style escaping. + +Per AC-13 the dispatcher's top-level handler sequence is:: + + license_mode = build_license_docker_args(os.environ) + peripheral_dir = Path(os.getcwd()) + if not (peripheral_dir / "Dockerfiles" / "gateware.Dockerfile").exists(): + sys.exit(...) + gateware_image_tag = build_docker_image(str(peripheral_dir))["gateware"] + sys.exit(_gateware_passthrough(args.argv, peripheral_dir, license_mode, + gateware_image_tag)) + +so the handler ALWAYS terminates via ``sys.exit`` -- tests wrap each +invocation in ``pytest.raises(SystemExit)``. + +Mocking strategy: + * ``synapse.cli.peripherals.subprocess.run`` -> recorder returning + ``CompletedProcess(returncode=0)``. + * ``synapse.cli.peripherals.build_docker_image`` -> returns + ``{"gateware": "fake-gw:latest-amd64"}``. + * ``os.getuid`` / ``os.getgid`` -> patched at the ``synapse.cli.peripherals.os`` + name so the ``--user`` argv element is deterministic. + * ``os.getcwd`` -> patched so the dispatcher sees a tmp-path peripheral + directory containing ``Dockerfiles/gateware.Dockerfile``. + * ``LM_LICENSE_FILE`` -> set / unset via ``monkeypatch.setenv`` / + ``monkeypatch.delenv``. +""" + +from __future__ import annotations + +import argparse +import importlib +import os +import subprocess +from types import SimpleNamespace + +import pytest + + +# --------------------------------------------------------------------------- +# Fixtures / helpers +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def peripherals(): + """Lazy-import ``synapse.cli.peripherals``. + + Deferring the import lets a per-test ImportError surface as a clean + failure instead of a collection crash. + """ + return importlib.import_module("synapse.cli.peripherals") + + +@pytest.fixture() +def gateware_mod(): + return importlib.import_module("synapse.cli.gateware") + + +def _build_root_parser(peripherals): + """Build a fresh root parser wired with ``peripherals.add_commands``. + + Mirrors what ``synapse.cli.__main__`` does at runtime, but without + triggering the broken transitive import of ``synapse.cli.settings`` + (the conftest stubs both). + """ + parser = argparse.ArgumentParser(prog="synapsectl") + subparsers = parser.add_subparsers(dest="cmd") + peripherals.add_commands(subparsers) + return parser + + +def _make_peripheral_dir(tmp_path): + """Create a tmp peripheral dir with Dockerfiles/gateware.Dockerfile. + + Returns the absolute path string of the dir; the dispatcher's + cwd-based check looks for ``Dockerfiles/gateware.Dockerfile`` under + this directory. + """ + pd = tmp_path / "fake-peripheral" + pd.mkdir() + (pd / "Dockerfiles").mkdir() + (pd / "Dockerfiles" / "gateware.Dockerfile").write_text( + "FROM ubuntu:22.04\nARG HOST_UID=1000\n" + ) + return pd + + +def _make_license_file(tmp_path): + """Create a tmp license file and return its realpath.""" + lic = tmp_path / "license.dat" + lic.write_text("FAKE LATTICE LICENSE\n") + return str(lic.resolve()) + + +def _install_dispatcher_stubs( + peripherals, + monkeypatch, + tmp_path, + *, + license_value, + uid=1234, + gid=5678, + getuid_raises=None, +): + """Install stubs for the AC-13 dispatcher path and return a recorder. + + ``license_value`` may be: + * a string -> ``LM_LICENSE_FILE`` is set to that value + * ``None`` -> ``LM_LICENSE_FILE`` is deleted from os.environ + + ``getuid_raises``: if not None, ``os.getuid`` is stubbed to raise this + exception instance/class (used to model Python-on-Windows). + """ + recorder = SimpleNamespace(calls=[]) + + def fake_run(argv, *args, **kwargs): + recorder.calls.append((argv, args, dict(kwargs))) + return subprocess.CompletedProcess(argv, 0, b"", b"") + + # subprocess.run -> recorder. Patched on the module under test so we + # capture the dispatcher's exact argv-list. + monkeypatch.setattr(peripherals.subprocess, "run", fake_run) + + # build_docker_image -> fixed dict so the dispatcher gets a known tag. + monkeypatch.setattr( + peripherals, + "build_docker_image", + lambda *a, **kw: { + "driver": "fake-driver:latest-amd64", + "gateware": "fake-gw:latest-amd64", + }, + ) + + # os.getcwd -> the fake peripheral dir. + pd = _make_peripheral_dir(tmp_path) + monkeypatch.setattr(peripherals.os, "getcwd", lambda: str(pd)) + + # os.getuid / os.getgid on the peripherals module. + if getuid_raises is not None: + + def _raises(*_a, **_kw): + raise getuid_raises + + monkeypatch.setattr(peripherals.os, "getuid", _raises) + else: + monkeypatch.setattr(peripherals.os, "getuid", lambda: uid) + monkeypatch.setattr(peripherals.os, "getgid", lambda: gid) + + # LM_LICENSE_FILE. + if license_value is None: + monkeypatch.delenv("LM_LICENSE_FILE", raising=False) + else: + monkeypatch.setenv("LM_LICENSE_FILE", license_value) + + return recorder, pd + + +def _dispatch(peripherals, argv_tail): + """Drive the ``gateware`` subcommand via the real argparse surface. + + ``argv_tail`` is the list AFTER ``peripherals gateware``, e.g. + ``["doctor"]`` or ``["validate", "--project", "src/gateware"]``. + + The dispatcher handler always ends in ``sys.exit``; the caller is + responsible for wrapping in ``pytest.raises(SystemExit)``. + """ + parser = _build_root_parser(peripherals) + args = parser.parse_args(["peripherals", "gateware", *argv_tail]) + args.func(args) + + +def _docker_argv(call): + """Return the docker-run argv list from a recorded subprocess.run call.""" + argv, _pos, _kw = call + assert isinstance(argv, list), ( + f"subprocess.run must receive a Python list (argv form), not a string; " + f"got: {argv!r}" + ) + return argv + + +def _tail_after_image_tag(argv, image_tag): + """Return the argv slice AFTER the gateware image tag (inclusive of + ``axon-peripheral-sdk`` onward). + """ + assert image_tag in argv, ( + f"image tag {image_tag!r} not present in docker argv: {argv!r}" + ) + idx = argv.index(image_tag) + return argv[idx + 1 :] + + +# --------------------------------------------------------------------------- +# AC-14 case 1: no-arg verb forwarded verbatim +# --------------------------------------------------------------------------- + + +def test_case_1_no_arg_verb_doctor_forwarded_verbatim( + peripherals, tmp_path, monkeypatch +): + """1: ``gateware doctor`` -> argv tail is exactly + ``["axon-peripheral-sdk", "doctor"]``.""" + lic = _make_license_file(tmp_path) + recorder, _pd = _install_dispatcher_stubs( + peripherals, monkeypatch, tmp_path, license_value=lic + ) + + with pytest.raises(SystemExit) as excinfo: + _dispatch(peripherals, ["doctor"]) + + assert excinfo.value.code == 0 + assert len(recorder.calls) == 1, ( + f"exactly one docker-run subprocess call expected; got: {recorder.calls!r}" + ) + argv = _docker_argv(recorder.calls[0]) + tail = _tail_after_image_tag(argv, "fake-gw:latest-amd64") + assert tail == ["axon-peripheral-sdk", "doctor"], ( + f"no-arg verb must be forwarded verbatim; got tail: {tail!r}" + ) + + # shell=False (the default) -- explicit check that nobody set shell=True. + _argv, _pos, kw = recorder.calls[0] + assert kw.get("shell", False) is False, ( + f"subprocess.run must be invoked with shell=False; got kwargs: {kw!r}" + ) + + +# --------------------------------------------------------------------------- +# AC-14 case 2: long flag with value forwarded verbatim +# --------------------------------------------------------------------------- + + +def test_case_2_long_flag_value_validate_forwarded_verbatim( + peripherals, tmp_path, monkeypatch +): + """2: ``gateware validate --project src/gateware`` -> tail is exactly + ``["axon-peripheral-sdk", "validate", "--project", "src/gateware"]``.""" + lic = _make_license_file(tmp_path) + recorder, _ = _install_dispatcher_stubs( + peripherals, monkeypatch, tmp_path, license_value=lic + ) + + with pytest.raises(SystemExit): + _dispatch(peripherals, ["validate", "--project", "src/gateware"]) + + assert len(recorder.calls) == 1 + argv = _docker_argv(recorder.calls[0]) + tail = _tail_after_image_tag(argv, "fake-gw:latest-amd64") + assert tail == [ + "axon-peripheral-sdk", + "validate", + "--project", + "src/gateware", + ], f"long-flag verb must be forwarded byte-for-byte; got tail: {tail!r}" + + +# --------------------------------------------------------------------------- +# AC-14 case 3: short flag with exotic value (contains '::') +# --------------------------------------------------------------------------- + + +def test_case_3_short_flag_with_double_colon_preserved( + peripherals, tmp_path, monkeypatch +): + """3: ``gateware sim -k some::test_id`` -> the ``::`` is preserved + byte-for-byte in argv form. + + Under shell-string concatenation the ``::`` might survive too, but a + naive ``shlex.quote(arg)`` wrapper would inject single-quotes around + the value. Argv-list form makes quoting unnecessary; this test locks + that contract in so a future maintainer can't slip a quote-helper in. + """ + lic = _make_license_file(tmp_path) + recorder, _ = _install_dispatcher_stubs( + peripherals, monkeypatch, tmp_path, license_value=lic + ) + + with pytest.raises(SystemExit): + _dispatch(peripherals, ["sim", "-k", "some::test_id"]) + + assert len(recorder.calls) == 1 + argv = _docker_argv(recorder.calls[0]) + tail = _tail_after_image_tag(argv, "fake-gw:latest-amd64") + assert tail == ["axon-peripheral-sdk", "sim", "-k", "some::test_id"], ( + f"short-flag-with-exotic-value must be preserved verbatim; got tail: {tail!r}" + ) + # Belt-and-suspenders: no element in argv should equal a quote-wrapped + # version of the exotic value. + assert "'some::test_id'" not in argv, ( + f"docker argv must not shell-quote the exotic value; got: {argv!r}" + ) + + +# --------------------------------------------------------------------------- +# AC-14 case 4: --user matches patched getuid()/getgid() +# --------------------------------------------------------------------------- + + +def test_case_4_user_flag_matches_patched_uid_gid(peripherals, tmp_path, monkeypatch): + """4: ``--user :`` is built from os.getuid()/os.getgid().""" + lic = _make_license_file(tmp_path) + recorder, _ = _install_dispatcher_stubs( + peripherals, monkeypatch, tmp_path, license_value=lic, uid=4242, gid=8484 + ) + + with pytest.raises(SystemExit): + _dispatch(peripherals, ["doctor"]) + + argv = _docker_argv(recorder.calls[0]) + assert "--user" in argv, f"--user flag missing; got: {argv!r}" + user_idx = argv.index("--user") + assert argv[user_idx + 1] == "4242:8484", ( + f"--user value must be 'uid:gid' from patched getuid/getgid; " + f"got: {argv[user_idx + 1]!r}" + ) + + +# --------------------------------------------------------------------------- +# AC-14 case 5: -v :/home/workspace bind-mount +# --------------------------------------------------------------------------- + + +def test_case_5_bind_mount_is_peripheral_dir_abspath( + peripherals, tmp_path, monkeypatch +): + """5: ``-v :/home/workspace`` is present.""" + lic = _make_license_file(tmp_path) + recorder, pd = _install_dispatcher_stubs( + peripherals, monkeypatch, tmp_path, license_value=lic + ) + + with pytest.raises(SystemExit): + _dispatch(peripherals, ["doctor"]) + + argv = _docker_argv(recorder.calls[0]) + expected_mount = f"{os.path.abspath(str(pd))}:/home/workspace" + assert expected_mount in argv, ( + f"docker argv must include workspace bind-mount {expected_mount!r}; " + f"got: {argv!r}" + ) + # The element immediately before the mount-spec must be a `-v`. + mount_idx = argv.index(expected_mount) + assert argv[mount_idx - 1] == "-v", ( + f"bind-mount must be introduced by '-v'; got argv[{mount_idx - 1}]: " + f"{argv[mount_idx - 1]!r}" + ) + + +# --------------------------------------------------------------------------- +# AC-14 case 6: -w /home/workspace working dir +# --------------------------------------------------------------------------- + + +def test_case_6_working_dir_is_home_workspace(peripherals, tmp_path, monkeypatch): + """6: ``-w /home/workspace`` is present in the docker argv.""" + lic = _make_license_file(tmp_path) + recorder, _ = _install_dispatcher_stubs( + peripherals, monkeypatch, tmp_path, license_value=lic + ) + + with pytest.raises(SystemExit): + _dispatch(peripherals, ["doctor"]) + + argv = _docker_argv(recorder.calls[0]) + assert "-w" in argv, f"-w flag missing; got: {argv!r}" + w_idx = argv.index("-w") + assert argv[w_idx + 1] == "/home/workspace", ( + f"-w value must be '/home/workspace'; got: {argv[w_idx + 1]!r}" + ) + + +# --------------------------------------------------------------------------- +# AC-14 case 7: subprocess.run called with a list, shell=False +# --------------------------------------------------------------------------- + + +def test_case_7_subprocess_run_is_argv_list_no_shell( + peripherals, tmp_path, monkeypatch +): + """7: ``subprocess.run`` first arg is a list; ``shell`` is not True.""" + lic = _make_license_file(tmp_path) + recorder, _ = _install_dispatcher_stubs( + peripherals, monkeypatch, tmp_path, license_value=lic + ) + + with pytest.raises(SystemExit): + _dispatch(peripherals, ["doctor"]) + + assert len(recorder.calls) == 1 + argv, _pos, kwargs = recorder.calls[0] + assert isinstance(argv, list), ( + f"subprocess.run first arg must be a list (argv form), not a string; " + f"got type: {type(argv).__name__} value: {argv!r}" + ) + assert kwargs.get("shell", False) is False, ( + f"subprocess.run must be called with shell=False (default); " + f"got kwargs: {kwargs!r}" + ) + + +# --------------------------------------------------------------------------- +# AC-14 case 8: floating LM_LICENSE_FILE (port@host) -> -e only, no -v +# --------------------------------------------------------------------------- + + +def test_case_8_floating_license_emits_env_no_bind(peripherals, tmp_path, monkeypatch): + """8: ``LM_LICENSE_FILE=27000@licenseserver`` -> only ``-e`` is added. + + No license-file ``-v`` bind-mount; the env var is forwarded as-is. + """ + recorder, _ = _install_dispatcher_stubs( + peripherals, monkeypatch, tmp_path, license_value="27000@licenseserver" + ) + + with pytest.raises(SystemExit): + _dispatch(peripherals, ["doctor"]) + + argv = _docker_argv(recorder.calls[0]) + assert "LM_LICENSE_FILE=27000@licenseserver" in argv, ( + f"floating-license env var must be forwarded verbatim; got: {argv!r}" + ) + env_idx = argv.index("LM_LICENSE_FILE=27000@licenseserver") + assert argv[env_idx - 1] == "-e", ( + f"floating-license must be introduced by '-e'; got argv[{env_idx - 1}]: " + f"{argv[env_idx - 1]!r}" + ) + # ZERO license-bind-mounts: no `-v` element should target the in-container + # license path. + license_binds = [ + a for a in argv if isinstance(a, str) and "/opt/lattice/license.dat" in a + ] + assert license_binds == [], ( + f"floating-license mode must not emit a license bind-mount; got: {license_binds!r}" + ) + + +# --------------------------------------------------------------------------- +# AC-14 case 9: file-path LM_LICENSE_FILE -> -v + -e pair +# --------------------------------------------------------------------------- + + +def test_case_9_file_path_license_emits_bind_and_env( + peripherals, tmp_path, monkeypatch +): + """9: ``LM_LICENSE_FILE=`` -> ``-v :/opt/lattice/license.dat:ro`` + AND ``-e LM_LICENSE_FILE=/opt/lattice/license.dat``.""" + lic = _make_license_file(tmp_path) # real on-disk file + recorder, _ = _install_dispatcher_stubs( + peripherals, monkeypatch, tmp_path, license_value=lic + ) + + with pytest.raises(SystemExit): + _dispatch(peripherals, ["doctor"]) + + argv = _docker_argv(recorder.calls[0]) + expected_bind = f"{lic}:/opt/lattice/license.dat:ro" + assert expected_bind in argv, ( + f"file-path license must bind-mount as {expected_bind!r}; got: {argv!r}" + ) + bind_idx = argv.index(expected_bind) + assert argv[bind_idx - 1] == "-v", ( + f"license bind-mount must be introduced by '-v'; got argv[{bind_idx - 1}]: " + f"{argv[bind_idx - 1]!r}" + ) + # Env var pointing at the in-container path. + assert "LM_LICENSE_FILE=/opt/lattice/license.dat" in argv, ( + f"file-path license must forward in-container env var; got: {argv!r}" + ) + env_idx = argv.index("LM_LICENSE_FILE=/opt/lattice/license.dat") + assert argv[env_idx - 1] == "-e", ( + f"in-container license env var must be introduced by '-e'; " + f"got argv[{env_idx - 1}]: {argv[env_idx - 1]!r}" + ) + + +# --------------------------------------------------------------------------- +# AC-14 case 10: LM_LICENSE_FILE unset -> LicenseUnsetError, no subprocess.run +# --------------------------------------------------------------------------- + + +def test_case_10_unset_license_does_not_invoke_subprocess( + peripherals, gateware_mod, tmp_path, monkeypatch, capsys +): + """10: ``LM_LICENSE_FILE`` unset -> dispatcher raises + ``LicenseUnsetError`` (or wraps it in SystemExit) AND subprocess.run is + never called. + + AC-13's call sequence shows ``build_license_docker_args(os.environ)`` + runs BEFORE any docker invocation; an unset license must therefore + short-circuit the dispatcher before subprocess.run is touched. + + Strengthening: this case has to reach the LICENSE branch, not the + argparse-invalid-choice branch (which would also raise SystemExit(2) + today because the `gateware` subcommand isn't registered yet). We + assert the error message mentions ``LM_LICENSE_FILE`` and does NOT + contain argparse's ``invalid choice`` text, proving the dispatcher + actually reached the license-resolution step. + """ + recorder, _ = _install_dispatcher_stubs( + peripherals, monkeypatch, tmp_path, license_value=None + ) + + with pytest.raises((SystemExit, gateware_mod.LicenseUnsetError)) as excinfo: + _dispatch(peripherals, ["doctor"]) + + # If the dispatcher chose to wrap in SystemExit, the exit code must be + # non-zero; raw LicenseUnsetError is also acceptable. + if isinstance(excinfo.value, SystemExit): + assert excinfo.value.code not in (0, None), ( + f"unset license must exit non-zero; got code: {excinfo.value.code!r}" + ) + + assert recorder.calls == [], ( + f"subprocess.run must NOT be invoked when LM_LICENSE_FILE is unset; " + f"got: {recorder.calls!r}" + ) + + # Anti-tautology: the error must come from the LICENSE branch, not + # argparse's invalid-choice path. argparse would emit "invalid choice" + # to stderr and never touch the license-resolution code. + captured = capsys.readouterr() + msg = (captured.out + captured.err + str(excinfo.value)).lower() + assert "invalid choice" not in msg, ( + f"case 10 must reach the license-resolution branch, NOT argparse's " + f"invalid-choice path (which fires today because `gateware` is not " + f"yet a registered subcommand). The presence of 'invalid choice' " + f"means AC-13's subparser registration hasn't happened yet. " + f"got: {captured.out + captured.err!r} value={excinfo.value!r}" + ) + assert "lm_license_file" in msg, ( + f"unset-license error must mention 'LM_LICENSE_FILE' so users know " + f"what env var to set; got: {captured.out + captured.err!r} " + f"value={excinfo.value!r}" + ) + + +# --------------------------------------------------------------------------- +# AC-14 case 11: `gateware --help` consumed by argparse, no subprocess.run +# --------------------------------------------------------------------------- + + +def test_case_11_gateware_help_consumed_by_argparse( + peripherals, tmp_path, monkeypatch, capsys +): + """11: ``peripherals gateware --help`` -> argparse exits 0, prints + synapsectl-side gateware help; subprocess.run NOT called. + + AC-13's `--help` dichotomy: when `--help` is the FIRST token after + `gateware`, argparse consumes it BEFORE REMAINDER captures anything. + """ + # Even for the --help path the dispatcher's pre-handler stubs are + # harmless: argparse exits inside parse_args before the handler runs. + lic = _make_license_file(tmp_path) + recorder, _ = _install_dispatcher_stubs( + peripherals, monkeypatch, tmp_path, license_value=lic + ) + + parser = _build_root_parser(peripherals) + with pytest.raises(SystemExit) as excinfo: + parser.parse_args(["peripherals", "gateware", "--help"]) + + assert excinfo.value.code == 0, ( + f"--help must exit cleanly with code 0; got: {excinfo.value.code!r}" + ) + assert recorder.calls == [], ( + f"--help path must not invoke subprocess.run; got: {recorder.calls!r}" + ) + captured = capsys.readouterr() + help_text = (captured.out + captured.err).lower() + # The synapsectl-side gateware subcommand help should reference either + # the verb name "gateware" or the pass-through concept. + assert "gateware" in help_text or "pass" in help_text, ( + f"gateware --help text must mention 'gateware' or 'pass'; " + f"got: {captured.out!r} stderr={captured.err!r}" + ) + + +# --------------------------------------------------------------------------- +# AC-14 case 12: `gateware doctor --help` IS forwarded +# --------------------------------------------------------------------------- + + +def test_case_12_verb_help_is_forwarded_to_sdk(peripherals, tmp_path, monkeypatch): + """12: ``peripherals gateware doctor --help`` -> REMAINDER captures + BOTH tokens; subprocess.run IS called with the verb + --help in the tail. + + Companion to case 11: when at least one non-``--help`` positional + appears first, the entire tail is REMAINDER-captured and forwarded + untouched to ``axon-peripheral-sdk`` so the SDK shows its own + per-verb help. + """ + lic = _make_license_file(tmp_path) + recorder, _ = _install_dispatcher_stubs( + peripherals, monkeypatch, tmp_path, license_value=lic + ) + + with pytest.raises(SystemExit): + _dispatch(peripherals, ["doctor", "--help"]) + + assert len(recorder.calls) == 1, ( + f"verb + --help must be forwarded (single docker call); got: {recorder.calls!r}" + ) + argv = _docker_argv(recorder.calls[0]) + tail = _tail_after_image_tag(argv, "fake-gw:latest-amd64") + assert tail == ["axon-peripheral-sdk", "doctor", "--help"], ( + f"verb-help tail must be forwarded verbatim; got: {tail!r}" + ) + + +# --------------------------------------------------------------------------- +# AC-14 case 13: `peripherals --help` lists exactly {build, deploy, gateware} +# --------------------------------------------------------------------------- + + +def test_case_13_peripherals_help_lists_three_subcommands(peripherals, capsys): + """13: ``peripherals --help`` lists exactly ``build``, ``deploy``, + ``gateware`` as subcommand entries -- no hard-coded SDK verbs. + + Locks the contract that synapsectl does NOT enumerate SDK verbs at + the argparse level; the SDK is the sole source of truth. + """ + parser = _build_root_parser(peripherals) + with pytest.raises(SystemExit) as excinfo: + parser.parse_args(["peripherals", "--help"]) + assert excinfo.value.code == 0 + captured = capsys.readouterr() + help_text = captured.out + + # All three required subcommands must be listed. + assert "build" in help_text, ( + f"peripherals --help must list 'build' subcommand; got: {help_text!r}" + ) + assert "deploy" in help_text, ( + f"peripherals --help must list 'deploy' subcommand; got: {help_text!r}" + ) + assert "gateware" in help_text, ( + f"peripherals --help must list 'gateware' subcommand; got: {help_text!r}" + ) + + # SDK verb names that MUST NOT appear as registered subcommands. We scan + # the help text for these names appearing as standalone tokens followed by + # a description (the argparse subparser-list format puts the verb name + # alone on a line or as the first token of a 2-space-indented line). + forbidden_sdk_verbs = [ + "validate", + "sim", + "regenerate", + "add-peripheral", + "list-profiles", + ] + for verb in forbidden_sdk_verbs: + # Reject only the argparse " " subparser + # entry shape, allowing the verb name to appear inside descriptive + # prose (e.g. "for SDK-side help, run gateware --help"). + lines = help_text.splitlines() + offending = [ + ln + for ln in lines + if ln.startswith(" " + verb + " ") + or ln.strip() == verb + or ln.startswith(" " + verb + "\t") + ] + assert offending == [], ( + f"peripherals --help must not list '{verb}' as a registered " + f"subcommand; got lines: {offending!r}" + ) + + +# --------------------------------------------------------------------------- +# AC-14 case 14: `peripherals nonsense` -> argparse invalid-choice, no docker +# --------------------------------------------------------------------------- + + +def test_case_14_invalid_subcommand_is_argparse_error_no_subprocess( + peripherals, tmp_path, monkeypatch, capsys +): + """14: ``peripherals nonsense`` (NOT build/deploy/gateware) -> argparse + ``SystemExit(2)`` with 'invalid choice' in stderr. subprocess.run NOT + invoked. + + This is the regression test against the rejected Amendment-5 + unknown-verb fall-through design. A future maintainer who re-introduces + blind fall-through MUST break this test. + """ + lic = _make_license_file(tmp_path) + recorder, _ = _install_dispatcher_stubs( + peripherals, monkeypatch, tmp_path, license_value=lic + ) + + parser = _build_root_parser(peripherals) + with pytest.raises(SystemExit) as excinfo: + parser.parse_args(["peripherals", "nonsense"]) + + assert excinfo.value.code == 2, ( + f"invalid subcommand must exit with argparse code 2; " + f"got: {excinfo.value.code!r}" + ) + captured = capsys.readouterr() + err_lower = captured.err.lower() + assert "invalid choice" in err_lower, ( + f"stderr must contain argparse's 'invalid choice' message; " + f"got: {captured.err!r}" + ) + # Strengthening (anti-tautology): the choice list mentioned in the + # error must include ALL THREE registered subcommands (build, deploy, + # gateware). Today the parser only registers {build, deploy} so the + # error reads "(choose from 'build', 'deploy')" which would pass a + # naive `"invalid choice" in err` check tautologically. After AC-13 + # lands, the error must reference 'gateware' alongside the other two. + assert "gateware" in err_lower, ( + f"the invalid-choice error must list 'gateware' as a valid " + f"subcommand (proves AC-13 registered it). Without this the " + f"test would pass tautologically against today's parser which " + f"only knows {{build, deploy}}. got: {captured.err!r}" + ) + assert "build" in err_lower and "deploy" in err_lower, ( + f"the invalid-choice error must also list 'build' and 'deploy'; " + f"got: {captured.err!r}" + ) + # subprocess.run must NOT have been called. + assert recorder.calls == [], ( + f"unknown subcommand must NOT trigger any subprocess.run; " + f"got: {recorder.calls!r}" + ) + + +# --------------------------------------------------------------------------- +# AC-14 case 15: future-verb forwarded verbatim (no known-verb gate) +# --------------------------------------------------------------------------- + + +def test_case_15_future_verb_forwarded_no_gate(peripherals, tmp_path, monkeypatch): + """15: ``gateware future-verb-2027`` -> REMAINDER captures the unknown + verb; subprocess.run IS called with the verb in the docker argv tail. + + This proves the dispatcher does NOT gate on a known-verb list -- a + future SDK release that adds a new verb works against today's + synapsectl without code changes. + """ + lic = _make_license_file(tmp_path) + recorder, _ = _install_dispatcher_stubs( + peripherals, monkeypatch, tmp_path, license_value=lic + ) + + with pytest.raises(SystemExit): + _dispatch(peripherals, ["future-verb-2027"]) + + assert len(recorder.calls) == 1, ( + f"future verb must be forwarded (single docker call); got: {recorder.calls!r}" + ) + argv = _docker_argv(recorder.calls[0]) + tail = _tail_after_image_tag(argv, "fake-gw:latest-amd64") + assert tail == ["axon-peripheral-sdk", "future-verb-2027"], ( + f"future verb must be forwarded verbatim; got tail: {tail!r}" + ) + + +# --------------------------------------------------------------------------- +# AC-14 case 16: POSIX-only -- os.getuid raises AttributeError -> SystemExit +# --------------------------------------------------------------------------- + + +def test_case_16_non_posix_host_exits_no_subprocess( + peripherals, tmp_path, monkeypatch, capsys +): + """16 (AC-13 POSIX-only): ``os.getuid`` raises ``AttributeError`` -> + dispatcher exits non-zero, subprocess.run NOT called. + + Per AC-13 lines 1035-1051 of the plan and AC-14 case 11 of the plan + body, the dispatcher takes the *strict* reading on non-POSIX hosts: + it raises a clear error rather than silently falling back to + 1000:1000. The printed message should reference POSIX (or + Linux/macOS) and point the user at ``axon-peripheral-sdk`` so they + have an actionable next step. + """ + lic = _make_license_file(tmp_path) + recorder, _ = _install_dispatcher_stubs( + peripherals, + monkeypatch, + tmp_path, + license_value=lic, + getuid_raises=AttributeError("module 'os' has no attribute 'getuid'"), + ) + + with pytest.raises(SystemExit) as excinfo: + _dispatch(peripherals, ["doctor"]) + + assert excinfo.value.code not in (0, None), ( + f"non-POSIX host must exit with a non-zero status; " + f"got code: {excinfo.value.code!r}" + ) + assert recorder.calls == [], ( + f"non-POSIX host must NOT invoke subprocess.run; got: {recorder.calls!r}" + ) + + captured = capsys.readouterr() + msg = (captured.out + captured.err + str(excinfo.value)).lower() + # The error should point the user at the right alternative AND mention + # the platform limitation. We accept any of the canonical phrasings. + assert "posix" in msg or "linux" in msg or "macos" in msg, ( + f"non-POSIX error message must mention POSIX / Linux / macOS; " + f"got: {captured.out + captured.err!r} value={excinfo.value!r}" + ) + assert "axon-peripheral-sdk" in msg, ( + f"non-POSIX error message must reference axon-peripheral-sdk as the " + f"alternative invocation path; " + f"got: {captured.out + captured.err!r} value={excinfo.value!r}" + ) diff --git a/synapse/tests/cli/test_gateware_runner.py b/synapse/tests/cli/test_gateware_runner.py new file mode 100644 index 00000000..77c3fbd8 --- /dev/null +++ b/synapse/tests/cli/test_gateware_runner.py @@ -0,0 +1,256 @@ +"""AC-10 / AC-6: unit tests for run_gateware_build(). + +The runner lives in `synapse.cli.gateware` per the plan's File Structure +section. Signature: + + run_gateware_build(peripheral_dir: str, image_tag: str, + env: Mapping[str, str] = os.environ) -> str + +Behavior (per AC-6): + 1. Calls build_license_docker_args(env); LicenseUnsetError propagates. + 2. Issues `docker run --rm --user dev -v :/home/workspace + -w /home/workspace /bin/bash -lc + 'axon-peripheral-sdk build --project src/gateware --pdc devkit --impl impl_1'`. + 3. Non-zero exit -> raises subprocess.CalledProcessError. + 4. After success, globs /src/gateware/build/bitstreams/sdk_*.bit + and returns the newest by mtime (warns on multi-match). + 5. Empty glob -> FileNotFoundError with message mentioning "sdk_*.bit". + +Sub-phase 4.4 (Tester): the xfail marker is removed — these tests now run +as live AC-6 acceptance gates and must fail until the Implementer lands +``run_gateware_build``. +""" + +from __future__ import annotations + +import importlib +import os +import subprocess +import time + +import pytest + + +@pytest.fixture() +def gateware(): + """Lazy import — avoids module-collection failure before AC-5/AC-6 land.""" + return importlib.import_module("synapse.cli.gateware") + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def peripheral_dir(tmp_path): + """Create a minimal peripheral dir with src/gateware/ + a license file.""" + pd = tmp_path / "myplugin" + (pd / "src" / "gateware").mkdir(parents=True) + license_file = tmp_path / "license.dat" + license_file.write_text("FEATURE radiant ...") + return pd, license_file + + +def _bitstreams_dir(peripheral_dir): + bs = os.path.join(str(peripheral_dir), "src", "gateware", "build", "bitstreams") + os.makedirs(bs, exist_ok=True) + return bs + + +# --------------------------------------------------------------------------- +# AC-10 case 10 / 13: docker-run argv shape +# --------------------------------------------------------------------------- + + +def test_runner_builds_docker_run_argv_with_project_flag( + gateware, peripheral_dir, monkeypatch +): + """Case 10/13: captured docker-run argv has the correct shape and ends + with the exact axon-peripheral-sdk invocation (the AC-6 / FINDING-1 + regression: `--project src/gateware --pdc devkit --impl impl_1`). + """ + pd, license_file = peripheral_dir + recorded: list[list[str]] = [] + + def fake_run(argv, *args, **kwargs): + recorded.append(list(argv)) + # Drop a fake .bit so the post-run glob succeeds. + bs = _bitstreams_dir(pd) + bit = os.path.join(bs, "sdk_topbuild.bit") + with open(bit, "w") as fp: + fp.write("bitstream") + return subprocess.CompletedProcess(argv, 0, b"", b"") + + monkeypatch.setattr(gateware.subprocess, "run", fake_run) + + result = gateware.run_gateware_build( + str(pd), + "myplugin-gateware:latest-arm64", + env={"LM_LICENSE_FILE": str(license_file)}, + ) + + assert len(recorded) == 1, "runner should issue exactly one docker run" + argv = recorded[0] + + # Sanity: docker run --rm + assert argv[0:3] == ["docker", "run", "--rm"] + + # The image tag is the second-to-last block before the entrypoint. + assert "myplugin-gateware:latest-arm64" in argv + + # Workdir is /home/workspace (case 13) + assert "-w" in argv + assert argv[argv.index("-w") + 1] == "/home/workspace" + + # Bind-mount the peripheral_dir abspath to /home/workspace. + abs_pd = os.path.abspath(str(pd)) + assert "-v" in argv + # There may be multiple -v (license + workspace); check that the + # workspace bind-mount is present. + v_indices = [i for i, tok in enumerate(argv) if tok == "-v"] + bind_targets = {argv[i + 1] for i in v_indices} + assert f"{abs_pd}:/home/workspace" in bind_targets + + # The SDK command is the final shell -lc payload. + assert "/bin/bash" in argv + bash_idx = argv.index("/bin/bash") + assert argv[bash_idx + 1] == "-lc" + sdk_cmd = argv[bash_idx + 2] + assert ( + sdk_cmd + == "axon-peripheral-sdk build --project src/gateware --pdc devkit --impl impl_1" + ) + + # And the returned path is the .bit we dropped. + assert result.endswith("sdk_topbuild.bit") + + +# --------------------------------------------------------------------------- +# AC-10 case 14: LM_LICENSE_FILE forwarded +# --------------------------------------------------------------------------- + + +def test_runner_forwards_floating_license_arg(gateware, peripheral_dir, monkeypatch): + """Case 14: port@host floating license -> -e LM_LICENSE_FILE= in argv.""" + pd, _ = peripheral_dir + recorded: list[list[str]] = [] + + def fake_run(argv, *args, **kwargs): + recorded.append(list(argv)) + bs = _bitstreams_dir(pd) + with open(os.path.join(bs, "sdk_topbuild.bit"), "w") as fp: + fp.write("x") + return subprocess.CompletedProcess(argv, 0, b"", b"") + + monkeypatch.setattr(gateware.subprocess, "run", fake_run) + + gateware.run_gateware_build( + str(pd), + "myplugin-gateware:latest-arm64", + env={"LM_LICENSE_FILE": "27000@licenseserver"}, + ) + + argv = recorded[0] + e_indices = [i for i, tok in enumerate(argv) if tok == "-e"] + e_pairs = [argv[i + 1] for i in e_indices] + assert "LM_LICENSE_FILE=27000@licenseserver" in e_pairs + + # And no bind-mount for the license (port@host mode is mount-free). + v_indices = [i for i, tok in enumerate(argv) if tok == "-v"] + bind_targets = [argv[i + 1] for i in v_indices] + assert not any("/opt/lattice/license.dat" in t for t in bind_targets) + + +# --------------------------------------------------------------------------- +# AC-10 case 15: unset env -> error, no subprocess.run +# --------------------------------------------------------------------------- + + +def test_runner_raises_when_license_unset_and_does_not_invoke_docker( + gateware, peripheral_dir, monkeypatch +): + """Case 15: LM_LICENSE_FILE unset -> LicenseUnsetError, subprocess.run unused.""" + pd, _ = peripheral_dir + called = [] + + def fake_run(argv, *args, **kwargs): # pragma: no cover - must NOT be called + called.append(argv) + return subprocess.CompletedProcess(argv, 0, b"", b"") + + monkeypatch.setattr(gateware.subprocess, "run", fake_run) + + with pytest.raises(gateware.LicenseUnsetError): + gateware.run_gateware_build( + str(pd), + "myplugin-gateware:latest-arm64", + env={}, + ) + + assert called == [] + + +# --------------------------------------------------------------------------- +# AC-10 case 11: bitstream glob returns newest of many +# --------------------------------------------------------------------------- + + +def test_runner_returns_newest_bit_when_multiple_emitted( + gateware, peripheral_dir, monkeypatch +): + """Case 11: glob with two .bit files of different mtimes -> newest wins.""" + pd, license_file = peripheral_dir + bs = _bitstreams_dir(pd) + older = os.path.join(bs, "sdk_old.bit") + newer = os.path.join(bs, "sdk_new.bit") + with open(older, "w") as fp: + fp.write("old") + time.sleep(0.05) + with open(newer, "w") as fp: + fp.write("new") + # Belt-and-suspenders: force mtimes so the test isn't flaky on + # coarse-granularity filesystems. + os.utime(older, (1_000_000, 1_000_000)) + os.utime(newer, (2_000_000, 2_000_000)) + + def fake_run(argv, *args, **kwargs): + return subprocess.CompletedProcess(argv, 0, b"", b"") + + monkeypatch.setattr(gateware.subprocess, "run", fake_run) + + result = gateware.run_gateware_build( + str(pd), + "myplugin-gateware:latest-arm64", + env={"LM_LICENSE_FILE": str(license_file)}, + ) + + assert os.path.abspath(result) == os.path.abspath(newer) + + +# --------------------------------------------------------------------------- +# AC-10 case 12: no .bit -> clear FileNotFoundError naming the glob +# --------------------------------------------------------------------------- + + +def test_runner_raises_with_glob_in_message_when_no_bit_emitted( + gateware, peripheral_dir, monkeypatch +): + """Case 12: docker run succeeds but no .bit lands -> FileNotFoundError + whose message names the expected glob pattern. + """ + pd, license_file = peripheral_dir + _bitstreams_dir(pd) # exists but empty + + def fake_run(argv, *args, **kwargs): + return subprocess.CompletedProcess(argv, 0, b"", b"") + + monkeypatch.setattr(gateware.subprocess, "run", fake_run) + + with pytest.raises(FileNotFoundError) as excinfo: + gateware.run_gateware_build( + str(pd), + "myplugin-gateware:latest-arm64", + env={"LM_LICENSE_FILE": str(license_file)}, + ) + + assert "sdk_*.bit" in str(excinfo.value) diff --git a/synapse/tests/cli/test_half_selectors.py b/synapse/tests/cli/test_half_selectors.py new file mode 100644 index 00000000..6ca1a017 --- /dev/null +++ b/synapse/tests/cli/test_half_selectors.py @@ -0,0 +1,1009 @@ +"""AC-11 → AC-7 / AC-8 (sub-phase 4.4), AC-12 + AC-15 (sub-phase 4.5). + +Tests the ``driver`` / ``gateware`` / ``both`` target subcommands on +``synapsectl peripherals build`` (AC-7) and ``... peripherals deploy`` +(AC-8), the ``--clean`` × half-selector matrix (AC-7 body), the +combined ``.deb`` layout contract (AC-12), and the +``_expected_bit_filename`` helper / new optional +``install.gateware_target`` manifest field (AC-15). + +The half selectors were originally mutually-exclusive ``--driver`` / +``--gateware`` flags; they are now ``build``/``deploy`` subcommands +(``driver``/``gateware``/``both``). ``half`` still drives the handlers, so +the cases that call ``build_cmd``/``deploy_cmd`` directly are unchanged; the +argparse-surface cases parse the subcommand form. + +Conventions: + * Cases A-H cover ``peripherals build``. + * Cases I-L cover ``peripherals deploy`` (incl. ``--package`` interaction). + * Cases M-O exercise AC-12: the combined / half-flagged ``.deb`` staging + layout. They consume ``_expected_so_filename`` / ``_expected_bit_filename`` + for symmetric naming derived from the manifest. + * Cases P1-P7 exercise AC-15: ``_expected_bit_filename`` fallback chain + (gateware_target -> .so stem -> manifest.name). + +Mocking strategy mirrors the prior Tester (``test_gateware_runner.py``): + * ``synapse.cli.peripherals.subprocess.run`` -> recorder + * ``synapse.cli.peripherals.build_peripheral_so`` -> recorder returning True + * ``synapse.cli.peripherals.build_peripheral_deb`` -> recorder returning True + * ``synapse.cli.peripherals.gateware.run_gateware_build`` -> recorder + returning the path of a fake ``.bit`` created under ``tmp_path`` + * ``synapse.cli.peripherals.deploy_package`` -> recorder + * ``synapse.cli.peripherals.ensure_docker`` -> True + * ``synapse.cli.peripherals.build_docker_image`` -> dict return + * ``synapse.cli.peripherals.find_deb_package`` -> a fake .deb path + +Tests don't need a real Docker daemon, real cmake/vcpkg, or a real Radiant +license — everything that would shell out is monkey-patched. +""" + +from __future__ import annotations + +import argparse +import importlib +import json +import os +import subprocess +from types import SimpleNamespace + +import pytest + + +# --------------------------------------------------------------------------- +# Helpers / fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def peripherals(): + """Lazy-import ``synapse.cli.peripherals``. + + Importing at module-collection time would touch the half-selector code + paths before AC-7/AC-8 land. A fixture defers the import so a clean + ``ImportError`` per test is more informative than a single collection + crash that masks every case. + """ + return importlib.import_module("synapse.cli.peripherals") + + +def _make_peripheral_dir( + tmp_path, + *, + name: str = "intan_rhd2132", + with_gateware: bool = True, + with_install_target: bool = True, + with_gateware_target: bool = False, +): + """Create a fake peripheral directory tree. + + Layout:: + + // + manifest.json + build/aarch64/ (fake .so so build_peripheral_deb finds it) + src/gateware/ (only if with_gateware) + src/gateware/peripheral.yaml (only if with_gateware) + src/gateware/build/SENTINEL_GATEWARE (sentinel for --clean tests) + build/aarch64/SENTINEL_DRIVER (sentinel for --clean tests) + + Returns the absolute path of ``/``. + """ + pd = tmp_path / name + pd.mkdir() + + # Manifest + install: dict = {} + if with_install_target: + install["target"] = f"/usr/lib/scifi/plugins/{name}.so" + if with_gateware_target: + install["gateware_target"] = f"/usr/lib/scifi/gateware/{name}.bit" + manifest = {"name": name, "version": "0.1.0"} + if install: + manifest["install"] = install + (pd / "manifest.json").write_text(json.dumps(manifest)) + + # Driver build dir + sentinel + driver_build = pd / "build" / "aarch64" + driver_build.mkdir(parents=True) + (driver_build / "SENTINEL_DRIVER").write_text("driver") + # Fake .so so build_peripheral_deb's existence check passes if invoked. + (driver_build / f"{name}.so").write_text("fake-so") + + # Gateware dir + sentinel + if with_gateware: + gw_dir = pd / "src" / "gateware" + gw_dir.mkdir(parents=True) + (gw_dir / "peripheral.yaml").write_text("radiant_version: '2024.2'\n") + gw_build = gw_dir / "build" + gw_build.mkdir() + (gw_build / "SENTINEL_GATEWARE").write_text("gateware") + # Also drop a fake .bit under build/bitstreams/ so run_gateware_build's + # stub has somewhere to point. + bs = gw_build / "bitstreams" + bs.mkdir() + (bs / f"sdk_{name}.bit").write_text("fake-bit") + + return pd + + +def _build_root_parser(peripherals): + """Build a fresh root parser wired with `peripherals.add_commands`. + + Avoids relying on `synapse.cli.__main__` (which the conftest stubs out). + Returns the parser; tests call ``parser.parse_args([...])``. + """ + parser = argparse.ArgumentParser(prog="synapsectl") + subparsers = parser.add_subparsers(dest="cmd") + peripherals.add_commands(subparsers) + return parser + + +def _install_common_stubs(peripherals, monkeypatch, tmp_path, *, fake_bit=None): + """Stub out everything that would shell out. + + Returns a ``SimpleNamespace`` with recorders so tests can introspect. + """ + recorders = SimpleNamespace( + build_so_calls=[], + run_gateware_calls=[], + build_deb_calls=[], + subprocess_calls=[], + deploy_calls=[], + ) + + def fake_build_so(*args, **kwargs): + recorders.build_so_calls.append((args, kwargs)) + return True + + def fake_build_deb(*args, **kwargs): + recorders.build_deb_calls.append((args, kwargs)) + return True + + def fake_run_gateware(*args, **kwargs): + recorders.run_gateware_calls.append((args, kwargs)) + if fake_bit is not None: + return str(fake_bit) + # Drop a fake .bit somewhere predictable. + path = tmp_path / "fake.bit" + path.write_text("bit") + return str(path) + + def fake_subprocess_run(argv, *args, **kwargs): + recorders.subprocess_calls.append( + (list(argv) if isinstance(argv, list) else argv, args, kwargs) + ) + return subprocess.CompletedProcess(argv, 0, b"", b"") + + def fake_deploy_package(uri, deb_path): + recorders.deploy_calls.append((uri, deb_path)) + + monkeypatch.setattr(peripherals, "build_peripheral_so", fake_build_so) + monkeypatch.setattr(peripherals, "build_peripheral_deb", fake_build_deb) + monkeypatch.setattr(peripherals, "ensure_docker", lambda: True) + monkeypatch.setattr( + peripherals, + "build_docker_image", + lambda *a, **kw: { + "driver": "fake-driver:latest-arm64", + "gateware": "fake-gateware:latest-arm64", + }, + ) + monkeypatch.setattr(peripherals.subprocess, "run", fake_subprocess_run) + monkeypatch.setattr(peripherals, "deploy_package", fake_deploy_package) + + # find_deb_package is called after build_peripheral_deb succeeds. + monkeypatch.setattr( + peripherals, + "find_deb_package", + lambda dist_dir: os.path.join(dist_dir, "fake_arm64.deb"), + ) + + # gateware sub-module attribute on peripherals must expose run_gateware_build. + # The plan says peripherals.py imports gateware module-level, so we either + # patch synapse.cli.gateware.run_gateware_build OR peripherals.gateware.run_gateware_build. + # Patch the source-of-truth (synapse.cli.gateware) so both names resolve. + gateware_mod = importlib.import_module("synapse.cli.gateware") + monkeypatch.setattr( + gateware_mod, "run_gateware_build", fake_run_gateware, raising=False + ) + # Best-effort: also patch a `peripherals.gateware` attribute if present. + if hasattr(peripherals, "gateware"): + monkeypatch.setattr( + peripherals.gateware, "run_gateware_build", fake_run_gateware, raising=False + ) + + return recorders + + +def _build_args(peripheral_dir, **overrides): + """Construct an args Namespace for build_cmd.""" + ns = SimpleNamespace( + peripheral_dir=str(peripheral_dir), + clean=False, + half="both", + uri=None, + package=None, + ) + for k, v in overrides.items(): + setattr(ns, k, v) + return ns + + +def _deploy_args(peripheral_dir, **overrides): + """Construct an args Namespace for deploy_cmd.""" + ns = SimpleNamespace( + peripheral_dir=str(peripheral_dir), + half="both", + uri=None, + package=None, + ) + for k, v in overrides.items(): + setattr(ns, k, v) + return ns + + +# =========================================================================== +# AC-7: peripherals build flag handling +# =========================================================================== + + +# --- Case A: no flag -> both halves ---------------------------------------- + + +def test_case_A_build_no_flag_runs_both_halves(peripherals, tmp_path, monkeypatch): + """A: ``peripherals build`` with no half flag runs driver AND gateware.""" + pd = _make_peripheral_dir(tmp_path) + recorders = _install_common_stubs(peripherals, monkeypatch, tmp_path) + + peripherals.build_cmd(_build_args(pd, half="both")) + + assert len(recorders.build_so_calls) == 1, "driver builder should run once" + assert len(recorders.run_gateware_calls) == 1, "gateware runner should run once" + assert len(recorders.build_deb_calls) == 1, ".deb staging should run once" + + +# --- Case A2: `both` subcommand parses to half="both" on build and deploy -- + + +def test_case_A2_both_subcommand_parses_to_both(peripherals, tmp_path): + """A2: ``build both`` and ``deploy both`` resolve ``args.half == "both"``. + + The explicit ``both`` target is the subcommand-era replacement for the old + flagless default; it must route to the same handler with ``half="both"``. + """ + pd = _make_peripheral_dir(tmp_path) + parser = _build_root_parser(peripherals) + + build_args = parser.parse_args(["peripherals", "build", "both", str(pd)]) + assert getattr(build_args, "half", None) == "both" + assert build_args.func is peripherals.build_cmd + + deploy_args = parser.parse_args(["peripherals", "deploy", "both", str(pd)]) + assert getattr(deploy_args, "half", None) == "both" + assert deploy_args.func is peripherals.deploy_cmd + + +# --- Case B: build driver -> driver half only ------------------------------ + + +def test_case_B_build_driver_skips_gateware(peripherals, tmp_path, monkeypatch): + """B: ``build driver`` -> ``run_gateware_build`` is NEVER invoked. + + Parses via the real argparse surface: the ``driver`` subcommand must set + ``args.half`` to ``"driver"`` and ``build_cmd`` must branch accordingly. + """ + pd = _make_peripheral_dir(tmp_path) + recorders = _install_common_stubs(peripherals, monkeypatch, tmp_path) + + parser = _build_root_parser(peripherals) + args = parser.parse_args(["peripherals", "build", "driver", str(pd)]) + assert getattr(args, "half", None) == "driver", ( + f"args.half must be 'driver' after the 'driver' subcommand; got: " + f"{getattr(args, 'half', None)!r}" + ) + # Carry over fields build_cmd expects. + args.uri = None + if not hasattr(args, "package"): + args.package = None + peripherals.build_cmd(args) + + assert len(recorders.build_so_calls) == 1 + assert recorders.run_gateware_calls == [], ( + "--driver must not invoke the gateware runner" + ) + assert len(recorders.build_deb_calls) == 1 + + +# --- Case C: --gateware -> gateware half only ------------------------------ + + +def test_case_C_build_gateware_skips_driver(peripherals, tmp_path, monkeypatch): + """C: ``--gateware`` -> cmake/vcpkg path (``build_peripheral_so``) NEVER invoked.""" + pd = _make_peripheral_dir(tmp_path) + recorders = _install_common_stubs(peripherals, monkeypatch, tmp_path) + + peripherals.build_cmd(_build_args(pd, half="gateware")) + + assert recorders.build_so_calls == [], ( + "--gateware must not invoke the driver builder" + ) + assert len(recorders.run_gateware_calls) == 1 + assert len(recorders.build_deb_calls) == 1 + + +# --- Case D: build with an invalid target -> argparse rejects -------------- + + +def test_case_D_build_invalid_target_is_invalid_choice_error( + peripherals, tmp_path, capsys +): + """D: ``build `` -> argparse `SystemExit(2)` + "invalid choice". + + The half selectors are now subcommands, so the old ``--driver --gateware`` + mutex collision is structurally impossible: each half is a distinct verb + and they cannot be combined. The replacement contract is that the only + accepted targets are ``driver`` / ``gateware`` / ``both``; anything else + is argparse's standard invalid-choice error. + """ + parser = _build_root_parser(peripherals) + with pytest.raises(SystemExit) as excinfo: + parser.parse_args(["peripherals", "build", "neither"]) + + assert excinfo.value.code == 2, ( + "argparse must reject an unknown build target with exit code 2" + ) + captured = capsys.readouterr() + err = captured.err.lower() + assert "invalid choice" in err, ( + f"stderr must reference argparse's 'invalid choice'; got: {captured.err!r}" + ) + assert "driver" in err and "gateware" in err and "both" in err, ( + f"stderr should enumerate the valid targets; got: {captured.err!r}" + ) + + +# --- Case D2: bare `build` (no target) prints help, builds nothing ---------- + + +def test_case_D2_build_no_target_prints_help_and_builds_nothing( + peripherals, tmp_path, monkeypatch, capsys +): + """D2: ``build`` with no target -> the parent prints help; no half runs. + + Bare ``build`` resolves to the help-printing default func, so dispatching + it must not invoke either builder. + """ + recorders = _install_common_stubs(peripherals, monkeypatch, tmp_path) + + parser = _build_root_parser(peripherals) + args = parser.parse_args(["peripherals", "build"]) + assert hasattr(args, "func"), "bare `build` must carry a default func" + assert getattr(args, "half", None) is None, ( + "bare `build` must not set a half; a target subcommand is required" + ) + args.func(args) # the help-printing default; must not raise + + assert recorders.build_so_calls == [] + assert recorders.run_gateware_calls == [] + assert recorders.build_deb_calls == [] + captured = capsys.readouterr() + assert "driver" in captured.out and "gateware" in captured.out, ( + f"bare `build` should print help listing the targets; got: {captured.out!r}" + ) + + +# --- Case E: --clean --driver cleans only the driver tree ------------------ + + +def test_case_E_clean_driver_does_not_touch_gateware_tree( + peripherals, tmp_path, monkeypatch +): + """E: ``build driver --clean`` -> driver clean fires, gateware tree untouched. + + Parses via argparse. After the subcommand split: + * ``build_peripheral_so`` is called with ``clean=True`` (the existing + driver-side clean lives inside that helper). + * ``run_gateware_build`` is NOT called (driver-only half). + * No ``subprocess.run`` argv references ``rm -rf src/gateware/build``. + * The gateware sentinel file survives on disk. + """ + pd = _make_peripheral_dir(tmp_path) + gw_sentinel = pd / "src" / "gateware" / "build" / "SENTINEL_GATEWARE" + driver_sentinel = pd / "build" / "aarch64" / "SENTINEL_DRIVER" + assert gw_sentinel.exists() and driver_sentinel.exists() + + recorders = _install_common_stubs(peripherals, monkeypatch, tmp_path) + + parser = _build_root_parser(peripherals) + args = parser.parse_args(["peripherals", "build", "driver", "--clean", str(pd)]) + assert getattr(args, "half", None) == "driver" + assert getattr(args, "clean", False) is True + args.uri = None + if not hasattr(args, "package"): + args.package = None + peripherals.build_cmd(args) + + # Driver builder must be invoked with clean=True. + assert len(recorders.build_so_calls) == 1, "driver builder must run under --driver" + pos_args, kwargs = recorders.build_so_calls[0] + saw_clean = bool(kwargs.get("clean")) or (len(pos_args) >= 4 and bool(pos_args[3])) + assert saw_clean, ( + "build_peripheral_so must receive clean=True under --clean --driver" + ) + + # Gateware runner must not run. + assert recorders.run_gateware_calls == [], ( + "--driver must not invoke the gateware runner" + ) + + # Gateware tree must NOT have been touched -- sentinel survives. + assert gw_sentinel.exists(), "--clean --driver must not touch src/gateware/build/" + + flat_argv = [ + " ".join(map(str, call[0])) + for call in recorders.subprocess_calls + if isinstance(call[0], list) + ] + gateware_clean_calls = [a for a in flat_argv if "rm -rf src/gateware/build" in a] + assert gateware_clean_calls == [], ( + "--clean --driver must NOT issue any gateware-side clean; got: " + f"{gateware_clean_calls!r}" + ) + + +# --- Case F: --clean --gateware cleans only the gateware tree -------------- + + +def test_case_F_clean_gateware_does_not_touch_driver_tree( + peripherals, tmp_path, monkeypatch +): + """F: ``--clean --gateware`` -> gateware clean fires, driver sentinel survives.""" + pd = _make_peripheral_dir(tmp_path) + driver_sentinel = pd / "build" / "aarch64" / "SENTINEL_DRIVER" + gw_sentinel = pd / "src" / "gateware" / "build" / "SENTINEL_GATEWARE" + assert driver_sentinel.exists() and gw_sentinel.exists() + + recorders = _install_common_stubs(peripherals, monkeypatch, tmp_path) + + peripherals.build_cmd(_build_args(pd, half="gateware", clean=True)) + + # Driver tree must not be touched. + assert driver_sentinel.exists(), "--clean --gateware must not touch build/aarch64/" + + flat_argv = [ + " ".join(map(str, call[0])) + for call in recorders.subprocess_calls + if isinstance(call[0], list) + ] + # The driver-side clean lives in build_peripheral_so (which we stubbed), + # so the recorded subprocess.run calls should not include a driver + # `rm -rf build/` invocation. The driver builder being uninvoked is + # already verified by build_so_calls == []; this is the belt. + driver_clean_calls = [ + a + for a in flat_argv + if "rm -rf build" in a and "rm -rf src/gateware/build" not in a + ] + assert recorders.build_so_calls == [], ( + "build_peripheral_so (which owns the driver clean) must not run " + "under --gateware" + ) + assert driver_clean_calls == [], ( + f"--clean --gateware must NOT issue a driver-side clean; got: " + f"{driver_clean_calls!r}" + ) + + +# --- Case G: --clean (no half) cleans both --------------------------------- + + +def test_case_G_clean_no_half_cleans_both(peripherals, tmp_path, monkeypatch): + """G: ``--clean`` alone (no half flag) -> both halves' cleans fire.""" + pd = _make_peripheral_dir(tmp_path) + recorders = _install_common_stubs(peripherals, monkeypatch, tmp_path) + + peripherals.build_cmd(_build_args(pd, half="both", clean=True)) + + # Driver builder must be invoked with clean=True (today's clean + # lives inside build_peripheral_so). + assert len(recorders.build_so_calls) == 1 + _, kwargs = recorders.build_so_calls[0] + pos_args, _ = recorders.build_so_calls[0] + # clean=True may be passed positionally or as a kwarg; check both. + saw_clean = bool(kwargs.get("clean")) or (len(pos_args) >= 4 and bool(pos_args[3])) + assert saw_clean, ( + "build_peripheral_so must receive clean=True under --clean (both halves)" + ) + + # The gateware-side clean must also fire — recorded as a subprocess.run + # call containing the gateware build dir path. + flat_argv = [ + " ".join(map(str, call[0])) + for call in recorders.subprocess_calls + if isinstance(call[0], list) + ] + gateware_clean_calls = [a for a in flat_argv if "rm -rf src/gateware/build" in a] + assert len(gateware_clean_calls) >= 1, ( + f"--clean (no half) must issue a gateware-side clean; got: {flat_argv!r}" + ) + + +# --- Case H: no --clean, no half -> nothing cleaned ------------------------ + + +def test_case_H_no_clean_no_half_cleans_nothing(peripherals, tmp_path, monkeypatch): + """H: ``peripherals build`` (no flags) -> neither half is cleaned. + + Both sentinels survive. The driver builder is invoked with clean=False; + the gateware-side cleaner subprocess.run is not invoked. + """ + pd = _make_peripheral_dir(tmp_path) + driver_sentinel = pd / "build" / "aarch64" / "SENTINEL_DRIVER" + gw_sentinel = pd / "src" / "gateware" / "build" / "SENTINEL_GATEWARE" + + recorders = _install_common_stubs(peripherals, monkeypatch, tmp_path) + + peripherals.build_cmd(_build_args(pd, half="both", clean=False)) + + assert driver_sentinel.exists() and gw_sentinel.exists() + + # build_peripheral_so must NOT receive clean=True. + assert len(recorders.build_so_calls) == 1 + pos_args, kwargs = recorders.build_so_calls[0] + saw_clean = bool(kwargs.get("clean")) or (len(pos_args) >= 4 and bool(pos_args[3])) + assert not saw_clean, ( + "build_peripheral_so must NOT receive clean=True when --clean is absent" + ) + + flat_argv = [ + " ".join(map(str, call[0])) + for call in recorders.subprocess_calls + if isinstance(call[0], list) + ] + gateware_clean_calls = [a for a in flat_argv if "rm -rf src/gateware/build" in a] + assert gateware_clean_calls == [], ( + f"no --clean flag must issue zero gateware-side cleans; got: {flat_argv!r}" + ) + + +# =========================================================================== +# AC-8: peripherals deploy flag handling +# =========================================================================== + + +# --- Case I: deploy --driver -u -------------------------------------- + + +def test_case_I_deploy_driver_only(peripherals, tmp_path, monkeypatch): + """I: ``peripherals deploy driver -u `` -> driver-only build then deploy. + + Parses via argparse: the ``driver`` subcommand under ``deploy`` must set + ``args.half`` to ``"driver"``. + """ + pd = _make_peripheral_dir(tmp_path) + recorders = _install_common_stubs(peripherals, monkeypatch, tmp_path) + + parser = _build_root_parser(peripherals) + args = parser.parse_args(["peripherals", "deploy", "driver", str(pd)]) + assert getattr(args, "half", None) == "driver", ( + f"args.half must be 'driver' after the 'driver' subcommand on deploy; got: " + f"{getattr(args, 'half', None)!r}" + ) + args.uri = "10.0.0.1" + if not hasattr(args, "package"): + args.package = None + peripherals.deploy_cmd(args) + + assert len(recorders.build_so_calls) == 1 + assert recorders.run_gateware_calls == [], ( + "--driver deploy must not invoke the gateware runner" + ) + assert len(recorders.deploy_calls) == 1, "deploy_package must be invoked once" + uri, deb_path = recorders.deploy_calls[0] + assert uri == "10.0.0.1" + assert deb_path.endswith(".deb") + + +# --- Case J: deploy --gateware -u ------------------------------------ + + +def test_case_J_deploy_gateware_only(peripherals, tmp_path, monkeypatch): + """J: ``peripherals deploy --gateware -u `` -> gateware-only build then deploy.""" + pd = _make_peripheral_dir(tmp_path) + recorders = _install_common_stubs(peripherals, monkeypatch, tmp_path) + + peripherals.deploy_cmd(_deploy_args(pd, half="gateware", uri="10.0.0.1")) + + assert recorders.build_so_calls == [], ( + "--gateware deploy must not invoke the driver builder" + ) + assert len(recorders.run_gateware_calls) == 1 + assert len(recorders.deploy_calls) == 1 + uri, deb_path = recorders.deploy_calls[0] + assert uri == "10.0.0.1" + + +# --- Case K: deploy with an invalid target -> argparse rejects ------------- + + +def test_case_K_deploy_invalid_target_is_invalid_choice_error( + peripherals, tmp_path, capsys +): + """K: ``deploy `` -> argparse `SystemExit(2)` + "invalid choice". + + Mirror of case D for ``deploy``: the half selectors are subcommands, so + the old ``--driver --gateware`` mutex is impossible. Only + ``driver`` / ``gateware`` / ``both`` are accepted targets. + """ + parser = _build_root_parser(peripherals) + with pytest.raises(SystemExit) as excinfo: + parser.parse_args(["peripherals", "deploy", "neither"]) + + assert excinfo.value.code == 2 + captured = capsys.readouterr() + err = captured.err.lower() + assert "invalid choice" in err, ( + f"stderr must reference argparse's 'invalid choice'; got: {captured.err!r}" + ) + assert "driver" in err and "gateware" in err and "both" in err + + +# --- Case L: deploy --package .deb --gateware -u --------------- + + +def test_case_L_deploy_package_short_circuit_ignores_half_flag( + peripherals, tmp_path, monkeypatch, capsys +): + """L: ``--package`` short-circuits; the ``--gateware`` flag is acknowledged + but does not redirect the .deb path. A warning naming both flags is + emitted to stdout (rich console).""" + pd = _make_peripheral_dir(tmp_path) + # Pre-build a fake .deb at a path the test will supply via --package. + fake_deb = tmp_path / "prebuilt.deb" + fake_deb.write_text("dpkg-stub") + + recorders = _install_common_stubs(peripherals, monkeypatch, tmp_path) + + peripherals.deploy_cmd( + _deploy_args(pd, half="gateware", uri="10.0.0.1", package=str(fake_deb)) + ) + + # deploy_package called once with the user-supplied .deb path. + assert len(recorders.deploy_calls) == 1 + uri, deb_path = recorders.deploy_calls[0] + assert uri == "10.0.0.1" + assert os.path.abspath(deb_path) == os.path.abspath(str(fake_deb)) + + # No build paths invoked. + assert recorders.build_so_calls == [] + assert recorders.run_gateware_calls == [] + + # The warning must mention `--gateware` (and reference `--package` or the + # fact that the half-selector is being ignored). + captured = capsys.readouterr() + out_lower = (captured.out + captured.err).lower() + assert "--gateware" in out_lower or "gateware" in out_lower + assert ( + "ignore" in out_lower or "--package" in out_lower or "package" in out_lower + ), ( + "the --package short-circuit must emit a warning that the half-selector " + f"is being ignored; got: {captured.out + captured.err!r}" + ) + + +# =========================================================================== +# AC-12: combined .deb composition (no-flag + half-flagged staging layout) +# =========================================================================== + + +# Probe shape: spy on ``tempfile.mkdtemp`` (called from inside +# build_peripheral_deb to create its staging dir), then walk the dir after +# the function returns. AC-12 broadens the signature to +# (peripheral_dir, manifest_dict, *, bit_path=None, include_driver_runtime=True); +# the tests below feed the manifest through ``build_cmd``'s normal codepath, +# so as long as ``build_cmd`` itself is updated to pass the manifest in (also +# AC-12 -- "Both call sites updated"), this exercises the real public surface. + + +def _captured_staging_files(staging_dir): + """Walk ``staging_dir`` and return a sorted list of relative file paths.""" + out: list[str] = [] + for root, _, files in os.walk(staging_dir): + rel_root = os.path.relpath(root, staging_dir) + for f in files: + out.append(os.path.normpath(os.path.join(rel_root, f))) + return sorted(out) + + +def _seed_runtime_libs_under(peripheral_dir): + """Pre-populate libscifi-peripheral-sdk.so* artifacts so the driver-half + extraction step has something to copy into the staging dir. + + AC-12 says ``build_peripheral_deb`` extracts the runtime libs from the + driver Docker image. The implementer is free to either run the real docker + cp under stubs or to look for already-extracted .so files on disk. To + keep the test agnostic to which path the implementation picks, we drop + the libs directly under build/aarch64/ where the driver builder would + normally place them. + """ + libs_dir = os.path.join(str(peripheral_dir), "build", "aarch64") + os.makedirs(libs_dir, exist_ok=True) + for fname in ( + "libscifi-peripheral-sdk.so", + "libscifi-peripheral-sdk.so.0", + "libscifi-peripheral-sdk.so.0.1.0", + ): + p = os.path.join(libs_dir, fname) + with open(p, "w") as fh: + fh.write("fake-runtime-lib") + + +def test_case_M_combined_deb_carries_both_so_and_bit( + peripherals, tmp_path, monkeypatch +): + """M (AC-12): no-flag .deb stages BOTH the .so and the .bit. + + Staging layout per AC-12 Public Interface Contract: + usr/lib/scifi/plugins/.so + usr/lib/scifi/gateware/.bit + usr/lib/libscifi-peripheral-sdk.so* + + Where = splitext(_expected_so_filename(manifest))[0] + and = splitext(_expected_bit_filename(manifest))[0]. + For this manifest, both stems resolve to "intan_rhd2132". + """ + pd = _make_peripheral_dir(tmp_path) + _seed_runtime_libs_under(pd) + fake_bit = tmp_path / "fake.bit" + fake_bit.write_text("BITSTREAM") + + staging_holder: dict = {} + real_mkdtemp = peripherals.tempfile.mkdtemp + + def spy_mkdtemp(*args, **kwargs): + d = real_mkdtemp(*args, **kwargs) + staging_holder["dir"] = d + return d + + # Capture the real build_peripheral_deb BEFORE installing the common stubs + # (which would otherwise replace it with a recorder). + real_deb = peripherals.build_peripheral_deb + monkeypatch.setattr(peripherals.tempfile, "mkdtemp", spy_mkdtemp) + _install_common_stubs(peripherals, monkeypatch, tmp_path, fake_bit=fake_bit) + # Allow the real build_peripheral_deb to run (not the common stub). + monkeypatch.setattr(peripherals, "build_peripheral_deb", real_deb) + # And stub the fpm subprocess call so it's a no-op. + monkeypatch.setattr( + peripherals.subprocess, + "run", + lambda *a, **kw: subprocess.CompletedProcess([], 0, b"", b""), + ) + + peripherals.build_cmd(_build_args(pd, half="both")) + + staging_dir = staging_holder.get("dir") + assert staging_dir is not None, "build_peripheral_deb must have run" + + # Derive expected basenames from the manifest via the public helpers so + # the assertion stays symmetric and survives a manifest swap. + manifest = json.loads((pd / "manifest.json").read_text()) + expected_so = peripherals._expected_so_filename(manifest) + expected_bit = peripherals._expected_bit_filename(manifest) + assert expected_so == "intan_rhd2132.so", ( + f"sanity: helper should derive 'intan_rhd2132.so' for this manifest, " + f"got {expected_so!r}" + ) + assert expected_bit == "intan_rhd2132.bit", ( + f"sanity: helper should derive 'intan_rhd2132.bit' (from .so stem) " + f"for this manifest with no gateware_target; got {expected_bit!r}" + ) + + files = _captured_staging_files(staging_dir) + assert any( + f.endswith(os.path.join("usr/lib/scifi/plugins", expected_so)) for f in files + ), f".so should be staged at usr/lib/scifi/plugins/{expected_so}; got: {files!r}" + assert any( + f.endswith(os.path.join("usr/lib/scifi/gateware", expected_bit)) for f in files + ), f".bit should be staged at usr/lib/scifi/gateware/{expected_bit}; got: {files!r}" + # Runtime libs must also be present in the combined .deb. + assert any("libscifi-peripheral-sdk" in f for f in files), ( + f"combined .deb must carry libscifi-peripheral-sdk runtime; got: {files!r}" + ) + + +def test_case_N_driver_only_deb_carries_so_but_no_bit( + peripherals, tmp_path, monkeypatch +): + """N (AC-12): ``--driver`` .deb has .so + runtime libs, NOT .bit.""" + pd = _make_peripheral_dir(tmp_path) + _seed_runtime_libs_under(pd) + + real_mkdtemp = peripherals.tempfile.mkdtemp + holder: dict = {} + + def spy_mkdtemp(*args, **kwargs): + d = real_mkdtemp(*args, **kwargs) + holder["dir"] = d + return d + + # Capture the real build_peripheral_deb BEFORE installing the common stubs. + real_deb = peripherals.build_peripheral_deb + monkeypatch.setattr(peripherals.tempfile, "mkdtemp", spy_mkdtemp) + _install_common_stubs(peripherals, monkeypatch, tmp_path) + monkeypatch.setattr( + peripherals.subprocess, + "run", + lambda *a, **kw: subprocess.CompletedProcess([], 0, b"", b""), + ) + # Don't stub build_peripheral_deb -- we want the real one to run. + monkeypatch.setattr(peripherals, "build_peripheral_deb", real_deb) + + peripherals.build_cmd(_build_args(pd, half="driver")) + + staging_dir = holder.get("dir") + assert staging_dir is not None + files = _captured_staging_files(staging_dir) + + manifest = json.loads((pd / "manifest.json").read_text()) + expected_so = peripherals._expected_so_filename(manifest) + assert any( + f.endswith(os.path.join("usr/lib/scifi/plugins", expected_so)) for f in files + ), ( + f"--driver .deb should stage .so at usr/lib/scifi/plugins/{expected_so}; got: {files!r}" + ) + assert not any(f.endswith(".bit") for f in files), ( + f"--driver .deb must not carry any .bit; got: {files!r}" + ) + # Runtime libs MUST be present under --driver. + assert any("libscifi-peripheral-sdk" in f for f in files), ( + f"--driver .deb must carry libscifi-peripheral-sdk runtime; got: {files!r}" + ) + + +def test_case_O_gateware_only_deb_carries_bit_but_no_so_no_runtime( + peripherals, tmp_path, monkeypatch +): + """O (AC-12): ``--gateware`` .deb has only the .bit; no .so, no runtime libs.""" + pd = _make_peripheral_dir(tmp_path) + _seed_runtime_libs_under(pd) + fake_bit = tmp_path / "fake.bit" + fake_bit.write_text("BITSTREAM") + + real_mkdtemp = peripherals.tempfile.mkdtemp + holder: dict = {} + + def spy_mkdtemp(*args, **kwargs): + d = real_mkdtemp(*args, **kwargs) + holder["dir"] = d + return d + + # Capture the real build_peripheral_deb BEFORE installing the common stubs. + real_deb = peripherals.build_peripheral_deb + monkeypatch.setattr(peripherals.tempfile, "mkdtemp", spy_mkdtemp) + _install_common_stubs(peripherals, monkeypatch, tmp_path, fake_bit=fake_bit) + monkeypatch.setattr( + peripherals.subprocess, + "run", + lambda *a, **kw: subprocess.CompletedProcess([], 0, b"", b""), + ) + monkeypatch.setattr(peripherals, "build_peripheral_deb", real_deb) + + peripherals.build_cmd(_build_args(pd, half="gateware")) + + staging_dir = holder.get("dir") + assert staging_dir is not None + files = _captured_staging_files(staging_dir) + + manifest = json.loads((pd / "manifest.json").read_text()) + expected_bit = peripherals._expected_bit_filename(manifest) + assert any( + f.endswith(os.path.join("usr/lib/scifi/gateware", expected_bit)) for f in files + ), ( + f"--gateware .deb should stage .bit at usr/lib/scifi/gateware/{expected_bit}; got: {files!r}" + ) + assert not any(f.endswith(".so") for f in files), ( + f"--gateware .deb must not carry any .so; got: {files!r}" + ) + assert not any("libscifi-peripheral-sdk" in f for f in files), ( + f"--gateware .deb must not carry libscifi-peripheral-sdk runtime; " + f"got: {files!r}" + ) + + +# =========================================================================== +# AC-15: _expected_bit_filename helper + install.gateware_target manifest field +# =========================================================================== + + +# Fallback chain per AC-15 + AC-12 helper contract (read in the plan): +# 1. install.gateware_target present and truthy -> basename(gateware_target) +# 2. else if install.target present -> splitext(basename(target))[0] + ".bit" +# 3. else -> ".bit" +# +# An empty-string gateware_target ("") is falsy by the `if target:` guard +# in the contract pseudocode in the plan, so it falls through to step 2/3. +# Cases P1..P7 cover each branch + a few adversarial corners. + + +def test_case_P1_expected_bit_filename_uses_explicit_gateware_target(peripherals): + """P1 (AC-15): install.gateware_target set -> returns its basename verbatim.""" + manifest = { + "name": "scifi-intan-rhd2132", + "install": { + "target": "/usr/lib/scifi/plugins/intan_rhd2132.so", + "gateware_target": "/usr/lib/scifi/gateware/intan_rhd2132.bit", + }, + } + assert peripherals._expected_bit_filename(manifest) == "intan_rhd2132.bit" + + +def test_case_P2_expected_bit_filename_falls_back_to_so_stem(peripherals): + """P2 (AC-15): no gateware_target -> derive from .so stem. + + For the current axon-peripheral-example manifest -- name + "scifi-intan-rhd2132" but install.target = ".../intan_rhd2132.so" -- + the .bit must be "intan_rhd2132.bit" (NOT "scifi-intan-rhd2132.bit"). + This is the symmetry-with-.so rule from the AC-12 rationale. + """ + manifest = { + "name": "scifi-intan-rhd2132", + "install": {"target": "/usr/lib/scifi/plugins/intan_rhd2132.so"}, + } + assert peripherals._expected_bit_filename(manifest) == "intan_rhd2132.bit" + + +def test_case_P3_expected_bit_filename_falls_back_to_manifest_name(peripherals): + """P3 (AC-15): no install.target AND no gateware_target -> ".bit".""" + manifest = {"name": "scifi-intan-rhd2132"} + assert peripherals._expected_bit_filename(manifest) == "scifi-intan-rhd2132.bit" + + +def test_case_P4_expected_bit_filename_empty_install_block(peripherals): + """P4 (AC-15): install = {} (empty block) + name = "foo" -> "foo.bit".""" + manifest = {"name": "foo", "install": {}} + assert peripherals._expected_bit_filename(manifest) == "foo.bit" + + +def test_case_P5_expected_bit_filename_no_install_key(peripherals): + """P5 (AC-15): no install key at all + name = "bar" -> "bar.bit".""" + manifest = {"name": "bar"} + assert peripherals._expected_bit_filename(manifest) == "bar.bit" + + +def test_case_P6_expected_bit_filename_trusts_unusual_basename(peripherals): + """P6 (AC-15): gateware_target with a non-derived basename is honored verbatim. + + The helper does not enforce that the .bit stem match the .so stem; the + user may intentionally pick a different name. Per AC-15: + "no cross-field constraint (the user may, intentionally or not, pick + different stems for the two artifacts)." + """ + manifest = { + "name": "scifi-intan-rhd2132", + "install": { + "target": "/usr/lib/scifi/plugins/intan_rhd2132.so", + "gateware_target": "/usr/lib/scifi/gateware/my_custom_name.bit", + }, + } + assert peripherals._expected_bit_filename(manifest) == "my_custom_name.bit" + + +def test_case_P7_expected_bit_filename_empty_gateware_target_falls_through( + peripherals, +): + """P7 (AC-15, adversarial): empty-string gateware_target is treated as "not set". + + The AC-12 helper contract guards with ``if target:`` -- an empty string is + falsy and we fall through to the .so-stem fallback. This guards against a + user accidentally writing ``"gateware_target": ""`` and getting a + confusing ``"".bit`` artifact. + """ + manifest = { + "name": "scifi-intan-rhd2132", + "install": { + "target": "/usr/lib/scifi/plugins/intan_rhd2132.so", + "gateware_target": "", + }, + } + # Empty-string falls through; .so stem wins. + assert peripherals._expected_bit_filename(manifest) == "intan_rhd2132.bit" diff --git a/synapse/tests/cli/test_license_mode.py b/synapse/tests/cli/test_license_mode.py new file mode 100644 index 00000000..ba3ff2f2 --- /dev/null +++ b/synapse/tests/cli/test_license_mode.py @@ -0,0 +1,268 @@ +"""AC-10 / AC-5: unit tests for the LM_LICENSE_FILE helper. + +The implementation lives in `synapse.cli.gateware` (per the plan's File +Structure section) and exposes: + + build_license_docker_args(env: Mapping[str, str]) -> list[str] + LicenseUnsetError (subclass of RuntimeError) + +Per AC-5: + * **Unset / empty** -> raises ``LicenseUnsetError`` whose ``str()`` mentions + ``LM_LICENSE_FILE``. + * **port@host floating** (regex ``^[^/\\s]+@[^/\\s]+$``) -> returns + ``["-e", f"LM_LICENSE_FILE={value}"]`` — no bind-mount. + * **File path** (anything else) -> ``Path.expanduser().resolve(strict=True)`` + then returns ``["-v", f"{resolved}:/opt/lattice/license.dat:ro", + "-e", "LM_LICENSE_FILE=/opt/lattice/license.dat"]``. + +These tests are written before the implementation exists, so they MUST fail at +import time today (TDD). +""" + +from __future__ import annotations + + +import importlib + +import pytest + + +@pytest.fixture() +def gateware(): + """Lazy-import `synapse.cli.gateware`. + + Defers the import to test-run time so collection succeeds even before + AC-5 lands. Each test individually fails with a clear ImportError if + the module is missing — instead of one opaque collection-time error + that masks every test. + """ + return importlib.import_module("synapse.cli.gateware") + + +# --------------------------------------------------------------------------- +# Path mode +# --------------------------------------------------------------------------- + + +def test_path_mode_absolute_existing_file(gateware, tmp_path, monkeypatch): + """Case 1: LM_LICENSE_FILE = absolute path to existing file -> bind-mount + MAC.""" + monkeypatch.setattr(gateware, "_host_mac_address", lambda: "aa:bb:cc:dd:ee:ff") + license_file = tmp_path / "license.dat" + license_file.write_text("FEATURE radiant ...") + + args = gateware.build_license_docker_args({"LM_LICENSE_FILE": str(license_file)}) + + resolved = str(license_file.resolve()) + assert args == [ + "-v", + f"{resolved}:/opt/lattice/license.dat:ro", + "-e", + "LM_LICENSE_FILE=/opt/lattice/license.dat", + "--mac-address", + "aa:bb:cc:dd:ee:ff", + ] + + +def test_path_mode_nonexistent_file_raises(gateware, tmp_path): + """Case 2: absolute path to non-existent file -> FileNotFoundError (strict resolve).""" + missing = tmp_path / "nope.dat" + with pytest.raises(FileNotFoundError): + gateware.build_license_docker_args({"LM_LICENSE_FILE": str(missing)}) + + +def test_path_with_at_in_directory_segment(gateware, tmp_path, monkeypatch): + """Case 7: path containing '@' (e.g. /home/user@work/license.dat) — the + regex rejects strings with '/', so this falls through to path mode. + """ + monkeypatch.setattr(gateware, "_host_mac_address", lambda: "aa:bb:cc:dd:ee:ff") + dir_with_at = tmp_path / "user@work" + dir_with_at.mkdir() + license_file = dir_with_at / "license.dat" + license_file.write_text("FEATURE radiant ...") + + args = gateware.build_license_docker_args({"LM_LICENSE_FILE": str(license_file)}) + + resolved = str(license_file.resolve()) + assert args == [ + "-v", + f"{resolved}:/opt/lattice/license.dat:ro", + "-e", + "LM_LICENSE_FILE=/opt/lattice/license.dat", + "--mac-address", + "aa:bb:cc:dd:ee:ff", + ] + + +def test_path_mode_expands_tilde(gateware, tmp_path, monkeypatch): + """Case 10: ~ expansion via Path.expanduser().""" + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setattr(gateware, "_host_mac_address", lambda: "aa:bb:cc:dd:ee:ff") + license_file = tmp_path / "license.dat" + license_file.write_text("FEATURE radiant ...") + + args = gateware.build_license_docker_args({"LM_LICENSE_FILE": "~/license.dat"}) + + resolved = str(license_file.resolve()) + assert args == [ + "-v", + f"{resolved}:/opt/lattice/license.dat:ro", + "-e", + "LM_LICENSE_FILE=/opt/lattice/license.dat", + "--mac-address", + "aa:bb:cc:dd:ee:ff", + ] + + +def test_path_mode_skips_mac_when_unavailable(gateware, tmp_path, monkeypatch): + """When uuid.getnode() falls back to a random MAC, _host_mac_address + returns None and the helper must NOT inject --mac-address — passing a + bogus random MAC into docker run is worse than passing nothing. + """ + monkeypatch.setattr(gateware, "_host_mac_address", lambda: None) + license_file = tmp_path / "license.dat" + license_file.write_text("FEATURE radiant ...") + + args = gateware.build_license_docker_args({"LM_LICENSE_FILE": str(license_file)}) + + assert "--mac-address" not in args + resolved = str(license_file.resolve()) + assert args == [ + "-v", + f"{resolved}:/opt/lattice/license.dat:ro", + "-e", + "LM_LICENSE_FILE=/opt/lattice/license.dat", + ] + + +def test_floating_mode_skips_mac_address(gateware, monkeypatch): + """Floating licenses talk to a license server over the network — hostid is + irrelevant. Even when _host_mac_address returns a real MAC, the helper + must not inject --mac-address in floating mode. + """ + monkeypatch.setattr(gateware, "_host_mac_address", lambda: "aa:bb:cc:dd:ee:ff") + args = gateware.build_license_docker_args( + {"LM_LICENSE_FILE": "27000@licenseserver"} + ) + assert "--mac-address" not in args + assert args == ["-e", "LM_LICENSE_FILE=27000@licenseserver"] + + +# --------------------------------------------------------------------------- +# port@host (floating) mode +# --------------------------------------------------------------------------- + + +def test_floating_single_server_named_host(gateware): + """Case 3: single-server port@host with named domain — accepted as floating.""" + args = gateware.build_license_docker_args( + {"LM_LICENSE_FILE": "1710@lic.example.org"} + ) + assert args == ["-e", "LM_LICENSE_FILE=1710@lic.example.org"] + # Critically, no -v flag — port@host mode is bind-mount-free. + assert "-v" not in args + + +def test_floating_port_number_bare_host(gateware): + """Case 4: single-server `27000@licenseserver` (bare hostname).""" + args = gateware.build_license_docker_args( + {"LM_LICENSE_FILE": "27000@licenseserver"} + ) + assert args == ["-e", "LM_LICENSE_FILE=27000@licenseserver"] + assert "-v" not in args + + +def test_floating_multi_server_redundant(gateware): + """Case 5: multi-server FlexLM redundancy `port1@host1:port2@host2`.""" + value = "27000@host1:27000@host2" + args = gateware.build_license_docker_args({"LM_LICENSE_FILE": value}) + assert args == ["-e", f"LM_LICENSE_FILE={value}"] + assert "-v" not in args + + +def test_floating_symbolic_port_name(gateware): + """Case 6: `port_num@host` — port as a symbolic name, not numeric.""" + args = gateware.build_license_docker_args({"LM_LICENSE_FILE": "port_num@host"}) + assert args == ["-e", "LM_LICENSE_FILE=port_num@host"] + assert "-v" not in args + + +# --------------------------------------------------------------------------- +# Unset / empty mode +# --------------------------------------------------------------------------- + + +def test_unset_raises_license_unset_error(gateware): + """Case 8: env missing the key entirely -> LicenseUnsetError.""" + with pytest.raises(gateware.LicenseUnsetError) as excinfo: + gateware.build_license_docker_args({}) + + msg = str(excinfo.value) + assert "LM_LICENSE_FILE" in msg + + +def test_empty_string_raises_license_unset_error(gateware): + """Case 9: empty string is treated identically to unset.""" + with pytest.raises(gateware.LicenseUnsetError) as excinfo: + gateware.build_license_docker_args({"LM_LICENSE_FILE": ""}) + + msg = str(excinfo.value) + assert "LM_LICENSE_FILE" in msg + + +def test_license_unset_error_is_runtime_error(gateware): + """The exception class must subclass RuntimeError per AC-5.""" + assert issubclass(gateware.LicenseUnsetError, RuntimeError) + + +# --------------------------------------------------------------------------- +# Adversarial: whitespace / newline-laden values +# --------------------------------------------------------------------------- + + +def test_whitespace_only_value_does_not_classify_as_floating(gateware, tmp_path): + """Adversarial: ' ' contains \\s so the regex rejects it as floating. + + AC-5 doesn't say what the helper does with a whitespace-only string that's + also not a valid path. The most defensible behavior is that + ``Path(" ").resolve(strict=True)`` raises FileNotFoundError (since no + such file exists in any cwd). We just assert it does NOT return the + floating-mode (-e only) shape, since the regex's \\s class rules out + whitespace. + """ + with pytest.raises(FileNotFoundError): + gateware.build_license_docker_args({"LM_LICENSE_FILE": " "}) + + +def test_value_with_embedded_newline_does_not_classify_as_floating(gateware): + """Adversarial: '27000@host\\n' contains \\s, so the regex rejects floating. + + With no '/' it's also not obviously a path. The strict path resolve will + fail since the literal "27000@host\\n" file doesn't exist. We just lock in + that the helper does NOT silently return floating-mode args (which would + forward an environment variable containing a newline into the container — + a security smell). + """ + with pytest.raises( + (FileNotFoundError, gateware.LicenseUnsetError, ValueError, OSError) + ): + gateware.build_license_docker_args({"LM_LICENSE_FILE": "27000@host\n"}) + + +# --------------------------------------------------------------------------- +# Helper purity: respects injected Mapping (does not read os.environ). +# --------------------------------------------------------------------------- + + +def test_helper_reads_from_passed_mapping_not_process_env( + gateware, monkeypatch, tmp_path +): + """The helper must not fall back to os.environ when the Mapping lacks the key.""" + # Set the process env to a value that, if leaked, would force a path-mode + # resolution attempt against /etc/lattice/license.dat (almost certainly + # absent on the test host) — i.e., a different code path from the empty + # Mapping the test passes in. + monkeypatch.setenv("LM_LICENSE_FILE", "/etc/lattice/license.dat") + + # Pass an empty Mapping; the helper must use that, not os.environ. + with pytest.raises(gateware.LicenseUnsetError): + gateware.build_license_docker_args({}) From 4a71a1e2f58d4c1877ace6a765f0b61cf7c2d747 Mon Sep 17 00:00:00 2001 From: Vishnu Therayil Sasikumar Date: Wed, 3 Jun 2026 14:31:21 -0700 Subject: [PATCH 04/21] feat(peripherals): tell axon-peripheral-sdk it was launched via synapsectl The gateware passthrough forwards 'synapsectl peripherals gateware ' verbatim to 'axon-peripheral-sdk ' inside the container. Export AXON_PERIPHERAL_SDK_FRONTEND='synapsectl peripherals gateware' on that docker run so the SDK brands its 'next steps' hints and --help examples with the command the user actually typed, not the forwarded binary name. Added a test asserting the -e marker is present, precedes the image tag, and does not leak into the forwarded SDK argv tail. --- synapse/cli/gateware.py | 6 +++ .../tests/cli/test_gateware_passthrough.py | 46 +++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/synapse/cli/gateware.py b/synapse/cli/gateware.py index 26e8241f..efbeab69 100644 --- a/synapse/cli/gateware.py +++ b/synapse/cli/gateware.py @@ -212,6 +212,12 @@ def _gateware_passthrough( "/home/workspace", "--user", f"{host_uid}:{host_gid}", + # Tell the SDK which frontend launched it so its user-facing "next + # steps" hints and --help examples name `synapsectl peripherals + # gateware ` (what the user actually typed) rather than the + # `axon-peripheral-sdk ` binary we forward to inside the container. + "-e", + "AXON_PERIPHERAL_SDK_FRONTEND=synapsectl peripherals gateware", *license_args, gateware_image_tag, "axon-peripheral-sdk", diff --git a/synapse/tests/cli/test_gateware_passthrough.py b/synapse/tests/cli/test_gateware_passthrough.py index 2cb3ec6d..9cfba11d 100644 --- a/synapse/tests/cli/test_gateware_passthrough.py +++ b/synapse/tests/cli/test_gateware_passthrough.py @@ -762,6 +762,52 @@ def test_case_15_future_verb_forwarded_no_gate(peripherals, tmp_path, monkeypatc ) +# --------------------------------------------------------------------------- +# Frontend marker: the SDK is told which CLI launched it so its user-facing +# "next steps" hints / --help examples name `synapsectl peripherals gateware`. +# --------------------------------------------------------------------------- + + +def test_frontend_env_marker_forwarded_to_sdk(peripherals, tmp_path, monkeypatch): + """The dispatcher must pass ``-e AXON_PERIPHERAL_SDK_FRONTEND=synapsectl + peripherals gateware`` so the SDK brands its hints/help with the frontend + prefix the user actually typed (rather than the forwarded binary name). + + The marker must precede the gateware image tag (it's a ``docker run`` flag, + not an SDK arg) and must NOT leak into the verbatim SDK argv tail. + """ + lic = _make_license_file(tmp_path) + recorder, _ = _install_dispatcher_stubs( + peripherals, monkeypatch, tmp_path, license_value=lic + ) + + with pytest.raises(SystemExit): + _dispatch(peripherals, ["new", "myproj"]) + + assert len(recorder.calls) == 1, ( + f"exactly one docker-run subprocess call expected; got: {recorder.calls!r}" + ) + argv = _docker_argv(recorder.calls[0]) + marker = "AXON_PERIPHERAL_SDK_FRONTEND=synapsectl peripherals gateware" + assert marker in argv, ( + f"docker argv must export the frontend marker {marker!r}; got: {argv!r}" + ) + marker_idx = argv.index(marker) + assert argv[marker_idx - 1] == "-e", ( + f"frontend marker must be introduced by '-e'; got argv[{marker_idx - 1}]: " + f"{argv[marker_idx - 1]!r}" + ) + # It is a docker flag, not an SDK arg: must sit before the image tag and + # never appear in the forwarded SDK tail. + assert marker_idx < argv.index("fake-gw:latest-amd64"), ( + "frontend marker must precede the image tag (it's a docker-run flag)" + ) + tail = _tail_after_image_tag(argv, "fake-gw:latest-amd64") + assert tail == ["axon-peripheral-sdk", "new", "myproj"], ( + f"SDK argv tail must stay verbatim (no marker leak); got: {tail!r}" + ) + + # --------------------------------------------------------------------------- # AC-14 case 16: POSIX-only -- os.getuid raises AttributeError -> SystemExit # --------------------------------------------------------------------------- From 16f68194b1b12a825fd2a16b2be3d1fce286f952 Mon Sep 17 00:00:00 2001 From: Vishnu Therayil Sasikumar Date: Wed, 3 Jun 2026 16:13:47 -0700 Subject: [PATCH 05/21] fix(peripherals): drop --pdc/--impl from the gateware build command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The axon-peripheral-sdk removed the --pdc and --impl build options. Drop them from _SDK_BUILD_CMD so 'synapsectl peripherals build gateware' keeps working against the updated SDK. The flags were optional (defaulting to devkit/impl_1), so 'build --project src/gateware' is equivalent and also works against the prior SDK — backward-compatible. Updated the runner test's pinned command string and docstrings to match. --- synapse/cli/gateware.py | 4 +--- synapse/tests/cli/test_gateware_runner.py | 11 ++++------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/synapse/cli/gateware.py b/synapse/cli/gateware.py index efbeab69..d73c58ca 100644 --- a/synapse/cli/gateware.py +++ b/synapse/cli/gateware.py @@ -115,9 +115,7 @@ def build_license_docker_args( return args -_SDK_BUILD_CMD = ( - "axon-peripheral-sdk build --project src/gateware --pdc devkit --impl impl_1" -) +_SDK_BUILD_CMD = "axon-peripheral-sdk build --project src/gateware" def run_gateware_build( diff --git a/synapse/tests/cli/test_gateware_runner.py b/synapse/tests/cli/test_gateware_runner.py index 77c3fbd8..35396092 100644 --- a/synapse/tests/cli/test_gateware_runner.py +++ b/synapse/tests/cli/test_gateware_runner.py @@ -10,7 +10,7 @@ 1. Calls build_license_docker_args(env); LicenseUnsetError propagates. 2. Issues `docker run --rm --user dev -v :/home/workspace -w /home/workspace /bin/bash -lc - 'axon-peripheral-sdk build --project src/gateware --pdc devkit --impl impl_1'`. + 'axon-peripheral-sdk build --project src/gateware'`. 3. Non-zero exit -> raises subprocess.CalledProcessError. 4. After success, globs /src/gateware/build/bitstreams/sdk_*.bit and returns the newest by mtime (warns on multi-match). @@ -67,8 +67,8 @@ def test_runner_builds_docker_run_argv_with_project_flag( gateware, peripheral_dir, monkeypatch ): """Case 10/13: captured docker-run argv has the correct shape and ends - with the exact axon-peripheral-sdk invocation (the AC-6 / FINDING-1 - regression: `--project src/gateware --pdc devkit --impl impl_1`). + with the exact axon-peripheral-sdk invocation (`--project src/gateware`; + the SDK no longer accepts `--pdc`/`--impl`). """ pd, license_file = peripheral_dir recorded: list[list[str]] = [] @@ -117,10 +117,7 @@ def fake_run(argv, *args, **kwargs): bash_idx = argv.index("/bin/bash") assert argv[bash_idx + 1] == "-lc" sdk_cmd = argv[bash_idx + 2] - assert ( - sdk_cmd - == "axon-peripheral-sdk build --project src/gateware --pdc devkit --impl impl_1" - ) + assert sdk_cmd == "axon-peripheral-sdk build --project src/gateware" # And the returned path is the .bit we dropped. assert result.endswith("sdk_topbuild.bit") From 55e8b3836c67d47be47adbebe4fcb22ad39f47b9 Mon Sep 17 00:00:00 2001 From: Vishnu Therayil Sasikumar Date: Mon, 8 Jun 2026 19:24:04 -0700 Subject: [PATCH 06/21] fix(peripherals): resolve gateware project from src/gateware in pass-through `synapsectl peripherals gateware ` run from a peripheral repo root failed for project-scoped SDK verbs (e.g. `generate`: "peripheral.yaml not found in /home/workspace"). The pass-through mounted the repo root at /home/workspace and ran the SDK there, but the SDK resolves its project from cwd, and peripheral.yaml lives in src/gateware/. Injecting `--project src/gateware` is not viable: validate/regenerate/ add-peripheral have no --project flag at all, and doctor/list-profiles/new reject it. Instead, when cwd has manifest.json AND src/gateware/, set the container working dir to /home/workspace/src/gateware. The bind-mount stays the repo root so `build` keeps full-repo visibility. The decision is directory-driven, not verb-driven, so argv is still forwarded verbatim with no verb allowlist; doctor/list-profiles/new are unaffected. Extract a shared _GATEWARE_PROJECT_SUBDIR constant (also used by the structured build command). Tests: 3 new pass-through cases (redirect when manifest+subdir present; stays root when src/gateware absent; stays root when manifest.json absent). --- synapse/cli/gateware.py | 28 +++++- .../tests/cli/test_gateware_passthrough.py | 88 +++++++++++++++++++ 2 files changed, 113 insertions(+), 3 deletions(-) diff --git a/synapse/cli/gateware.py b/synapse/cli/gateware.py index d73c58ca..829e7bab 100644 --- a/synapse/cli/gateware.py +++ b/synapse/cli/gateware.py @@ -115,7 +115,12 @@ def build_license_docker_args( return args -_SDK_BUILD_CMD = "axon-peripheral-sdk build --project src/gateware" +# The gateware project lives in this subdir of a peripheral repo, by +# convention shared with the structured build below and the pass-through's +# implicit-project workdir redirect. +_GATEWARE_PROJECT_SUBDIR = "src/gateware" + +_SDK_BUILD_CMD = f"axon-peripheral-sdk build --project {_GATEWARE_PROJECT_SUBDIR}" def run_gateware_build( @@ -200,14 +205,31 @@ def _gateware_passthrough( except AttributeError: sys.exit(_NON_POSIX_MSG) + abs_peripheral_dir = os.path.abspath(peripheral_dir) + + # When invoked from a peripheral project root (manifest.json present) that + # has a gateware subproject, run the SDK with its cwd inside src/gateware so + # every project-scoped verb resolves peripheral.yaml from its cwd default -- + # including verbs with no --project flag (validate/regenerate/add-peripheral). + # The bind-mount stays the repo root, so `build` still sees the whole repo. + # The verb is never inspected: this is purely a directory-driven decision, + # so the pass-through keeps forwarding argv verbatim with no verb allowlist. + workdir = "/home/workspace" + if os.path.isfile( + os.path.join(abs_peripheral_dir, "manifest.json") + ) and os.path.isdir( + os.path.join(abs_peripheral_dir, *_GATEWARE_PROJECT_SUBDIR.split("/")) + ): + workdir = f"/home/workspace/{_GATEWARE_PROJECT_SUBDIR}" + cmd = [ "docker", "run", "--rm", "-v", - f"{os.path.abspath(peripheral_dir)}:/home/workspace", + f"{abs_peripheral_dir}:/home/workspace", "-w", - "/home/workspace", + workdir, "--user", f"{host_uid}:{host_gid}", # Tell the SDK which frontend launched it so its user-facing "next diff --git a/synapse/tests/cli/test_gateware_passthrough.py b/synapse/tests/cli/test_gateware_passthrough.py index 9cfba11d..f41aaaef 100644 --- a/synapse/tests/cli/test_gateware_passthrough.py +++ b/synapse/tests/cli/test_gateware_passthrough.py @@ -376,6 +376,94 @@ def test_case_6_working_dir_is_home_workspace(peripherals, tmp_path, monkeypatch ) +# --------------------------------------------------------------------------- +# Implicit gateware project: workdir redirects to src/gateware when the +# pass-through is invoked from a peripheral project root (manifest.json present). +# The repo root stays bind-mounted at /home/workspace so `build` still sees the +# whole repo; only the container's working directory moves into src/gateware so +# every project-scoped SDK verb resolves peripheral.yaml from its cwd default. +# --------------------------------------------------------------------------- + + +def test_workdir_redirects_to_gateware_when_manifest_and_subdir_present( + peripherals, tmp_path, monkeypatch +): + """manifest.json + src/gateware/ present -> ``-w /home/workspace/src/gateware`` + while the bind-mount stays the repo root.""" + lic = _make_license_file(tmp_path) + recorder, pd = _install_dispatcher_stubs( + peripherals, monkeypatch, tmp_path, license_value=lic + ) + (pd / "manifest.json").write_text('{"name": "x"}') + (pd / "src" / "gateware").mkdir(parents=True) + + # `validate` has no --project flag; it must find the project via cwd. + with pytest.raises(SystemExit): + _dispatch(peripherals, ["validate"]) + + argv = _docker_argv(recorder.calls[0]) + w_idx = argv.index("-w") + assert argv[w_idx + 1] == "/home/workspace/src/gateware", ( + f"-w must redirect into src/gateware; got: {argv[w_idx + 1]!r}" + ) + # Mount is still the repo root, so `build` keeps full-repo visibility. + assert f"{os.path.abspath(str(pd))}:/home/workspace" in argv, ( + f"bind-mount must stay the repo root; got: {argv!r}" + ) + # argv is still forwarded verbatim -- no injected --project. + tail = _tail_after_image_tag(argv, "fake-gw:latest-amd64") + assert tail == ["axon-peripheral-sdk", "validate"], ( + f"argv must stay verbatim (no --project injected); got: {tail!r}" + ) + + +def test_workdir_stays_root_when_manifest_present_but_no_gateware_subdir( + peripherals, tmp_path, monkeypatch +): + """manifest.json present but no src/gateware/ -> ``-w /home/workspace``. + + A driver-only peripheral has nowhere to redirect; keep the root workdir. + """ + lic = _make_license_file(tmp_path) + recorder, pd = _install_dispatcher_stubs( + peripherals, monkeypatch, tmp_path, license_value=lic + ) + (pd / "manifest.json").write_text('{"name": "x"}') + + with pytest.raises(SystemExit): + _dispatch(peripherals, ["doctor"]) + + argv = _docker_argv(recorder.calls[0]) + w_idx = argv.index("-w") + assert argv[w_idx + 1] == "/home/workspace", ( + f"-w must stay /home/workspace without src/gateware; got: {argv[w_idx + 1]!r}" + ) + + +def test_workdir_stays_root_when_no_manifest_even_if_gateware_subdir_present( + peripherals, tmp_path, monkeypatch +): + """No manifest.json (e.g. cwd already inside the project) -> ``-w /home/workspace``. + + The redirect is keyed on manifest.json so running from inside src/gateware + (which has peripheral.yaml but no manifest.json) is left untouched. + """ + lic = _make_license_file(tmp_path) + recorder, pd = _install_dispatcher_stubs( + peripherals, monkeypatch, tmp_path, license_value=lic + ) + (pd / "src" / "gateware").mkdir(parents=True) + + with pytest.raises(SystemExit): + _dispatch(peripherals, ["regenerate"]) + + argv = _docker_argv(recorder.calls[0]) + w_idx = argv.index("-w") + assert argv[w_idx + 1] == "/home/workspace", ( + f"-w must stay /home/workspace without manifest.json; got: {argv[w_idx + 1]!r}" + ) + + # --------------------------------------------------------------------------- # AC-14 case 7: subprocess.run called with a list, shell=False # --------------------------------------------------------------------------- From bf38e30b903033ab2f1eadb4a33504b2abcb9b62 Mon Sep 17 00:00:00 2001 From: Vishnu Therayil Sasikumar Date: Mon, 8 Jun 2026 22:00:30 -0700 Subject: [PATCH 07/21] feat(peripherals): improve gateware pass-through UX (leading opts, colors, license-optional) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Several refinements to `synapsectl peripherals gateware `: - Leading SDK options now forwarded. argparse.REMAINDER only captures from the first positional, so `gateware --install-completion` (a top-level SDK flag) was rejected. parse_args_with_passthrough() folds leftover tokens into the pass-through argv (gated by a `_passthrough_extra` marker on the subparser); other commands still hard-error on unknown args. __main__ uses it. - Colors preserved. Allocate a docker `-t` when stdout is a tty so the SDK's rich/typer output keeps its colors; omitted when piped/CI. - License made optional for the pass-through. Forward LM_LICENSE_FILE when set, omit when unset; a set-but-missing file path (FileNotFoundError) warns and continues instead of crashing. Only Radiant verbs (`build`) need a license, enforced SDK-side — so help/doctor/generate/validate/sim work without one. - Tests renamed from test_case__* to descriptive names; added coverage for leading options, tty/-t, and the unset / set-but-missing license paths. --- synapse/cli/__main__.py | 2 +- synapse/cli/gateware.py | 12 + synapse/cli/peripherals.py | 56 +++- .../tests/cli/test_gateware_passthrough.py | 266 ++++++++++++------ 4 files changed, 244 insertions(+), 92 deletions(-) diff --git a/synapse/cli/__main__.py b/synapse/cli/__main__.py index 2fbce520..ccd89f1e 100755 --- a/synapse/cli/__main__.py +++ b/synapse/cli/__main__.py @@ -83,7 +83,7 @@ def main(): peripherals.add_commands(subparsers) settings.add_commands(subparsers) deploy_model.add_commands(subparsers) - args = parser.parse_args() + args = peripherals.parse_args_with_passthrough(parser) # If we need to setup the device URI, do that now args = setup_device_uri(args) diff --git a/synapse/cli/gateware.py b/synapse/cli/gateware.py index 829e7bab..95a2bf88 100644 --- a/synapse/cli/gateware.py +++ b/synapse/cli/gateware.py @@ -179,6 +179,11 @@ def run_gateware_build( return chosen +def _stdout_is_tty() -> bool: + """Whether our stdout is a terminal (indirection kept for monkeypatching).""" + return sys.stdout.isatty() + + def _gateware_passthrough( argv: Sequence[str], peripheral_dir: str, @@ -222,10 +227,17 @@ def _gateware_passthrough( ): workdir = f"/home/workspace/{_GATEWARE_PROJECT_SUBDIR}" + # Allocate a pseudo-TTY when our own stdout is a terminal so the SDK's + # rich/typer output keeps its colors (inside a plain `docker run` pipe the + # SDK sees a non-tty and strips them). Guarded on isatty so piped/redirected + # output stays clean and CI never gets a TTY it can't attach. + tty_flag = ["-t"] if _stdout_is_tty() else [] + cmd = [ "docker", "run", "--rm", + *tty_flag, "-v", f"{abs_peripheral_dir}:/home/workspace", "-w", diff --git a/synapse/cli/peripherals.py b/synapse/cli/peripherals.py index 0155ee43..8f5aeda6 100644 --- a/synapse/cli/peripherals.py +++ b/synapse/cli/peripherals.py @@ -135,6 +135,12 @@ def add_commands(subparsers: argparse._SubParsersAction): # truth for verbs and flags; synapsectl does NOT gate on a known-verb # list. peripheral_dir is intentionally NOT a positional here -- REMAINDER # would swallow it -- the dispatcher uses os.getcwd() instead. + # + # REMAINDER only starts capturing at the first positional, so a LEADING + # option (e.g. `gateware --install-completion`, a top-level SDK flag) is + # otherwise rejected by argparse before REMAINDER engages. The + # `_passthrough_extra` marker tells parse_args_with_passthrough to fold any + # such leftover tokens into `argv` instead of erroring -- see that helper. gateware_parser = peripherals_subparsers.add_parser( "gateware", help="Pass arguments through to axon-peripheral-sdk inside the gateware container.", @@ -149,7 +155,28 @@ def add_commands(subparsers: argparse._SubParsersAction): nargs=argparse.REMAINDER, help="SDK verb and its arguments (forwarded verbatim).", ) - gateware_parser.set_defaults(func=gateware_cmd) + gateware_parser.set_defaults(func=gateware_cmd, _passthrough_extra=True) + + +def parse_args_with_passthrough(parser: argparse.ArgumentParser, argv=None): + """Parse CLI args, folding leftover tokens into a pass-through command's argv. + + Plain ``parser.parse_args`` rejects a leading option after + ``peripherals gateware`` (e.g. ``--install-completion``) because + ``argparse.REMAINDER`` only captures from the first positional onward. We + parse with ``parse_known_args`` instead and, when the selected command is + flagged ``_passthrough_extra`` (the gateware dispatcher), append the + leftover tokens to its ``argv`` so they reach the SDK verbatim. For every + other command, leftovers remain a hard error -- preserving argparse's usual + ``unrecognized arguments`` behavior and typo-catching. + """ + args, extra = parser.parse_known_args(argv) + if extra: + if getattr(args, "_passthrough_extra", False): + args.argv = list(getattr(args, "argv", None) or []) + list(extra) + else: + parser.error("unrecognized arguments: " + " ".join(extra)) + return args # --------------------------------------------------------------------------- @@ -802,8 +829,9 @@ def gateware_cmd(args) -> None: Order of operations (mirrors the plan's AC-13 spec): - 1. Resolve LM_LICENSE_FILE -> docker flags. Unset license short-circuits - before any docker invocation. + 1. Resolve LM_LICENSE_FILE -> docker flags. Forwarded when set; when unset + the SDK runs WITHOUT license args (only Radiant verbs like `build` need + a license, and the SDK enforces that itself) -- no short-circuit here. 2. Resolve the peripheral dir to ``os.getcwd()``. REMAINDER captures every token after ``gateware``, so a positional ``peripheral_dir`` cannot coexist with the pass-through; cwd is the only sensible default. @@ -813,11 +841,27 @@ def gateware_cmd(args) -> None: 5. Delegate to :func:`gateware._gateware_passthrough` and ``sys.exit`` on its return code. """ + # Forward the Radiant license when set, but do NOT require it: the + # pass-through must stay usable for verbs that don't touch Radiant + # (help/doctor/list-profiles/generate/validate/sim). Only `build` runs + # Radiant, and the SDK's build preflight owns the "license required" error. + # + # Unset -> run with no license args, silently. Set-but-bad (a file path + # that doesn't exist makes build_license_docker_args raise FileNotFoundError + # from Path.resolve(strict=True)) -> warn so the misconfig isn't masked, but + # still omit the args and continue, so non-Radiant verbs work and `build` + # fails SDK-side with clear guidance. try: license_args = gateware.build_license_docker_args(os.environ) - except LicenseUnsetError as exc: - console.print(f"[bold red]Error:[/bold red] {exc}") - sys.exit(1) + except LicenseUnsetError: + license_args = [] + except FileNotFoundError as exc: + console.print( + f"[yellow]Warning:[/yellow] LM_LICENSE_FILE is set but its license " + f"file was not found ({exc}); continuing without a license. Radiant " + f"commands (e.g. `build`) will fail until it points at a real file." + ) + license_args = [] peripheral_dir = os.path.abspath(os.getcwd()) diff --git a/synapse/tests/cli/test_gateware_passthrough.py b/synapse/tests/cli/test_gateware_passthrough.py index f41aaaef..1b1b11bd 100644 --- a/synapse/tests/cli/test_gateware_passthrough.py +++ b/synapse/tests/cli/test_gateware_passthrough.py @@ -170,9 +170,15 @@ def _dispatch(peripherals, argv_tail): The dispatcher handler always ends in ``sys.exit``; the caller is responsible for wrapping in ``pytest.raises(SystemExit)``. + + Routes through ``parse_args_with_passthrough`` (the real ``main()`` parse + path) rather than ``parser.parse_args`` so leading SDK options like + ``--install-completion`` are folded into ``argv`` instead of rejected. """ parser = _build_root_parser(peripherals) - args = parser.parse_args(["peripherals", "gateware", *argv_tail]) + args = peripherals.parse_args_with_passthrough( + parser, ["peripherals", "gateware", *argv_tail] + ) args.func(args) @@ -202,10 +208,8 @@ def _tail_after_image_tag(argv, image_tag): # --------------------------------------------------------------------------- -def test_case_1_no_arg_verb_doctor_forwarded_verbatim( - peripherals, tmp_path, monkeypatch -): - """1: ``gateware doctor`` -> argv tail is exactly +def test_no_arg_verb_doctor_forwarded_verbatim(peripherals, tmp_path, monkeypatch): + """``gateware doctor`` -> argv tail is exactly ``["axon-peripheral-sdk", "doctor"]``.""" lic = _make_license_file(tmp_path) recorder, _pd = _install_dispatcher_stubs( @@ -237,10 +241,10 @@ def test_case_1_no_arg_verb_doctor_forwarded_verbatim( # --------------------------------------------------------------------------- -def test_case_2_long_flag_value_validate_forwarded_verbatim( +def test_long_flag_value_validate_forwarded_verbatim( peripherals, tmp_path, monkeypatch ): - """2: ``gateware validate --project src/gateware`` -> tail is exactly + """``gateware validate --project src/gateware`` -> tail is exactly ``["axon-peripheral-sdk", "validate", "--project", "src/gateware"]``.""" lic = _make_license_file(tmp_path) recorder, _ = _install_dispatcher_stubs( @@ -266,10 +270,8 @@ def test_case_2_long_flag_value_validate_forwarded_verbatim( # --------------------------------------------------------------------------- -def test_case_3_short_flag_with_double_colon_preserved( - peripherals, tmp_path, monkeypatch -): - """3: ``gateware sim -k some::test_id`` -> the ``::`` is preserved +def test_short_flag_with_double_colon_preserved(peripherals, tmp_path, monkeypatch): + """``gateware sim -k some::test_id`` -> the ``::`` is preserved byte-for-byte in argv form. Under shell-string concatenation the ``::`` might survive too, but a @@ -303,8 +305,8 @@ def test_case_3_short_flag_with_double_colon_preserved( # --------------------------------------------------------------------------- -def test_case_4_user_flag_matches_patched_uid_gid(peripherals, tmp_path, monkeypatch): - """4: ``--user :`` is built from os.getuid()/os.getgid().""" +def test_user_flag_matches_patched_uid_gid(peripherals, tmp_path, monkeypatch): + """``--user :`` is built from os.getuid()/os.getgid().""" lic = _make_license_file(tmp_path) recorder, _ = _install_dispatcher_stubs( peripherals, monkeypatch, tmp_path, license_value=lic, uid=4242, gid=8484 @@ -327,10 +329,8 @@ def test_case_4_user_flag_matches_patched_uid_gid(peripherals, tmp_path, monkeyp # --------------------------------------------------------------------------- -def test_case_5_bind_mount_is_peripheral_dir_abspath( - peripherals, tmp_path, monkeypatch -): - """5: ``-v :/home/workspace`` is present.""" +def test_bind_mount_is_peripheral_dir_abspath(peripherals, tmp_path, monkeypatch): + """``-v :/home/workspace`` is present.""" lic = _make_license_file(tmp_path) recorder, pd = _install_dispatcher_stubs( peripherals, monkeypatch, tmp_path, license_value=lic @@ -358,8 +358,8 @@ def test_case_5_bind_mount_is_peripheral_dir_abspath( # --------------------------------------------------------------------------- -def test_case_6_working_dir_is_home_workspace(peripherals, tmp_path, monkeypatch): - """6: ``-w /home/workspace`` is present in the docker argv.""" +def test_working_dir_is_home_workspace(peripherals, tmp_path, monkeypatch): + """``-w /home/workspace`` is present in the docker argv.""" lic = _make_license_file(tmp_path) recorder, _ = _install_dispatcher_stubs( peripherals, monkeypatch, tmp_path, license_value=lic @@ -469,10 +469,8 @@ def test_workdir_stays_root_when_no_manifest_even_if_gateware_subdir_present( # --------------------------------------------------------------------------- -def test_case_7_subprocess_run_is_argv_list_no_shell( - peripherals, tmp_path, monkeypatch -): - """7: ``subprocess.run`` first arg is a list; ``shell`` is not True.""" +def test_subprocess_run_is_argv_list_no_shell(peripherals, tmp_path, monkeypatch): + """``subprocess.run`` first arg is a list; ``shell`` is not True.""" lic = _make_license_file(tmp_path) recorder, _ = _install_dispatcher_stubs( peripherals, monkeypatch, tmp_path, license_value=lic @@ -498,8 +496,8 @@ def test_case_7_subprocess_run_is_argv_list_no_shell( # --------------------------------------------------------------------------- -def test_case_8_floating_license_emits_env_no_bind(peripherals, tmp_path, monkeypatch): - """8: ``LM_LICENSE_FILE=27000@licenseserver`` -> only ``-e`` is added. +def test_floating_license_emits_env_no_bind(peripherals, tmp_path, monkeypatch): + """``LM_LICENSE_FILE=27000@licenseserver`` -> only ``-e`` is added. No license-file ``-v`` bind-mount; the env var is forwarded as-is. """ @@ -534,10 +532,8 @@ def test_case_8_floating_license_emits_env_no_bind(peripherals, tmp_path, monkey # --------------------------------------------------------------------------- -def test_case_9_file_path_license_emits_bind_and_env( - peripherals, tmp_path, monkeypatch -): - """9: ``LM_LICENSE_FILE=`` -> ``-v :/opt/lattice/license.dat:ro`` +def test_file_path_license_emits_bind_and_env(peripherals, tmp_path, monkeypatch): + """``LM_LICENSE_FILE=`` -> ``-v :/opt/lattice/license.dat:ro`` AND ``-e LM_LICENSE_FILE=/opt/lattice/license.dat``.""" lic = _make_license_file(tmp_path) # real on-disk file recorder, _ = _install_dispatcher_stubs( @@ -573,59 +569,78 @@ def test_case_9_file_path_license_emits_bind_and_env( # --------------------------------------------------------------------------- -def test_case_10_unset_license_does_not_invoke_subprocess( - peripherals, gateware_mod, tmp_path, monkeypatch, capsys +def test_unset_license_still_runs_without_license_args( + peripherals, gateware_mod, tmp_path, monkeypatch ): - """10: ``LM_LICENSE_FILE`` unset -> dispatcher raises - ``LicenseUnsetError`` (or wraps it in SystemExit) AND subprocess.run is - never called. - - AC-13's call sequence shows ``build_license_docker_args(os.environ)`` - runs BEFORE any docker invocation; an unset license must therefore - short-circuit the dispatcher before subprocess.run is touched. - - Strengthening: this case has to reach the LICENSE branch, not the - argparse-invalid-choice branch (which would also raise SystemExit(2) - today because the `gateware` subcommand isn't registered yet). We - assert the error message mentions ``LM_LICENSE_FILE`` and does NOT - contain argparse's ``invalid choice`` text, proving the dispatcher - actually reached the license-resolution step. + """``LM_LICENSE_FILE`` unset -> the pass-through STILL runs the SDK, + forwarding no license ``-v``/``-e`` args. + + The license is only needed by verbs that actually run Radiant (``build``); + requiring it up front here would block ``help``/``doctor``/``generate``/ + ``validate``/``sim``, none of which touch Radiant. So an unset license must + NOT short-circuit the dispatcher — the SDK's ``build`` preflight is the + single place that requires a license. We assert subprocess.run IS invoked + with the verb forwarded verbatim and no license mount/env present. """ recorder, _ = _install_dispatcher_stubs( peripherals, monkeypatch, tmp_path, license_value=None ) - with pytest.raises((SystemExit, gateware_mod.LicenseUnsetError)) as excinfo: + with pytest.raises(SystemExit): _dispatch(peripherals, ["doctor"]) - # If the dispatcher chose to wrap in SystemExit, the exit code must be - # non-zero; raw LicenseUnsetError is also acceptable. - if isinstance(excinfo.value, SystemExit): - assert excinfo.value.code not in (0, None), ( - f"unset license must exit non-zero; got code: {excinfo.value.code!r}" - ) + assert len(recorder.calls) == 1, ( + f"unset license must NOT block the pass-through; subprocess.run should " + f"still run the SDK; got: {recorder.calls!r}" + ) + argv = _docker_argv(recorder.calls[0]) + tail = _tail_after_image_tag(argv, "fake-gw:latest-amd64") + assert tail == ["axon-peripheral-sdk", "doctor"], ( + f"verb must be forwarded verbatim; got: {tail!r}" + ) + flat = " ".join(argv) + assert "LM_LICENSE_FILE" not in flat, ( + f"no license env may be forwarded when unset; got: {argv!r}" + ) + assert "/opt/lattice/license.dat" not in flat, ( + f"no license bind-mount when unset; got: {argv!r}" + ) - assert recorder.calls == [], ( - f"subprocess.run must NOT be invoked when LM_LICENSE_FILE is unset; " - f"got: {recorder.calls!r}" + +def test_set_but_missing_license_warns_and_still_runs( + peripherals, tmp_path, monkeypatch, capsys +): + """LM_LICENSE_FILE set to a NON-EXISTENT file -> the pass-through warns (the + misconfig is surfaced, not silently masked) but STILL runs the SDK with no + license args. + + build_license_docker_args raises FileNotFoundError (from + Path.resolve(strict=True)), NOT LicenseUnsetError, for a set-but-bad path; + the dispatcher must catch it too so non-Radiant verbs keep working (only + `build` fails SDK-side).""" + recorder, _ = _install_dispatcher_stubs( + peripherals, + monkeypatch, + tmp_path, + license_value=str(tmp_path / "does_not_exist.dat"), ) - # Anti-tautology: the error must come from the LICENSE branch, not - # argparse's invalid-choice path. argparse would emit "invalid choice" - # to stderr and never touch the license-resolution code. - captured = capsys.readouterr() - msg = (captured.out + captured.err + str(excinfo.value)).lower() - assert "invalid choice" not in msg, ( - f"case 10 must reach the license-resolution branch, NOT argparse's " - f"invalid-choice path (which fires today because `gateware` is not " - f"yet a registered subcommand). The presence of 'invalid choice' " - f"means AC-13's subparser registration hasn't happened yet. " - f"got: {captured.out + captured.err!r} value={excinfo.value!r}" + with pytest.raises(SystemExit): + _dispatch(peripherals, ["doctor"]) + + assert len(recorder.calls) == 1, ( + "a set-but-missing license must NOT block the pass-through" + ) + argv = _docker_argv(recorder.calls[0]) + tail = _tail_after_image_tag(argv, "fake-gw:latest-amd64") + assert tail == ["axon-peripheral-sdk", "doctor"], f"verb verbatim; got: {tail!r}" + flat = " ".join(argv) + assert "LM_LICENSE_FILE" not in flat and "/opt/lattice/license.dat" not in flat, ( + f"no license args forwarded for a bad path; got: {argv!r}" ) - assert "lm_license_file" in msg, ( - f"unset-license error must mention 'LM_LICENSE_FILE' so users know " - f"what env var to set; got: {captured.out + captured.err!r} " - f"value={excinfo.value!r}" + out = capsys.readouterr().out + assert "Warning" in out and "LM_LICENSE_FILE" in out, ( + f"a set-but-missing license must emit a warning; got: {out!r}" ) @@ -634,10 +649,8 @@ def test_case_10_unset_license_does_not_invoke_subprocess( # --------------------------------------------------------------------------- -def test_case_11_gateware_help_consumed_by_argparse( - peripherals, tmp_path, monkeypatch, capsys -): - """11: ``peripherals gateware --help`` -> argparse exits 0, prints +def test_gateware_help_consumed_by_argparse(peripherals, tmp_path, monkeypatch, capsys): + """``peripherals gateware --help`` -> argparse exits 0, prints synapsectl-side gateware help; subprocess.run NOT called. AC-13's `--help` dichotomy: when `--help` is the FIRST token after @@ -675,8 +688,8 @@ def test_case_11_gateware_help_consumed_by_argparse( # --------------------------------------------------------------------------- -def test_case_12_verb_help_is_forwarded_to_sdk(peripherals, tmp_path, monkeypatch): - """12: ``peripherals gateware doctor --help`` -> REMAINDER captures +def test_verb_help_is_forwarded_to_sdk(peripherals, tmp_path, monkeypatch): + """``peripherals gateware doctor --help`` -> REMAINDER captures BOTH tokens; subprocess.run IS called with the verb + --help in the tail. Companion to case 11: when at least one non-``--help`` positional @@ -707,8 +720,8 @@ def test_case_12_verb_help_is_forwarded_to_sdk(peripherals, tmp_path, monkeypatc # --------------------------------------------------------------------------- -def test_case_13_peripherals_help_lists_three_subcommands(peripherals, capsys): - """13: ``peripherals --help`` lists exactly ``build``, ``deploy``, +def test_peripherals_help_lists_three_subcommands(peripherals, capsys): + """``peripherals --help`` lists exactly ``build``, ``deploy``, ``gateware`` as subcommand entries -- no hard-coded SDK verbs. Locks the contract that synapsectl does NOT enumerate SDK verbs at @@ -766,10 +779,10 @@ def test_case_13_peripherals_help_lists_three_subcommands(peripherals, capsys): # --------------------------------------------------------------------------- -def test_case_14_invalid_subcommand_is_argparse_error_no_subprocess( +def test_invalid_subcommand_is_argparse_error_no_subprocess( peripherals, tmp_path, monkeypatch, capsys ): - """14: ``peripherals nonsense`` (NOT build/deploy/gateware) -> argparse + """``peripherals nonsense`` (NOT build/deploy/gateware) -> argparse ``SystemExit(2)`` with 'invalid choice' in stderr. subprocess.run NOT invoked. @@ -824,8 +837,8 @@ def test_case_14_invalid_subcommand_is_argparse_error_no_subprocess( # --------------------------------------------------------------------------- -def test_case_15_future_verb_forwarded_no_gate(peripherals, tmp_path, monkeypatch): - """15: ``gateware future-verb-2027`` -> REMAINDER captures the unknown +def test_future_verb_forwarded_no_gate(peripherals, tmp_path, monkeypatch): + """``gateware future-verb-2027`` -> REMAINDER captures the unknown verb; subprocess.run IS called with the verb in the docker argv tail. This proves the dispatcher does NOT gate on a known-verb list -- a @@ -901,9 +914,7 @@ def test_frontend_env_marker_forwarded_to_sdk(peripherals, tmp_path, monkeypatch # --------------------------------------------------------------------------- -def test_case_16_non_posix_host_exits_no_subprocess( - peripherals, tmp_path, monkeypatch, capsys -): +def test_non_posix_host_exits_no_subprocess(peripherals, tmp_path, monkeypatch, capsys): """16 (AC-13 POSIX-only): ``os.getuid`` raises ``AttributeError`` -> dispatcher exits non-zero, subprocess.run NOT called. @@ -947,3 +958,88 @@ def test_case_16_non_posix_host_exits_no_subprocess( f"alternative invocation path; " f"got: {captured.out + captured.err!r} value={excinfo.value!r}" ) + + +# --------------------------------------------------------------------------- +# Leading SDK options (e.g. --install-completion) forwarded verbatim. +# argparse.REMAINDER only captures from the first positional, so a leading +# option is folded into argv by parse_args_with_passthrough instead of being +# rejected as "unrecognized arguments". +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("opt", ["--install-completion", "--show-completion"]) +def test_leading_sdk_option_forwarded_verbatim(peripherals, tmp_path, monkeypatch, opt): + """``gateware --install-completion`` -> tail is exactly + ``["axon-peripheral-sdk", "--install-completion"]`` (no argparse rejection).""" + lic = _make_license_file(tmp_path) + recorder, _ = _install_dispatcher_stubs( + peripherals, monkeypatch, tmp_path, license_value=lic + ) + + with pytest.raises(SystemExit): + _dispatch(peripherals, [opt]) + + argv = _docker_argv(recorder.calls[0]) + tail = _tail_after_image_tag(argv, "fake-gw:latest-amd64") + assert tail == ["axon-peripheral-sdk", opt], ( + f"leading SDK option must be forwarded verbatim; got: {tail!r}" + ) + + +def test_non_passthrough_command_still_errors_on_unknown_args(peripherals): + """parse_args_with_passthrough preserves the strict error for non-gateware + commands -- only the gateware pass-through folds leftovers into argv.""" + parser = _build_root_parser(peripherals) + with pytest.raises(SystemExit) as excinfo: + peripherals.parse_args_with_passthrough( + parser, ["peripherals", "build", "both", "--bogus-flag"] + ) + assert excinfo.value.code == 2, ( + "unknown args on a non-pass-through command must stay an argparse error" + ) + + +# --------------------------------------------------------------------------- +# Pseudo-TTY allocation so the SDK's rich/typer output keeps its colors. +# --------------------------------------------------------------------------- + + +def test_tty_flag_added_when_stdout_is_tty( + peripherals, gateware_mod, tmp_path, monkeypatch +): + """When stdout is a tty, ``-t`` is present right after ``docker run --rm``.""" + lic = _make_license_file(tmp_path) + recorder, _ = _install_dispatcher_stubs( + peripherals, monkeypatch, tmp_path, license_value=lic + ) + monkeypatch.setattr(gateware_mod, "_stdout_is_tty", lambda: True) + + with pytest.raises(SystemExit): + _dispatch(peripherals, ["doctor"]) + + argv = _docker_argv(recorder.calls[0]) + assert "-t" in argv, f"-t must be present when stdout is a tty; got: {argv!r}" + rm_idx = argv.index("--rm") + assert argv[rm_idx + 1] == "-t", ( + f"-t must immediately follow 'docker run --rm'; got: {argv!r}" + ) + + +def test_tty_flag_absent_when_stdout_not_tty( + peripherals, gateware_mod, tmp_path, monkeypatch +): + """When stdout is NOT a tty (pipe/CI), ``-t`` is omitted so output stays clean.""" + lic = _make_license_file(tmp_path) + recorder, _ = _install_dispatcher_stubs( + peripherals, monkeypatch, tmp_path, license_value=lic + ) + monkeypatch.setattr(gateware_mod, "_stdout_is_tty", lambda: False) + + with pytest.raises(SystemExit): + _dispatch(peripherals, ["doctor"]) + + argv = _docker_argv(recorder.calls[0]) + assert "-t" not in argv, ( + f"-t must be omitted when stdout is not a tty; got: {argv!r}" + ) From afc0d9ef52afe7b6d1183c16b78d67a2cb7459c0 Mon Sep 17 00:00:00 2001 From: calvinleng-science Date: Wed, 10 Jun 2026 11:29:17 -0700 Subject: [PATCH 08/21] feat(peripherals): read probe usb_pid from the bitstream build summary --- synapse/cli/gateware.py | 44 ++++++++ .../cli/test_custom_gateware_packaging.py | 100 ++++++++++++++++++ 2 files changed, 144 insertions(+) create mode 100644 synapse/tests/cli/test_custom_gateware_packaging.py diff --git a/synapse/cli/gateware.py b/synapse/cli/gateware.py index 95a2bf88..33d148c7 100644 --- a/synapse/cli/gateware.py +++ b/synapse/cli/gateware.py @@ -10,6 +10,7 @@ from __future__ import annotations import glob +import json import os import re import subprocess @@ -179,6 +180,49 @@ def run_gateware_build( return chosen +def summary_path_for(bit_path: str) -> str: + """Return the same-stem ``.summary.json`` path for *bit_path*. + + The gateware build emits ``sdk_.summary.json`` next to each + ``sdk_.bit``. + """ + stem, _ = os.path.splitext(bit_path) + return f"{stem}.summary.json" + + +def read_usb_pid(bit_path: str) -> int: + """Return ``['project']['usb_pid']`` from the bitstream's summary JSON. + + The custom-bitstream manifest fragment needs the probe USB product id the + gateware targets; the gateware toolchain records it in the build summary. + + Raises: + FileNotFoundError: no ``.summary.json`` exists next to *bit_path*. + ValueError: the summary is not valid JSON, or ``project.usb_pid`` is + missing or not an integer. + """ + path = summary_path_for(bit_path) + if not os.path.exists(path): + raise FileNotFoundError( + f"Bitstream summary not found: {path}. The gateware build is " + "expected to emit a .summary.json next to each .bit; rebuild " + "with an axon-peripheral-sdk that records project.usb_pid." + ) + with open(path, "r", encoding="utf-8") as fp: + try: + summary = json.load(fp) + except json.JSONDecodeError as exc: + raise ValueError(f"Bitstream summary {path} is not valid JSON: {exc}") + project = summary.get("project") + usb_pid = project.get("usb_pid") if isinstance(project, dict) else None + if not isinstance(usb_pid, int) or isinstance(usb_pid, bool): + raise ValueError( + f"Bitstream summary {path} is missing ['project']['usb_pid'] " + "(expected an integer USB product id)" + ) + return usb_pid + + def _stdout_is_tty() -> bool: """Whether our stdout is a terminal (indirection kept for monkeypatching).""" return sys.stdout.isatty() diff --git a/synapse/tests/cli/test_custom_gateware_packaging.py b/synapse/tests/cli/test_custom_gateware_packaging.py new file mode 100644 index 00000000..ff34783f --- /dev/null +++ b/synapse/tests/cli/test_custom_gateware_packaging.py @@ -0,0 +1,100 @@ +"""Custom gateware packaging: summary parsing, deb selection, gateware deb. + +Covers the synapse-python half of the custom-bitstreams design (spec: +docs/superpowers/specs/2026-06-09-custom-gateware-bitstreams-design.md): + + * ``gateware.summary_path_for`` / ``gateware.read_usb_pid`` + * ``build.find_deb_package`` package_name filtering + * ``peripherals.build_gateware_deb`` staging layout + fpm invocation + +The two-deb build/deploy command flows live in test_half_selectors.py. +""" + +from __future__ import annotations + +import importlib +import json +import os + +import pytest + + +@pytest.fixture() +def gateware(): + """Lazy import (mirrors test_half_selectors.py) so conftest stubs apply.""" + return importlib.import_module("synapse.cli.gateware") + + +@pytest.fixture() +def buildmod(): + return importlib.import_module("synapse.cli.build") + + +@pytest.fixture() +def peripherals(): + return importlib.import_module("synapse.cli.peripherals") + + +def _write_summary(bit_path, payload): + """Drop ``.summary.json`` next to *bit_path*; payload str = raw.""" + stem, _ = os.path.splitext(str(bit_path)) + path = f"{stem}.summary.json" + with open(path, "w", encoding="utf-8") as fp: + if isinstance(payload, str): + fp.write(payload) + else: + json.dump(payload, fp) + return path + + +# --------------------------------------------------------------------------- +# gateware.summary_path_for / gateware.read_usb_pid +# --------------------------------------------------------------------------- + + +def test_summary_path_for_same_stem(gateware, tmp_path): + bit = tmp_path / "sdk_via_v0.0.0.bit" + assert gateware.summary_path_for(str(bit)) == str( + tmp_path / "sdk_via_v0.0.0.summary.json" + ) + + +def test_read_usb_pid_happy_path(gateware, tmp_path): + bit = tmp_path / "sdk_x.bit" + bit.write_text("bit") + _write_summary(bit, {"project": {"name": "gateware", "usb_pid": 4}}) + assert gateware.read_usb_pid(str(bit)) == 4 + + +def test_read_usb_pid_missing_summary_raises(gateware, tmp_path): + bit = tmp_path / "sdk_x.bit" + bit.write_text("bit") + with pytest.raises(FileNotFoundError) as exc_info: + gateware.read_usb_pid(str(bit)) + assert "summary" in str(exc_info.value).lower() + + +def test_read_usb_pid_invalid_json_raises(gateware, tmp_path): + bit = tmp_path / "sdk_x.bit" + bit.write_text("bit") + _write_summary(bit, "{not json") + with pytest.raises(ValueError): + gateware.read_usb_pid(str(bit)) + + +def test_read_usb_pid_missing_key_raises(gateware, tmp_path): + bit = tmp_path / "sdk_x.bit" + bit.write_text("bit") + # Real shape observed from the SDK today (usb_pid not yet emitted). + _write_summary(bit, {"project": {"name": "gateware", "git_sha": "77d672b"}}) + with pytest.raises(ValueError) as exc_info: + gateware.read_usb_pid(str(bit)) + assert "usb_pid" in str(exc_info.value) + + +def test_read_usb_pid_rejects_non_int(gateware, tmp_path): + bit = tmp_path / "sdk_x.bit" + bit.write_text("bit") + _write_summary(bit, {"project": {"usb_pid": "4"}}) + with pytest.raises(ValueError): + gateware.read_usb_pid(str(bit)) From 8d54070a0a81001460ae3ec60c6c5cfe81ef69cd Mon Sep 17 00:00:00 2001 From: calvinleng-science Date: Wed, 10 Jun 2026 11:34:33 -0700 Subject: [PATCH 09/21] fix(peripherals): reject non-object bitstream summaries with ValueError --- synapse/cli/gateware.py | 2 +- synapse/tests/cli/test_custom_gateware_packaging.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/synapse/cli/gateware.py b/synapse/cli/gateware.py index 33d148c7..9c1a3815 100644 --- a/synapse/cli/gateware.py +++ b/synapse/cli/gateware.py @@ -213,7 +213,7 @@ def read_usb_pid(bit_path: str) -> int: summary = json.load(fp) except json.JSONDecodeError as exc: raise ValueError(f"Bitstream summary {path} is not valid JSON: {exc}") - project = summary.get("project") + project = summary.get("project") if isinstance(summary, dict) else None usb_pid = project.get("usb_pid") if isinstance(project, dict) else None if not isinstance(usb_pid, int) or isinstance(usb_pid, bool): raise ValueError( diff --git a/synapse/tests/cli/test_custom_gateware_packaging.py b/synapse/tests/cli/test_custom_gateware_packaging.py index ff34783f..7abe17e5 100644 --- a/synapse/tests/cli/test_custom_gateware_packaging.py +++ b/synapse/tests/cli/test_custom_gateware_packaging.py @@ -98,3 +98,11 @@ def test_read_usb_pid_rejects_non_int(gateware, tmp_path): _write_summary(bit, {"project": {"usb_pid": "4"}}) with pytest.raises(ValueError): gateware.read_usb_pid(str(bit)) + + +def test_read_usb_pid_non_object_summary_raises(gateware, tmp_path): + bit = tmp_path / "sdk_x.bit" + bit.write_text("bit") + _write_summary(bit, [1, 2]) + with pytest.raises(ValueError): + gateware.read_usb_pid(str(bit)) From e6e35ca1bb8c4c15f8e674851b59723ecbc5c731 Mon Sep 17 00:00:00 2001 From: calvinleng-science Date: Wed, 10 Jun 2026 11:35:43 -0700 Subject: [PATCH 10/21] feat(peripherals): let find_deb_package select a deb by package name --- synapse/cli/build.py | 23 +++++++++------ .../cli/test_custom_gateware_packaging.py | 28 +++++++++++++++++++ 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/synapse/cli/build.py b/synapse/cli/build.py index baaef5f9..1e485403 100644 --- a/synapse/cli/build.py +++ b/synapse/cli/build.py @@ -635,15 +635,22 @@ def package_app(app_dir: str, app_name: str) -> bool: return build_deb_package(app_dir, app_name) -def find_deb_package(dist_dir: str) -> str | None: - """Return the path to the .deb generated in *app_dir* or *None*.""" - for file in os.listdir(dist_dir): - if file.endswith(".deb"): - return os.path.join(dist_dir, file) +def find_deb_package(dist_dir: str, package_name: str | None = None) -> str | None: + """Return the path to a .deb generated in *dist_dir* or ``None``. - console.print( - f"[bold red]Error:[/bold red] Could not find .deb package in {dist_dir}" - ) + With *package_name*, only ``_*.deb`` matches — a peripheral + dist/ holds both the driver and the ``-gateware`` deb, and the driver name + is a strict prefix of the gateware name. + """ + for file in sorted(os.listdir(dist_dir)): + if not file.endswith(".deb"): + continue + if package_name is not None and not file.startswith(f"{package_name}_"): + continue + return os.path.join(dist_dir, file) + + wanted = f"{package_name} .deb package" if package_name else ".deb package" + console.print(f"[bold red]Error:[/bold red] Could not find {wanted} in {dist_dir}") return None diff --git a/synapse/tests/cli/test_custom_gateware_packaging.py b/synapse/tests/cli/test_custom_gateware_packaging.py index 7abe17e5..fb2d9455 100644 --- a/synapse/tests/cli/test_custom_gateware_packaging.py +++ b/synapse/tests/cli/test_custom_gateware_packaging.py @@ -106,3 +106,31 @@ def test_read_usb_pid_non_object_summary_raises(gateware, tmp_path): _write_summary(bit, [1, 2]) with pytest.raises(ValueError): gateware.read_usb_pid(str(bit)) + + +# --------------------------------------------------------------------------- +# build.find_deb_package package_name filtering +# --------------------------------------------------------------------------- + + +def test_find_deb_package_unfiltered_back_compat(buildmod, tmp_path): + (tmp_path / "anything_0.1.0_arm64.deb").write_text("deb") + found = buildmod.find_deb_package(str(tmp_path)) + assert found is not None and found.endswith("anything_0.1.0_arm64.deb") + + +def test_find_deb_package_filters_by_package_name(buildmod, tmp_path): + # A peripheral dist/ now holds BOTH debs; the driver name is a strict + # prefix of the gateware name, so matching must be on "_". + (tmp_path / "via_0.1.0_arm64.deb").write_text("deb") + (tmp_path / "via-gateware_0.1.0_arm64.deb").write_text("deb") + driver = buildmod.find_deb_package(str(tmp_path), "via") + gw = buildmod.find_deb_package(str(tmp_path), "via-gateware") + assert driver is not None and driver.endswith(os.sep + "via_0.1.0_arm64.deb") + assert gw is not None and gw.endswith(os.sep + "via-gateware_0.1.0_arm64.deb") + + +def test_find_deb_package_no_match_returns_none(buildmod, tmp_path, capsys): + (tmp_path / "other_0.1.0_arm64.deb").write_text("deb") + assert buildmod.find_deb_package(str(tmp_path), "via") is None + assert "could not find" in capsys.readouterr().out.lower() From 2de9078bac14e84f597b74672559176ffccdbd30 Mon Sep 17 00:00:00 2001 From: calvinleng-science Date: Wed, 10 Jun 2026 11:50:55 -0700 Subject: [PATCH 11/21] feat(peripherals): package custom gateware as a -gateware .deb with manifest fragment --- synapse/cli/peripherals.py | 148 ++++++++++++++++++ synapse/tests/cli/conftest.py | 28 ++++ .../cli/test_custom_gateware_packaging.py | 76 +++++++++ 3 files changed, 252 insertions(+) diff --git a/synapse/cli/peripherals.py b/synapse/cli/peripherals.py index 8f5aeda6..007fd217 100644 --- a/synapse/cli/peripherals.py +++ b/synapse/cli/peripherals.py @@ -17,6 +17,7 @@ from __future__ import annotations import argparse +import json import os import shutil import subprocess @@ -594,6 +595,153 @@ def build_peripheral_deb( # /tmp eventually cleans itself. +# Suffix appended to the plugin name to form the gateware package name. +GATEWARE_DEB_SUFFIX = "-gateware" +# Owns /opt/scifi/bitstreams and the canonical manifest the fragment's +# relative `artifact` resolves against. +BITSTREAMS_PACKAGE = "axonprobe-bitstreams" + + +def build_gateware_deb( + peripheral_dir: str, + manifest: dict, + *, + bit_path: str, + usb_pid: int, + version: str = "0.1.0", +) -> bool: + """Stage the custom bitstream + manifest fragment, then fpm a .deb. + + Layout inside the ``-gateware`` .deb: + /opt/scifi/bitstreams/custom/.bit + /opt/scifi/bitstreams/custom/.manifest.json + + The fragment carries ``{"name", "usb_pid", "artifact"}`` with ``artifact`` + relative to /opt/scifi/bitstreams (canonical-manifest convention); + scifi-probe-updater globs custom/*.manifest.json to list flashable + custom gateware per probe. + """ + plugin_name = manifest["name"] + package_name = f"{plugin_name}{GATEWARE_DEB_SUFFIX}" + + if not os.path.exists(bit_path): + console.print( + f"[bold red]Error:[/bold red] Gateware .bit not found at {bit_path}" + ) + return False + + staging_dir = tempfile.mkdtemp(prefix="synapse-gateware-package-") + try: + custom_dir = os.path.join(staging_dir, "opt", "scifi", "bitstreams", "custom") + os.makedirs(custom_dir, exist_ok=True) + shutil.copy2(bit_path, os.path.join(custom_dir, f"{plugin_name}.bit")) + + fragment = { + "name": plugin_name, + "usb_pid": usb_pid, + "artifact": f"custom/{plugin_name}.bit", + } + fragment_path = os.path.join(custom_dir, f"{plugin_name}.manifest.json") + with open(fragment_path, "w", encoding="utf-8") as fp: + json.dump(fragment, fp, indent=2) + fp.write("\n") + + postinstall_path = os.path.join(staging_dir, "postinstall.sh") + with open(postinstall_path, "w", encoding="utf-8") as fp: + fp.write( + "#!/bin/bash\n" + "set -e\n" + "echo 'Custom gateware installed. Flash probes from the device " + "UI (Probe Updates) or scifi-probe-updater.'\n" + "exit 0\n" + ) + # Contents are embedded as the deb's postinst (via --after-install); + # dpkg makes maintainer scripts executable itself. + os.chmod(postinstall_path, 0o644) + + dist_dir = os.path.join(peripheral_dir, "dist") + os.makedirs(dist_dir, exist_ok=True) + + # Input is "opt" (not ".") so postinstall.sh is NOT packaged as a + # payload file — the driver deb installs alongside this one, and two + # packages shipping /postinstall.sh would dpkg-conflict. + fpm_args = [ + "fpm", + "-s", + "dir", + "-t", + "deb", + "-n", + package_name, + "-f", + "-v", + version, + "-C", + "/pkg", + "--deb-no-default-config-files", + "--vendor", + "Science Corporation", + "--description", + "Synapse peripheral custom gateware", + "--architecture", + "arm64", + "--category", + SECTION_LABEL, + "--depends", + BITSTREAMS_PACKAGE, + "--after-install", + "/pkg/postinstall.sh", + "opt", + ] + + console.print( + f"[yellow]Packaging gateware .deb (Docker image: {FPM_IMAGE}) ...[/yellow]" + ) + docker_fpm_cmd = [ + "docker", + "run", + "--rm", + "--platform", + "linux/amd64", + "--volume", + f"{staging_dir}:/pkg", + "--volume", + f"{dist_dir}:/out", + "-w", + "/out", + FPM_IMAGE, + ] + fpm_args + + subprocess.run( + docker_fpm_cmd, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + deb_files = [ + f + for f in os.listdir(dist_dir) + if f.startswith(f"{package_name}_") and f.endswith(".deb") + ] + if not deb_files: + console.print( + f"[bold red]Error:[/bold red] fpm completed but no {package_name} " + f".deb found in {dist_dir}." + ) + return False + + console.print("[green]Gateware .deb created successfully![/green]") + return True + + except subprocess.CalledProcessError as exc: + console.print(f"[bold red]Error:[/bold red] fpm failed: {exc}") + return False + # Leave staging_dir on disk for inspection if something goes wrong; + # /tmp eventually cleans itself. + + # --------------------------------------------------------------------------- # Gateware half helpers # --------------------------------------------------------------------------- diff --git a/synapse/tests/cli/conftest.py b/synapse/tests/cli/conftest.py index 28447373..9085c347 100644 --- a/synapse/tests/cli/conftest.py +++ b/synapse/tests/cli/conftest.py @@ -41,3 +41,31 @@ def _main(): # pragma: no cover - never invoked in tests _install_cli_import_stubs() + + +def fake_fpm_run(dist_dir: str, calls: list): + """Return a ``subprocess.run`` stub that records argv and fakes fpm. + + When the recorded argv contains ``fpm`` (the real call runs fpm inside a + docker image, so ``"fpm"`` is an element of the docker argv), drop a + ``__arm64.deb`` into *dist_dir* so the caller's post-fpm + "did a .deb land?" verification passes. All other argv (docker clean, + runtime extraction) are recorded and succeed as no-ops. + """ + import os + import subprocess as _subprocess + + def run(argv, *args, **kwargs): + argv_list = list(argv) if isinstance(argv, (list, tuple)) else [argv] + calls.append(argv_list) + if "fpm" in argv_list: + name = argv_list[argv_list.index("-n") + 1] + version = argv_list[argv_list.index("-v") + 1] + os.makedirs(dist_dir, exist_ok=True) + with open( + os.path.join(dist_dir, f"{name}_{version}_arm64.deb"), "w" + ) as fh: + fh.write("fake-deb") + return _subprocess.CompletedProcess(argv_list, 0, b"", b"") + + return run diff --git a/synapse/tests/cli/test_custom_gateware_packaging.py b/synapse/tests/cli/test_custom_gateware_packaging.py index fb2d9455..a3d16c3d 100644 --- a/synapse/tests/cli/test_custom_gateware_packaging.py +++ b/synapse/tests/cli/test_custom_gateware_packaging.py @@ -134,3 +134,79 @@ def test_find_deb_package_no_match_returns_none(buildmod, tmp_path, capsys): (tmp_path / "other_0.1.0_arm64.deb").write_text("deb") assert buildmod.find_deb_package(str(tmp_path), "via") is None assert "could not find" in capsys.readouterr().out.lower() + + +# --------------------------------------------------------------------------- +# peripherals.build_gateware_deb +# --------------------------------------------------------------------------- + +# synapse/tests/cli/ is a package (has __init__.py), so import the helper +# package-qualified rather than relying on pytest's rootdir sys.path insert. +from synapse.tests.cli.conftest import fake_fpm_run + + +def _spy_mkdtemp(peripherals, monkeypatch, holder: list): + real_mkdtemp = peripherals.tempfile.mkdtemp + + def spy(*args, **kwargs): + d = real_mkdtemp(*args, **kwargs) + holder.append(d) + return d + + monkeypatch.setattr(peripherals.tempfile, "mkdtemp", spy) + + +def test_build_gateware_deb_stages_bit_fragment_and_depends( + peripherals, tmp_path, monkeypatch +): + pd = tmp_path / "plugin" + pd.mkdir() + bit = tmp_path / "sdk_x.bit" + bit.write_text("BITSTREAM") + manifest = {"name": "scifi-my-chip", "version": "0.2.0"} + + staging: list = [] + _spy_mkdtemp(peripherals, monkeypatch, staging) + calls: list = [] + dist_dir = os.path.join(str(pd), "dist") + monkeypatch.setattr(peripherals.subprocess, "run", fake_fpm_run(dist_dir, calls)) + + ok = peripherals.build_gateware_deb( + str(pd), manifest, bit_path=str(bit), usb_pid=4, version="0.2.0" + ) + assert ok is True + assert len(staging) == 1 + + bit_dst = os.path.join( + staging[0], "opt", "scifi", "bitstreams", "custom", "scifi-my-chip.bit" + ) + frag_dst = os.path.join( + staging[0], "opt", "scifi", "bitstreams", "custom", + "scifi-my-chip.manifest.json", + ) + assert os.path.exists(bit_dst), "bitstream staged under custom/ as .bit" + with open(frag_dst, "r", encoding="utf-8") as fh: + frag = json.load(fh) + assert frag == { + "name": "scifi-my-chip", + "usb_pid": 4, + "artifact": "custom/scifi-my-chip.bit", + } + + fpm_call = next(c for c in calls if "fpm" in c) + assert fpm_call[fpm_call.index("-n") + 1] == "scifi-my-chip-gateware" + assert fpm_call[fpm_call.index("--depends") + 1] == "axonprobe-bitstreams" + # fpm input must be "opt" (not "."): postinstall.sh must NOT ship in the + # payload, or the driver and gateware debs would dpkg-conflict on + # /postinstall.sh. + assert fpm_call[-1] == "opt" + + +def test_build_gateware_deb_missing_bit_errors(peripherals, tmp_path, capsys): + pd = tmp_path / "plugin" + pd.mkdir() + ok = peripherals.build_gateware_deb( + str(pd), {"name": "x"}, bit_path=str(tmp_path / "nope.bit"), usb_pid=1 + ) + assert ok is False + assert "not found" in capsys.readouterr().out.lower() From 6775bb443d5f3cd30ea19b45bde5e68125fc71d3 Mon Sep 17 00:00:00 2001 From: calvinleng-science Date: Wed, 10 Jun 2026 11:52:03 -0700 Subject: [PATCH 12/21] test(peripherals): parse fpm args from the fpm token in fake_fpm_run --- synapse/cli/peripherals.py | 4 ++-- synapse/tests/cli/conftest.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/synapse/cli/peripherals.py b/synapse/cli/peripherals.py index 007fd217..765cc1ae 100644 --- a/synapse/cli/peripherals.py +++ b/synapse/cli/peripherals.py @@ -703,9 +703,9 @@ def build_gateware_deb( "--rm", "--platform", "linux/amd64", - "--volume", + "-v", f"{staging_dir}:/pkg", - "--volume", + "-v", f"{dist_dir}:/out", "-w", "/out", diff --git a/synapse/tests/cli/conftest.py b/synapse/tests/cli/conftest.py index 9085c347..40e55ef5 100644 --- a/synapse/tests/cli/conftest.py +++ b/synapse/tests/cli/conftest.py @@ -59,8 +59,9 @@ def run(argv, *args, **kwargs): argv_list = list(argv) if isinstance(argv, (list, tuple)) else [argv] calls.append(argv_list) if "fpm" in argv_list: - name = argv_list[argv_list.index("-n") + 1] - version = argv_list[argv_list.index("-v") + 1] + fpm_argv = argv_list[argv_list.index("fpm"):] + name = fpm_argv[fpm_argv.index("-n") + 1] + version = fpm_argv[fpm_argv.index("-v") + 1] os.makedirs(dist_dir, exist_ok=True) with open( os.path.join(dist_dir, f"{name}_{version}_arm64.deb"), "w" From 96ac7bafadd2c20b8463cf19b779b0beb04e177f Mon Sep 17 00:00:00 2001 From: calvinleng-science Date: Wed, 10 Jun 2026 12:24:02 -0700 Subject: [PATCH 13/21] feat(peripherals): split build/deploy into driver and -gateware debs Switch build_cmd and deploy_cmd to the two-deb flow via _build_debs: driver deb (so + SDK runtime, fpm input "usr") is built first, then a separate -gateware deb (bit + manifest fragment under opt/scifi/bitstreams/custom/). build_peripheral_deb is now driver-only (so_path required, bit_path removed, _expected_bit_filename deleted). _run_fpm extracts the shared docker/fpm/verify block used by both packagers. _gateware_usb_pid wraps gateware.read_usb_pid and aborts cleanly when the .summary.json is absent. Test harness gains build_gateware_deb recorder, summary-writing fake_run_gateware, and two-arg find_deb_package stub; cases M/N/O rewritten for two-deb layout; P1-P7 deleted; Q and R added for deploy streaming and missing-summary abort path. --- synapse/cli/peripherals.py | 506 +++++++++++------------ synapse/tests/cli/test_half_selectors.py | 377 ++++++++--------- 2 files changed, 397 insertions(+), 486 deletions(-) diff --git a/synapse/cli/peripherals.py b/synapse/cli/peripherals.py index 765cc1ae..5d4288b8 100644 --- a/synapse/cli/peripherals.py +++ b/synapse/cli/peripherals.py @@ -199,26 +199,6 @@ def _expected_so_filename(manifest: dict) -> str: return f"{manifest['name']}.so" -def _expected_bit_filename(manifest: dict) -> str: - """Return the basename of the .bit this plugin produces. - - Reads manifest.install.gateware_target if present (e.g. - "/usr/lib/scifi/gateware/via.bit" → "via.bit"), - otherwise falls back to the .so stem (e.g. - "via.so" → "via.bit"), which itself falls back - to ".bit" when install.target is also absent. - - An empty-string gateware_target is treated as "not set" and falls - through to the .so-stem fallback. - """ - install = manifest.get("install") or {} - target = install.get("gateware_target") - if target: - return os.path.basename(target) - so_stem = os.path.splitext(_expected_so_filename(manifest))[0] - return f"{so_stem}.bit" - - # --------------------------------------------------------------------------- # Build .so # --------------------------------------------------------------------------- @@ -364,144 +344,170 @@ def build_peripheral_so( # --------------------------------------------------------------------------- +def _run_fpm( + staging_dir: str, dist_dir: str, fpm_args: list, package_name: str +) -> bool: + """Run fpm inside the packaging image and verify a .deb landed. + + *fpm_args* is the complete fpm argv (starting with ``"fpm"``). Returns + False (with console errors, including fpm's stderr) on failure. + """ + docker_fpm_cmd = [ + "docker", + "run", + "--rm", + "--platform", + "linux/amd64", + "-v", + f"{staging_dir}:/pkg", + "-v", + f"{dist_dir}:/out", + "-w", + "/out", + FPM_IMAGE, + ] + fpm_args + + try: + subprocess.run( + docker_fpm_cmd, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + except subprocess.CalledProcessError as exc: + console.print(f"[bold red]Error:[/bold red] fpm failed: {exc}") + if exc.stderr: + console.print(exc.stderr) + return False + + deb_files = [ + f + for f in os.listdir(dist_dir) + if f.startswith(f"{package_name}_") and f.endswith(".deb") + ] + if not deb_files: + console.print( + f"[bold red]Error:[/bold red] fpm completed but no {package_name} " + f".deb found in {dist_dir}." + ) + return False + return True + + def build_peripheral_deb( peripheral_dir: str, manifest: dict, *, - bit_path: Optional[str] = None, - so_path: Optional[str] = None, + so_path: str, version: str = "0.1.0", ) -> bool: - """Stage plugin artifacts + SDK runtime, then run fpm to produce a .deb. + """Stage the driver .so + SDK runtime, then run fpm to produce a .deb. - Layout inside the .deb (entries present per the supplied paths): - /usr/lib/scifi/plugins/ ← when so_path is provided - /usr/lib/scifi/gateware/ ← when bit_path is provided - /usr/lib/libscifi-peripheral-sdk.so.* ← only when so_path is provided + Layout inside the .deb: + /usr/lib/scifi/plugins/ + /usr/lib/libscifi-peripheral-sdk.so.* Section is set to `synapse-peripherals` so scifi-server's DeployApp gate accepts it (sibling accept-list entry next to `synapse-apps`). """ - if so_path is None and bit_path is None: - console.print( - "[bold red]Error:[/bold red] build_peripheral_deb requires at least one of " - "so_path or bit_path." - ) - return False - plugin_name = manifest["name"] so_filename = _expected_so_filename(manifest) - bit_filename = _expected_bit_filename(manifest) staging_dir = tempfile.mkdtemp(prefix="synapse-peripheral-package-") try: # 1. Stage the plugin .so at /usr/lib/scifi/plugins/.so - if so_path is not None: - if not os.path.exists(so_path): - console.print( - f"[bold red]Error:[/bold red] Plugin .so not found at {so_path}" - ) - return False - plugin_dst = os.path.join(staging_dir, "usr", "lib", "scifi", "plugins") - os.makedirs(plugin_dst, exist_ok=True) - shutil.copy2(so_path, os.path.join(plugin_dst, so_filename)) + if not os.path.exists(so_path): + console.print( + f"[bold red]Error:[/bold red] Plugin .so not found at {so_path}" + ) + return False + plugin_dst = os.path.join(staging_dir, "usr", "lib", "scifi", "plugins") + os.makedirs(plugin_dst, exist_ok=True) + shutil.copy2(so_path, os.path.join(plugin_dst, so_filename)) - # 1b. Stage the gateware .bit at /usr/lib/scifi/gateware/.bit - if bit_path is not None: - if not os.path.exists(bit_path): + # 2. Stage libscifi-peripheral-sdk.so* from the builder image at /usr/lib. + # The SDK ships via `apt-get install scifi-peripheral-sdk` inside the + # builder Dockerfile, so it's the same source the linker resolved against + # at build time — guaranteeing ABI alignment for the plugin. + sdk_dst = os.path.join(staging_dir, "usr", "lib") + os.makedirs(sdk_dst, exist_ok=True) + + # Prefer libs already produced on disk next to the .so (the driver + # builder may stage them there). Fall back to extracting from the + # builder image only if none are present locally. + local_libs_dir = os.path.join(peripheral_dir, "build", "aarch64") + local_libs = ( + [ + f + for f in os.listdir(local_libs_dir) + if f.startswith("libscifi-peripheral-sdk.so") + ] + if os.path.isdir(local_libs_dir) + else [] + ) + if local_libs: + for fname in local_libs: + shutil.copy2( + os.path.join(local_libs_dir, fname), + os.path.join(sdk_dst, fname), + ) + else: + try: + image_tag = build_docker_image( + peripheral_dir, "axon-peripheral", roles=["driver"] + )["driver"] + except ( + subprocess.CalledProcessError, + FileNotFoundError, + KeyError, + ) as exc: console.print( - f"[bold red]Error:[/bold red] Gateware .bit not found at {bit_path}" + f"[bold red]Error:[/bold red] Failed to build driver Docker image: {exc}" ) return False - gw_dst = os.path.join(staging_dir, "usr", "lib", "scifi", "gateware") - os.makedirs(gw_dst, exist_ok=True) - shutil.copy2(bit_path, os.path.join(gw_dst, bit_filename)) + arch_suffix = detect_arch() + platform_opt = ( + "linux/arm64" if arch_suffix == "arm64" else "linux/amd64" + ) - # 2. Stage libscifi-peripheral-sdk.so* from the builder image at /usr/lib. - # Only when the driver half is part of this .deb — a gateware-only - # package has no need for the C++ runtime. The SDK ships via - # `apt-get install scifi-peripheral-sdk` inside the builder Dockerfile, - # so it's the same source the linker resolved against at build time — - # guaranteeing ABI alignment for the plugin. - if so_path is not None: - sdk_dst = os.path.join(staging_dir, "usr", "lib") - os.makedirs(sdk_dst, exist_ok=True) - - # Prefer libs already produced on disk next to the .so (the driver - # builder may stage them there). Fall back to extracting from the - # builder image only if none are present locally. - local_libs_dir = os.path.join(peripheral_dir, "build", "aarch64") - local_libs = ( - [ - f - for f in os.listdir(local_libs_dir) - if f.startswith("libscifi-peripheral-sdk.so") - ] - if os.path.isdir(local_libs_dir) - else [] + console.print( + f"[yellow]Extracting SDK runtime from Docker image [bold]{image_tag}[/bold]...[/yellow]" ) - if local_libs: - for fname in local_libs: - shutil.copy2( - os.path.join(local_libs_dir, fname), - os.path.join(sdk_dst, fname), - ) - else: - try: - image_tag = build_docker_image( - peripheral_dir, "axon-peripheral", roles=["driver"] - )["driver"] - except ( - subprocess.CalledProcessError, - FileNotFoundError, - KeyError, - ) as exc: - console.print( - f"[bold red]Error:[/bold red] Failed to build driver Docker image: {exc}" - ) - return False - arch_suffix = detect_arch() - platform_opt = ( - "linux/arm64" if arch_suffix == "arm64" else "linux/amd64" + extract_cmd = [ + "docker", + "run", + "--rm", + "--platform", + platform_opt, + "-v", + f"{sdk_dst}:/out", + image_tag, + "/bin/bash", + "-c", + r"find /usr/lib -maxdepth 1 -name 'libscifi-peripheral-sdk.so*' -exec cp -a {} /out/ \;", + ] + try: + subprocess.run(extract_cmd, check=True) + except subprocess.CalledProcessError as exc: + console.print( + f"[bold red]Error:[/bold red] Failed to extract SDK runtime: {exc}" ) + return False + sdk_files = [ + f + for f in os.listdir(sdk_dst) + if f.startswith("libscifi-peripheral-sdk.so") + ] + if not sdk_files: console.print( - f"[yellow]Extracting SDK runtime from Docker image [bold]{image_tag}[/bold]...[/yellow]" + "[bold red]Error:[/bold red] SDK runtime libraries not found in builder image. " + "Make sure your Dockerfile installs scifi-peripheral-sdk." ) - extract_cmd = [ - "docker", - "run", - "--rm", - "--platform", - platform_opt, - "-v", - f"{sdk_dst}:/out", - image_tag, - "/bin/bash", - "-c", - r"find /usr/lib -maxdepth 1 -name 'libscifi-peripheral-sdk.so*' -exec cp -a {} /out/ \;", - ] - try: - subprocess.run(extract_cmd, check=True) - except subprocess.CalledProcessError as exc: - console.print( - f"[bold red]Error:[/bold red] Failed to extract SDK runtime: {exc}" - ) - return False - - sdk_files = [ - f - for f in os.listdir(sdk_dst) - if f.startswith("libscifi-peripheral-sdk.so") - ] - if not sdk_files: - console.print( - "[bold red]Error:[/bold red] SDK runtime libraries not found in builder image. " - "Make sure your Dockerfile installs scifi-peripheral-sdk." - ) - return False - - # 4. Postinstall: nudge the user to restart scifi-server. + return False + + # 3. Postinstall: nudge the user to restart scifi-server. # Restarting automatically could interrupt an active recording session, # so leave it manual. postinstall_path = os.path.join(staging_dir, "postinstall.sh") @@ -518,7 +524,7 @@ def build_peripheral_deb( # file's own exec bit never reaches the package. os.chmod(postinstall_path, 0o644) - # 5. Run fpm inside the cdrx/fpm-ubuntu image (matches apps' packaging path). + # 4. Run fpm inside the cdrx/fpm-ubuntu image (matches apps' packaging path). dist_dir = os.path.join(peripheral_dir, "dist") os.makedirs(dist_dir, exist_ok=True) @@ -546,43 +552,16 @@ def build_peripheral_deb( SECTION_LABEL, "--after-install", "/pkg/postinstall.sh", - ".", + # Input is "usr" (not ".") so postinstall.sh is NOT packaged as a + # payload file — the -gateware deb installs alongside this one, + # and two packages shipping /postinstall.sh would dpkg-conflict. + "usr", ] console.print( f"[yellow]Packaging plugin .deb (Docker image: {FPM_IMAGE}) ...[/yellow]" ) - docker_fpm_cmd = [ - "docker", - "run", - "--rm", - "--platform", - "linux/amd64", - "-v", - f"{staging_dir}:/pkg", - "-v", - f"{dist_dir}:/out", - "-w", - "/out", - FPM_IMAGE, - ] + fpm_args - - subprocess.run( - docker_fpm_cmd, - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - - # Verify a .deb actually landed. - deb_files = [ - f for f in os.listdir(dist_dir) if f.endswith(".deb") and "arm64" in f - ] - if not deb_files: - console.print( - f"[bold red]Error:[/bold red] fpm completed but no .deb found in {dist_dir}." - ) + if not _run_fpm(staging_dir, dist_dir, fpm_args, plugin_name): return False console.print("[green]Plugin .deb created successfully![/green]") @@ -697,39 +676,7 @@ def build_gateware_deb( console.print( f"[yellow]Packaging gateware .deb (Docker image: {FPM_IMAGE}) ...[/yellow]" ) - docker_fpm_cmd = [ - "docker", - "run", - "--rm", - "--platform", - "linux/amd64", - "-v", - f"{staging_dir}:/pkg", - "-v", - f"{dist_dir}:/out", - "-w", - "/out", - FPM_IMAGE, - ] + fpm_args - - subprocess.run( - docker_fpm_cmd, - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - - deb_files = [ - f - for f in os.listdir(dist_dir) - if f.startswith(f"{package_name}_") and f.endswith(".deb") - ] - if not deb_files: - console.print( - f"[bold red]Error:[/bold red] fpm completed but no {package_name} " - f".deb found in {dist_dir}." - ) + if not _run_fpm(staging_dir, dist_dir, fpm_args, package_name): return False console.print("[green]Gateware .deb created successfully![/green]") @@ -794,36 +741,31 @@ def _run_gateware_half(peripheral_dir: str) -> Optional[str]: return None -# --------------------------------------------------------------------------- -# `peripherals build` -# --------------------------------------------------------------------------- - - -def build_cmd(args) -> None: - """Handle ``synapsectl peripherals build``.""" +def _gateware_usb_pid(bit_path: str) -> Optional[int]: + """Read the probe USB product id from the bitstream's summary, or None.""" + try: + return gateware.read_usb_pid(bit_path) + except (FileNotFoundError, ValueError) as exc: + console.print(f"[bold red]Error:[/bold red] {exc}") + return None - if not ensure_docker(): - return - peripheral_dir = os.path.abspath(args.peripheral_dir) - - manifest = validate_manifest(os.path.join(peripheral_dir, "manifest.json")) - if not manifest: - return +def _build_debs( + peripheral_dir: str, manifest: dict, half: str, *, clean: bool = False +) -> Optional[list]: + """Build the requested halves; return built .deb paths or None on failure. + Driver deb first, then the -gateware deb — deploy streams them in this + order so the plugin lands before its gateware shows up as flashable. + """ plugin_name = manifest["name"] version = manifest.get("version", "0.1.0") - so_filename = _expected_so_filename(manifest) - half = getattr(args, "half", "both") do_driver = half in ("driver", "both") do_gateware = half in ("gateware", "both") + dist_dir = os.path.join(peripheral_dir, "dist") + debs: list = [] - console.print( - f"[bold]Building peripheral plugin:[/bold] [yellow]{plugin_name}[/yellow] " - f"(artifact: [cyan]{so_filename}[/cyan])" - ) - - if do_gateware and args.clean: + if do_gateware and clean: try: gateware_image_tag = build_docker_image( peripheral_dir, "axon-peripheral", roles=["gateware"] @@ -832,43 +774,80 @@ def build_cmd(args) -> None: console.print( f"[bold red]Error:[/bold red] Failed to build gateware Docker image: {exc}" ) - return + return None _clean_gateware_tree(peripheral_dir, gateware_image_tag) - so_path: Optional[str] = None - bit_path: Optional[str] = None - if do_driver: + so_filename = _expected_so_filename(manifest) if not build_peripheral_so( - peripheral_dir, plugin_name, so_filename, clean=args.clean + peripheral_dir, plugin_name, so_filename, clean=clean ): - return + return None so_path = os.path.join(peripheral_dir, "build/aarch64", so_filename) + if not build_peripheral_deb( + peripheral_dir, manifest, so_path=so_path, version=version + ): + return None + deb = find_deb_package(dist_dir, plugin_name) + if deb is None: + return None + debs.append(deb) if do_gateware: bit_path = _run_gateware_half(peripheral_dir) if bit_path is None: - return + return None + usb_pid = _gateware_usb_pid(bit_path) + if usb_pid is None: + return None + if not build_gateware_deb( + peripheral_dir, manifest, bit_path=bit_path, usb_pid=usb_pid, version=version + ): + return None + deb = find_deb_package(dist_dir, f"{plugin_name}{GATEWARE_DEB_SUFFIX}") + if deb is None: + return None + debs.append(deb) + + return debs + + +# --------------------------------------------------------------------------- +# `peripherals build` +# --------------------------------------------------------------------------- + + +def build_cmd(args) -> None: + """Handle ``synapsectl peripherals build``.""" + + if not ensure_docker(): + return - if not build_peripheral_deb( - peripheral_dir, - manifest, - so_path=so_path, - bit_path=bit_path, - version=version, - ): + peripheral_dir = os.path.abspath(args.peripheral_dir) + + manifest = validate_manifest(os.path.join(peripheral_dir, "manifest.json")) + if not manifest: return - deb_path = find_deb_package(os.path.join(peripheral_dir, "dist")) - if not deb_path: + console.print( + f"[bold]Building peripheral plugin:[/bold] [yellow]{manifest['name']}[/yellow]" + ) + + debs = _build_debs( + peripheral_dir, manifest, getattr(args, "half", "both"), clean=args.clean + ) + if debs is None: return + package_lines = "\n".join(f"Package: [bold]{d}[/bold]" for d in debs) console.print( Panel( f"[green]Build complete![/green]\n\n" - f"Plugin: [bold]{plugin_name}[/bold] v{version}\n" - f"Package: [bold]{deb_path}[/bold]\n\n" - f"Deploy with: [cyan]synapsectl -u peripherals deploy .[/cyan]", + f"Plugin: [bold]{manifest['name']}[/bold] " + f"v{manifest.get('version', '0.1.0')}\n" + f"{package_lines}\n\n" + f"Deploy with: [cyan]synapsectl -u peripherals deploy " + f"{getattr(args, 'half', 'both')} .[/cyan]", title="Build Successful", border_style="green", box=box.DOUBLE, @@ -898,14 +877,14 @@ def deploy_cmd(args) -> None: f"[yellow]Warning: --{half} ignored when --package is provided; " f"deploying the supplied .deb as-is.[/yellow]" ) - deb_package: Optional[str] = os.path.abspath(args.package) - if not os.path.exists(deb_package): + deb_packages = [os.path.abspath(args.package)] + if not os.path.exists(deb_packages[0]): console.print( - f"[bold red]Error:[/bold red] Provided package not found: {deb_package}" + f"[bold red]Error:[/bold red] Provided package not found: {deb_packages[0]}" ) return console.print( - f"[bold]Deploying pre-built plugin:[/bold] [yellow]{os.path.basename(deb_package)}[/yellow]" + f"[bold]Deploying pre-built plugin:[/bold] [yellow]{os.path.basename(deb_packages[0])}[/yellow]" ) else: if not ensure_docker(): @@ -916,50 +895,25 @@ def deploy_cmd(args) -> None: if not manifest: return - plugin_name = manifest["name"] - version = manifest.get("version", "0.1.0") - so_filename = _expected_so_filename(manifest) - do_driver = half in ("driver", "both") - do_gateware = half in ("gateware", "both") - console.print( - f"[bold]Deploying peripheral plugin:[/bold] [yellow]{plugin_name}[/yellow]" + f"[bold]Deploying peripheral plugin:[/bold] [yellow]{manifest['name']}[/yellow]" ) - so_path: Optional[str] = None - bit_path: Optional[str] = None - - if do_driver: - if not build_peripheral_so(peripheral_dir, plugin_name, so_filename): - return - so_path = os.path.join(peripheral_dir, "build/aarch64", so_filename) - - if do_gateware: - bit_path = _run_gateware_half(peripheral_dir) - if bit_path is None: - return - - if not build_peripheral_deb( - peripheral_dir, - manifest, - so_path=so_path, - bit_path=bit_path, - version=version, - ): - return - - deb_package = find_deb_package(os.path.join(peripheral_dir, "dist")) - if not deb_package: + debs = _build_debs(peripheral_dir, manifest, half) + if debs is None: return + deb_packages = debs if not args.uri: console.print( - "[yellow]No URI provided. Package created but not deployed.[/yellow]" + "[yellow]No URI provided. Package(s) created but not deployed.[/yellow]" ) - console.print(f"[green]Package available at:[/green] {deb_package}") + for deb in deb_packages: + console.print(f"[green]Package available at:[/green] {deb}") return - deploy_package(args.uri, deb_package) + for deb in deb_packages: + deploy_package(args.uri, deb) # --------------------------------------------------------------------------- diff --git a/synapse/tests/cli/test_half_selectors.py b/synapse/tests/cli/test_half_selectors.py index 6ca1a017..e4ea02d5 100644 --- a/synapse/tests/cli/test_half_selectors.py +++ b/synapse/tests/cli/test_half_selectors.py @@ -1,11 +1,9 @@ -"""AC-11 → AC-7 / AC-8 (sub-phase 4.4), AC-12 + AC-15 (sub-phase 4.5). +"""AC-11 → AC-7 / AC-8 (sub-phase 4.4), AC-12 (sub-phase 4.5), two-deb flow. Tests the ``driver`` / ``gateware`` / ``both`` target subcommands on ``synapsectl peripherals build`` (AC-7) and ``... peripherals deploy`` -(AC-8), the ``--clean`` × half-selector matrix (AC-7 body), the -combined ``.deb`` layout contract (AC-12), and the -``_expected_bit_filename`` helper / new optional -``install.gateware_target`` manifest field (AC-15). +(AC-8), the ``--clean`` × half-selector matrix (AC-7 body), and the +two-deb staging layout (driver deb + separate -gateware deb). The half selectors were originally mutually-exclusive ``--driver`` / ``--gateware`` flags; they are now ``build``/``deploy`` subcommands @@ -16,16 +14,14 @@ Conventions: * Cases A-H cover ``peripherals build``. * Cases I-L cover ``peripherals deploy`` (incl. ``--package`` interaction). - * Cases M-O exercise AC-12: the combined / half-flagged ``.deb`` staging - layout. They consume ``_expected_so_filename`` / ``_expected_bit_filename`` - for symmetric naming derived from the manifest. - * Cases P1-P7 exercise AC-15: ``_expected_bit_filename`` fallback chain - (gateware_target -> .so stem -> manifest.name). + * Cases M-O exercise the two-deb staging layout under real packagers + fake fpm. + * Cases Q-R exercise two-deb deploy streaming and error paths. Mocking strategy mirrors the prior Tester (``test_gateware_runner.py``): * ``synapse.cli.peripherals.subprocess.run`` -> recorder * ``synapse.cli.peripherals.build_peripheral_so`` -> recorder returning True * ``synapse.cli.peripherals.build_peripheral_deb`` -> recorder returning True + * ``synapse.cli.peripherals.build_gateware_deb`` -> recorder returning True * ``synapse.cli.peripherals.gateware.run_gateware_build`` -> recorder returning the path of a fake ``.bit`` created under ``tmp_path`` * ``synapse.cli.peripherals.deploy_package`` -> recorder @@ -48,6 +44,8 @@ import pytest +from synapse.tests.cli.conftest import fake_fpm_run + # --------------------------------------------------------------------------- # Helpers / fixtures @@ -147,6 +145,7 @@ def _install_common_stubs(peripherals, monkeypatch, tmp_path, *, fake_bit=None): build_so_calls=[], run_gateware_calls=[], build_deb_calls=[], + build_gateware_deb_calls=[], subprocess_calls=[], deploy_calls=[], ) @@ -159,14 +158,22 @@ def fake_build_deb(*args, **kwargs): recorders.build_deb_calls.append((args, kwargs)) return True + def fake_build_gateware_deb(*args, **kwargs): + recorders.build_gateware_deb_calls.append((args, kwargs)) + return True + def fake_run_gateware(*args, **kwargs): recorders.run_gateware_calls.append((args, kwargs)) if fake_bit is not None: - return str(fake_bit) - # Drop a fake .bit somewhere predictable. - path = tmp_path / "fake.bit" - path.write_text("bit") - return str(path) + bit = str(fake_bit) + else: + path = tmp_path / "fake.bit" + path.write_text("bit") + bit = str(path) + stem, _ = os.path.splitext(bit) + with open(f"{stem}.summary.json", "w", encoding="utf-8") as fh: + json.dump({"project": {"name": "gateware", "usb_pid": 4}}, fh) + return bit def fake_subprocess_run(argv, *args, **kwargs): recorders.subprocess_calls.append( @@ -179,6 +186,7 @@ def fake_deploy_package(uri, deb_path): monkeypatch.setattr(peripherals, "build_peripheral_so", fake_build_so) monkeypatch.setattr(peripherals, "build_peripheral_deb", fake_build_deb) + monkeypatch.setattr(peripherals, "build_gateware_deb", fake_build_gateware_deb) monkeypatch.setattr(peripherals, "ensure_docker", lambda: True) monkeypatch.setattr( peripherals, @@ -191,11 +199,13 @@ def fake_deploy_package(uri, deb_path): monkeypatch.setattr(peripherals.subprocess, "run", fake_subprocess_run) monkeypatch.setattr(peripherals, "deploy_package", fake_deploy_package) - # find_deb_package is called after build_peripheral_deb succeeds. + # find_deb_package is called per deb with the package name. monkeypatch.setattr( peripherals, "find_deb_package", - lambda dist_dir: os.path.join(dist_dir, "fake_arm64.deb"), + lambda dist_dir, package_name=None: os.path.join( + dist_dir, f"{package_name or 'fake'}_arm64.deb" + ), ) # gateware sub-module attribute on peripherals must expose run_gateware_build. @@ -259,7 +269,10 @@ def test_case_A_build_no_flag_runs_both_halves(peripherals, tmp_path, monkeypatc assert len(recorders.build_so_calls) == 1, "driver builder should run once" assert len(recorders.run_gateware_calls) == 1, "gateware runner should run once" - assert len(recorders.build_deb_calls) == 1, ".deb staging should run once" + assert len(recorders.build_deb_calls) == 1, "driver .deb staging should run once" + assert len(recorders.build_gateware_deb_calls) == 1, ( + "gateware .deb staging should run once" + ) # --- Case A2: `both` subcommand parses to half="both" on build and deploy -- @@ -312,6 +325,9 @@ def test_case_B_build_driver_skips_gateware(peripherals, tmp_path, monkeypatch): "--driver must not invoke the gateware runner" ) assert len(recorders.build_deb_calls) == 1 + assert recorders.build_gateware_deb_calls == [], ( + "driver-only build must not stage a gateware deb" + ) # --- Case C: --gateware -> gateware half only ------------------------------ @@ -328,7 +344,10 @@ def test_case_C_build_gateware_skips_driver(peripherals, tmp_path, monkeypatch): "--gateware must not invoke the driver builder" ) assert len(recorders.run_gateware_calls) == 1 - assert len(recorders.build_deb_calls) == 1 + assert recorders.build_deb_calls == [], ( + "gateware-only build must not stage the driver deb" + ) + assert len(recorders.build_gateware_deb_calls) == 1 # --- Case D: build with an invalid target -> argparse rejects -------------- @@ -386,6 +405,7 @@ def test_case_D2_build_no_target_prints_help_and_builds_nothing( assert recorders.build_so_calls == [] assert recorders.run_gateware_calls == [] assert recorders.build_deb_calls == [] + assert recorders.build_gateware_deb_calls == [] captured = capsys.readouterr() assert "driver" in captured.out and "gateware" in captured.out, ( f"bare `build` should print help listing the targets; got: {captured.out!r}" @@ -692,17 +712,12 @@ def test_case_L_deploy_package_short_circuit_ignores_half_flag( # =========================================================================== -# AC-12: combined .deb composition (no-flag + half-flagged staging layout) +# Two-deb staging layout (driver deb + -gateware deb) # =========================================================================== -# Probe shape: spy on ``tempfile.mkdtemp`` (called from inside -# build_peripheral_deb to create its staging dir), then walk the dir after -# the function returns. AC-12 broadens the signature to -# (peripheral_dir, manifest_dict, *, bit_path=None, include_driver_runtime=True); -# the tests below feed the manifest through ``build_cmd``'s normal codepath, -# so as long as ``build_cmd`` itself is updated to pass the manifest in (also -# AC-12 -- "Both call sites updated"), this exercises the real public surface. +# These cases run the REAL packagers under a fake fpm stub so the staging +# layout written to disk can be asserted on directly. def _captured_staging_files(staging_dir): @@ -738,272 +753,214 @@ def _seed_runtime_libs_under(peripheral_dir): fh.write("fake-runtime-lib") -def test_case_M_combined_deb_carries_both_so_and_bit( +def test_case_M_both_emits_driver_deb_and_gateware_deb( peripherals, tmp_path, monkeypatch ): - """M (AC-12): no-flag .deb stages BOTH the .so and the .bit. - - Staging layout per AC-12 Public Interface Contract: - usr/lib/scifi/plugins/.so - usr/lib/scifi/gateware/.bit - usr/lib/libscifi-peripheral-sdk.so* - - Where = splitext(_expected_so_filename(manifest))[0] - and = splitext(_expected_bit_filename(manifest))[0]. - For this manifest, both stems resolve to "intan_rhd2132". - """ + """M: ``build both`` stages a driver-only deb AND a separate gateware deb.""" pd = _make_peripheral_dir(tmp_path) _seed_runtime_libs_under(pd) fake_bit = tmp_path / "fake.bit" fake_bit.write_text("BITSTREAM") - staging_holder: dict = {} + staging_dirs: list = [] real_mkdtemp = peripherals.tempfile.mkdtemp def spy_mkdtemp(*args, **kwargs): d = real_mkdtemp(*args, **kwargs) - staging_holder["dir"] = d + staging_dirs.append(d) return d - # Capture the real build_peripheral_deb BEFORE installing the common stubs - # (which would otherwise replace it with a recorder). - real_deb = peripherals.build_peripheral_deb + # Capture the real packagers BEFORE the common stubs replace them. + real_driver_deb = peripherals.build_peripheral_deb + real_gateware_deb = peripherals.build_gateware_deb monkeypatch.setattr(peripherals.tempfile, "mkdtemp", spy_mkdtemp) _install_common_stubs(peripherals, monkeypatch, tmp_path, fake_bit=fake_bit) - # Allow the real build_peripheral_deb to run (not the common stub). - monkeypatch.setattr(peripherals, "build_peripheral_deb", real_deb) - # And stub the fpm subprocess call so it's a no-op. + monkeypatch.setattr(peripherals, "build_peripheral_deb", real_driver_deb) + monkeypatch.setattr(peripherals, "build_gateware_deb", real_gateware_deb) + # Fake fpm so each packager's "did a .deb land?" verification passes. + calls: list = [] monkeypatch.setattr( peripherals.subprocess, "run", - lambda *a, **kw: subprocess.CompletedProcess([], 0, b"", b""), + fake_fpm_run(os.path.join(str(pd), "dist"), calls), ) peripherals.build_cmd(_build_args(pd, half="both")) - staging_dir = staging_holder.get("dir") - assert staging_dir is not None, "build_peripheral_deb must have run" + assert len(staging_dirs) == 2, ( + f"one staging dir per deb (driver, then gateware); got {staging_dirs!r}" + ) + driver_files = _captured_staging_files(staging_dirs[0]) + gateware_files = _captured_staging_files(staging_dirs[1]) - # Derive expected basenames from the manifest via the public helpers so - # the assertion stays symmetric and survives a manifest swap. - manifest = json.loads((pd / "manifest.json").read_text()) - expected_so = peripherals._expected_so_filename(manifest) - expected_bit = peripherals._expected_bit_filename(manifest) - assert expected_so == "intan_rhd2132.so", ( - f"sanity: helper should derive 'intan_rhd2132.so' for this manifest, " - f"got {expected_so!r}" + assert any( + f.endswith(os.path.join("usr/lib/scifi/plugins", "intan_rhd2132.so")) + for f in driver_files + ), f"driver deb stages the .so; got: {driver_files!r}" + assert any("libscifi-peripheral-sdk" in f for f in driver_files), ( + f"driver deb carries the SDK runtime; got: {driver_files!r}" ) - assert expected_bit == "intan_rhd2132.bit", ( - f"sanity: helper should derive 'intan_rhd2132.bit' (from .so stem) " - f"for this manifest with no gateware_target; got {expected_bit!r}" + assert not any(f.endswith(".bit") for f in driver_files), ( + f"driver deb must not carry the bitstream; got: {driver_files!r}" ) - files = _captured_staging_files(staging_dir) assert any( - f.endswith(os.path.join("usr/lib/scifi/plugins", expected_so)) for f in files - ), f".so should be staged at usr/lib/scifi/plugins/{expected_so}; got: {files!r}" + f.endswith(os.path.join("opt/scifi/bitstreams/custom", "intan_rhd2132.bit")) + for f in gateware_files + ), f"gateware deb stages the bit under custom/; got: {gateware_files!r}" assert any( - f.endswith(os.path.join("usr/lib/scifi/gateware", expected_bit)) for f in files - ), f".bit should be staged at usr/lib/scifi/gateware/{expected_bit}; got: {files!r}" - # Runtime libs must also be present in the combined .deb. - assert any("libscifi-peripheral-sdk" in f for f in files), ( - f"combined .deb must carry libscifi-peripheral-sdk runtime; got: {files!r}" + f.endswith( + os.path.join("opt/scifi/bitstreams/custom", "intan_rhd2132.manifest.json") + ) + for f in gateware_files + ), f"gateware deb stages the manifest fragment; got: {gateware_files!r}" + assert not any(f.endswith(".so") for f in gateware_files), ( + f"gateware deb must not carry any .so; got: {gateware_files!r}" ) + # Both fpm invocations happened, with distinct package names. + fpm_names = [] + for c in calls: + if "fpm" in c: + fpm_argv = c[c.index("fpm"):] + fpm_names.append(fpm_argv[fpm_argv.index("-n") + 1]) + assert fpm_names == ["intan_rhd2132", "intan_rhd2132-gateware"] -def test_case_N_driver_only_deb_carries_so_but_no_bit( + +def test_case_N_driver_deb_fpm_input_excludes_postinstall( peripherals, tmp_path, monkeypatch ): - """N (AC-12): ``--driver`` .deb has .so + runtime libs, NOT .bit.""" + """N: the driver deb's fpm input is ``usr`` — /postinstall.sh must not ship + as payload (it would dpkg-conflict with the gateware deb's).""" pd = _make_peripheral_dir(tmp_path) _seed_runtime_libs_under(pd) - real_mkdtemp = peripherals.tempfile.mkdtemp - holder: dict = {} - - def spy_mkdtemp(*args, **kwargs): - d = real_mkdtemp(*args, **kwargs) - holder["dir"] = d - return d - - # Capture the real build_peripheral_deb BEFORE installing the common stubs. - real_deb = peripherals.build_peripheral_deb - monkeypatch.setattr(peripherals.tempfile, "mkdtemp", spy_mkdtemp) + real_driver_deb = peripherals.build_peripheral_deb _install_common_stubs(peripherals, monkeypatch, tmp_path) + monkeypatch.setattr(peripherals, "build_peripheral_deb", real_driver_deb) + calls: list = [] monkeypatch.setattr( peripherals.subprocess, "run", - lambda *a, **kw: subprocess.CompletedProcess([], 0, b"", b""), + fake_fpm_run(os.path.join(str(pd), "dist"), calls), ) - # Don't stub build_peripheral_deb -- we want the real one to run. - monkeypatch.setattr(peripherals, "build_peripheral_deb", real_deb) peripherals.build_cmd(_build_args(pd, half="driver")) - staging_dir = holder.get("dir") - assert staging_dir is not None - files = _captured_staging_files(staging_dir) - - manifest = json.loads((pd / "manifest.json").read_text()) - expected_so = peripherals._expected_so_filename(manifest) - assert any( - f.endswith(os.path.join("usr/lib/scifi/plugins", expected_so)) for f in files - ), ( - f"--driver .deb should stage .so at usr/lib/scifi/plugins/{expected_so}; got: {files!r}" - ) - assert not any(f.endswith(".bit") for f in files), ( - f"--driver .deb must not carry any .bit; got: {files!r}" - ) - # Runtime libs MUST be present under --driver. - assert any("libscifi-peripheral-sdk" in f for f in files), ( - f"--driver .deb must carry libscifi-peripheral-sdk runtime; got: {files!r}" + fpm_call = next(c for c in calls if "fpm" in c) + assert fpm_call[-1] == "usr", ( + f"driver fpm input must be 'usr' so postinstall.sh isn't payload; " + f"got: {fpm_call!r}" ) + # postinstall.sh is still wired as the maintainer script. + assert fpm_call[fpm_call.index("--after-install") + 1] == "/pkg/postinstall.sh" -def test_case_O_gateware_only_deb_carries_bit_but_no_so_no_runtime( +def test_case_O_gateware_only_build_emits_only_gateware_deb( peripherals, tmp_path, monkeypatch ): - """O (AC-12): ``--gateware`` .deb has only the .bit; no .so, no runtime libs.""" + """O: ``build gateware`` stages ONLY the gateware deb (bit + fragment).""" pd = _make_peripheral_dir(tmp_path) _seed_runtime_libs_under(pd) fake_bit = tmp_path / "fake.bit" fake_bit.write_text("BITSTREAM") + staging_dirs: list = [] real_mkdtemp = peripherals.tempfile.mkdtemp - holder: dict = {} def spy_mkdtemp(*args, **kwargs): d = real_mkdtemp(*args, **kwargs) - holder["dir"] = d + staging_dirs.append(d) return d - # Capture the real build_peripheral_deb BEFORE installing the common stubs. - real_deb = peripherals.build_peripheral_deb + real_gateware_deb = peripherals.build_gateware_deb monkeypatch.setattr(peripherals.tempfile, "mkdtemp", spy_mkdtemp) _install_common_stubs(peripherals, monkeypatch, tmp_path, fake_bit=fake_bit) + monkeypatch.setattr(peripherals, "build_gateware_deb", real_gateware_deb) + calls: list = [] monkeypatch.setattr( peripherals.subprocess, "run", - lambda *a, **kw: subprocess.CompletedProcess([], 0, b"", b""), + fake_fpm_run(os.path.join(str(pd), "dist"), calls), ) - monkeypatch.setattr(peripherals, "build_peripheral_deb", real_deb) peripherals.build_cmd(_build_args(pd, half="gateware")) - staging_dir = holder.get("dir") - assert staging_dir is not None - files = _captured_staging_files(staging_dir) - - manifest = json.loads((pd / "manifest.json").read_text()) - expected_bit = peripherals._expected_bit_filename(manifest) + assert len(staging_dirs) == 1 + files = _captured_staging_files(staging_dirs[0]) assert any( - f.endswith(os.path.join("usr/lib/scifi/gateware", expected_bit)) for f in files - ), ( - f"--gateware .deb should stage .bit at usr/lib/scifi/gateware/{expected_bit}; got: {files!r}" - ) - assert not any(f.endswith(".so") for f in files), ( - f"--gateware .deb must not carry any .so; got: {files!r}" - ) - assert not any("libscifi-peripheral-sdk" in f for f in files), ( - f"--gateware .deb must not carry libscifi-peripheral-sdk runtime; " - f"got: {files!r}" - ) + f.endswith(os.path.join("opt/scifi/bitstreams/custom", "intan_rhd2132.bit")) + for f in files + ), f"gateware deb stages the bit; got: {files!r}" + with open( + os.path.join( + staging_dirs[0], + "opt", "scifi", "bitstreams", "custom", "intan_rhd2132.manifest.json", + ), + "r", + encoding="utf-8", + ) as fh: + frag = json.load(fh) + assert frag == { + "name": "intan_rhd2132", + "usb_pid": 4, + "artifact": "custom/intan_rhd2132.bit", + } + assert not any(f.endswith(".so") for f in files) + assert not any("libscifi-peripheral-sdk" in f for f in files) # =========================================================================== -# AC-15: _expected_bit_filename helper + install.gateware_target manifest field +# Two-deb deploy + summary error path # =========================================================================== -# Fallback chain per AC-15 + AC-12 helper contract (read in the plan): -# 1. install.gateware_target present and truthy -> basename(gateware_target) -# 2. else if install.target present -> splitext(basename(target))[0] + ".bit" -# 3. else -> ".bit" -# -# An empty-string gateware_target ("") is falsy by the `if target:` guard -# in the contract pseudocode in the plan, so it falls through to step 2/3. -# Cases P1..P7 cover each branch + a few adversarial corners. - - -def test_case_P1_expected_bit_filename_uses_explicit_gateware_target(peripherals): - """P1 (AC-15): install.gateware_target set -> returns its basename verbatim.""" - manifest = { - "name": "scifi-intan-rhd2132", - "install": { - "target": "/usr/lib/scifi/plugins/intan_rhd2132.so", - "gateware_target": "/usr/lib/scifi/gateware/intan_rhd2132.bit", - }, - } - assert peripherals._expected_bit_filename(manifest) == "intan_rhd2132.bit" - - -def test_case_P2_expected_bit_filename_falls_back_to_so_stem(peripherals): - """P2 (AC-15): no gateware_target -> derive from .so stem. - - For the current axon-peripheral-example manifest -- name - "scifi-intan-rhd2132" but install.target = ".../intan_rhd2132.so" -- - the .bit must be "intan_rhd2132.bit" (NOT "scifi-intan-rhd2132.bit"). - This is the symmetry-with-.so rule from the AC-12 rationale. - """ - manifest = { - "name": "scifi-intan-rhd2132", - "install": {"target": "/usr/lib/scifi/plugins/intan_rhd2132.so"}, - } - assert peripherals._expected_bit_filename(manifest) == "intan_rhd2132.bit" - - -def test_case_P3_expected_bit_filename_falls_back_to_manifest_name(peripherals): - """P3 (AC-15): no install.target AND no gateware_target -> ".bit".""" - manifest = {"name": "scifi-intan-rhd2132"} - assert peripherals._expected_bit_filename(manifest) == "scifi-intan-rhd2132.bit" +def test_case_Q_deploy_both_streams_two_debs(peripherals, tmp_path, monkeypatch): + """Q: ``deploy both`` makes two DeployApp calls — driver deb first.""" + pd = _make_peripheral_dir(tmp_path) + recorders = _install_common_stubs(peripherals, monkeypatch, tmp_path) + peripherals.deploy_cmd(_deploy_args(pd, half="both", uri="10.0.0.1")) -def test_case_P4_expected_bit_filename_empty_install_block(peripherals): - """P4 (AC-15): install = {} (empty block) + name = "foo" -> "foo.bit".""" - manifest = {"name": "foo", "install": {}} - assert peripherals._expected_bit_filename(manifest) == "foo.bit" + assert len(recorders.deploy_calls) == 2, "one DeployApp stream per deb" + uris = [u for u, _ in recorders.deploy_calls] + paths = [p for _, p in recorders.deploy_calls] + assert uris == ["10.0.0.1", "10.0.0.1"] + assert paths[0].endswith("intan_rhd2132_arm64.deb") + assert paths[1].endswith("intan_rhd2132-gateware_arm64.deb") -def test_case_P5_expected_bit_filename_no_install_key(peripherals): - """P5 (AC-15): no install key at all + name = "bar" -> "bar.bit".""" - manifest = {"name": "bar"} - assert peripherals._expected_bit_filename(manifest) == "bar.bit" +def test_case_R_gateware_build_aborts_without_usb_pid( + peripherals, tmp_path, monkeypatch, capsys +): + """R: a bitstream with no .summary.json aborts BEFORE deb staging.""" + pd = _make_peripheral_dir(tmp_path) + # Bit WITHOUT a sibling summary: bypass the harness's summary-writing stub. + bare_bit = tmp_path / "bare.bit" + bare_bit.write_text("bit") + recorders = _install_common_stubs(peripherals, monkeypatch, tmp_path) + def fake_run_gateware_no_summary(*args, **kwargs): + recorders.run_gateware_calls.append((args, kwargs)) + return str(bare_bit) -def test_case_P6_expected_bit_filename_trusts_unusual_basename(peripherals): - """P6 (AC-15): gateware_target with a non-derived basename is honored verbatim. + gateware_mod = importlib.import_module("synapse.cli.gateware") + monkeypatch.setattr( + gateware_mod, "run_gateware_build", fake_run_gateware_no_summary + ) + monkeypatch.setattr( + peripherals.gateware, "run_gateware_build", fake_run_gateware_no_summary + ) - The helper does not enforce that the .bit stem match the .so stem; the - user may intentionally pick a different name. Per AC-15: - "no cross-field constraint (the user may, intentionally or not, pick - different stems for the two artifacts)." - """ - manifest = { - "name": "scifi-intan-rhd2132", - "install": { - "target": "/usr/lib/scifi/plugins/intan_rhd2132.so", - "gateware_target": "/usr/lib/scifi/gateware/my_custom_name.bit", - }, - } - assert peripherals._expected_bit_filename(manifest) == "my_custom_name.bit" + peripherals.build_cmd(_build_args(pd, half="gateware")) + assert recorders.build_gateware_deb_calls == [], ( + "no gateware deb staging without a usb_pid" + ) + out = capsys.readouterr().out.lower() + assert "summary" in out -def test_case_P7_expected_bit_filename_empty_gateware_target_falls_through( - peripherals, -): - """P7 (AC-15, adversarial): empty-string gateware_target is treated as "not set". - The AC-12 helper contract guards with ``if target:`` -- an empty string is - falsy and we fall through to the .so-stem fallback. This guards against a - user accidentally writing ``"gateware_target": ""`` and getting a - confusing ``"".bit`` artifact. - """ - manifest = { - "name": "scifi-intan-rhd2132", - "install": { - "target": "/usr/lib/scifi/plugins/intan_rhd2132.so", - "gateware_target": "", - }, - } - # Empty-string falls through; .so stem wins. - assert peripherals._expected_bit_filename(manifest) == "intan_rhd2132.bit" +# --------------------------------------------------------------------------- +# (end of file — P1-P7 / _expected_bit_filename tests removed with that helper) +# --------------------------------------------------------------------------- From c127247d617658dbb325e966dfffe394fdd0d3b7 Mon Sep 17 00:00:00 2001 From: calvinleng-science Date: Wed, 10 Jun 2026 13:35:51 -0700 Subject: [PATCH 14/21] fix(peripherals): stop deploy after a failed package stream --- synapse/cli/deploy.py | 12 +- synapse/cli/peripherals.py | 431 +++++++++++------------ synapse/tests/cli/test_half_selectors.py | 22 ++ 3 files changed, 247 insertions(+), 218 deletions(-) diff --git a/synapse/cli/deploy.py b/synapse/cli/deploy.py index 26525573..84b3f36d 100644 --- a/synapse/cli/deploy.py +++ b/synapse/cli/deploy.py @@ -71,7 +71,13 @@ def create_metadata(file_path, console): def deploy_package(ip_address, deb_package_path): - """Deploy the package to the device""" + """Deploy a .deb package to the device via gRPC DeployApp streaming. + + Returns True when the package was streamed and all device responses were + received without error; False on any failure (connection, gRPC, or I/O). + Callers can use the return value to decide whether to continue with + subsequent packages. + """ package_filename = os.path.basename(deb_package_path) console.clear_live() @@ -178,6 +184,7 @@ def chunk_generator(): response_panel.renderable = Group(*display_items) response_panel.border_style = "red" live.refresh() + return False except Exception as e: # For the outer exception, also preserve any progress made @@ -195,6 +202,9 @@ def chunk_generator(): response_panel.renderable = Group(*display_items) response_panel.border_style = "red" live.refresh() + return False + + return True def deploy_cmd(args): diff --git a/synapse/cli/peripherals.py b/synapse/cli/peripherals.py index 5d4288b8..7a9a5dd3 100644 --- a/synapse/cli/peripherals.py +++ b/synapse/cli/peripherals.py @@ -415,163 +415,159 @@ def build_peripheral_deb( so_filename = _expected_so_filename(manifest) staging_dir = tempfile.mkdtemp(prefix="synapse-peripheral-package-") - try: - # 1. Stage the plugin .so at /usr/lib/scifi/plugins/.so - if not os.path.exists(so_path): + # Leave staging_dir on disk for inspection if something goes wrong; + # /tmp eventually cleans itself. + + # 1. Stage the plugin .so at /usr/lib/scifi/plugins/.so + if not os.path.exists(so_path): + console.print( + f"[bold red]Error:[/bold red] Plugin .so not found at {so_path}" + ) + return False + plugin_dst = os.path.join(staging_dir, "usr", "lib", "scifi", "plugins") + os.makedirs(plugin_dst, exist_ok=True) + shutil.copy2(so_path, os.path.join(plugin_dst, so_filename)) + + # 2. Stage libscifi-peripheral-sdk.so* from the builder image at /usr/lib. + # The SDK ships via `apt-get install scifi-peripheral-sdk` inside the + # builder Dockerfile, so it's the same source the linker resolved against + # at build time — guaranteeing ABI alignment for the plugin. + sdk_dst = os.path.join(staging_dir, "usr", "lib") + os.makedirs(sdk_dst, exist_ok=True) + + # Prefer libs already produced on disk next to the .so (the driver + # builder may stage them there). Fall back to extracting from the + # builder image only if none are present locally. + local_libs_dir = os.path.join(peripheral_dir, "build", "aarch64") + local_libs = ( + [ + f + for f in os.listdir(local_libs_dir) + if f.startswith("libscifi-peripheral-sdk.so") + ] + if os.path.isdir(local_libs_dir) + else [] + ) + if local_libs: + for fname in local_libs: + shutil.copy2( + os.path.join(local_libs_dir, fname), + os.path.join(sdk_dst, fname), + ) + else: + try: + image_tag = build_docker_image( + peripheral_dir, "axon-peripheral", roles=["driver"] + )["driver"] + except ( + subprocess.CalledProcessError, + FileNotFoundError, + KeyError, + ) as exc: console.print( - f"[bold red]Error:[/bold red] Plugin .so not found at {so_path}" + f"[bold red]Error:[/bold red] Failed to build driver Docker image: {exc}" ) return False - plugin_dst = os.path.join(staging_dir, "usr", "lib", "scifi", "plugins") - os.makedirs(plugin_dst, exist_ok=True) - shutil.copy2(so_path, os.path.join(plugin_dst, so_filename)) - - # 2. Stage libscifi-peripheral-sdk.so* from the builder image at /usr/lib. - # The SDK ships via `apt-get install scifi-peripheral-sdk` inside the - # builder Dockerfile, so it's the same source the linker resolved against - # at build time — guaranteeing ABI alignment for the plugin. - sdk_dst = os.path.join(staging_dir, "usr", "lib") - os.makedirs(sdk_dst, exist_ok=True) - - # Prefer libs already produced on disk next to the .so (the driver - # builder may stage them there). Fall back to extracting from the - # builder image only if none are present locally. - local_libs_dir = os.path.join(peripheral_dir, "build", "aarch64") - local_libs = ( - [ - f - for f in os.listdir(local_libs_dir) - if f.startswith("libscifi-peripheral-sdk.so") - ] - if os.path.isdir(local_libs_dir) - else [] + arch_suffix = detect_arch() + platform_opt = ( + "linux/arm64" if arch_suffix == "arm64" else "linux/amd64" ) - if local_libs: - for fname in local_libs: - shutil.copy2( - os.path.join(local_libs_dir, fname), - os.path.join(sdk_dst, fname), - ) - else: - try: - image_tag = build_docker_image( - peripheral_dir, "axon-peripheral", roles=["driver"] - )["driver"] - except ( - subprocess.CalledProcessError, - FileNotFoundError, - KeyError, - ) as exc: - console.print( - f"[bold red]Error:[/bold red] Failed to build driver Docker image: {exc}" - ) - return False - arch_suffix = detect_arch() - platform_opt = ( - "linux/arm64" if arch_suffix == "arm64" else "linux/amd64" - ) + console.print( + f"[yellow]Extracting SDK runtime from Docker image [bold]{image_tag}[/bold]...[/yellow]" + ) + extract_cmd = [ + "docker", + "run", + "--rm", + "--platform", + platform_opt, + "-v", + f"{sdk_dst}:/out", + image_tag, + "/bin/bash", + "-c", + r"find /usr/lib -maxdepth 1 -name 'libscifi-peripheral-sdk.so*' -exec cp -a {} /out/ \;", + ] + try: + subprocess.run(extract_cmd, check=True) + except subprocess.CalledProcessError as exc: console.print( - f"[yellow]Extracting SDK runtime from Docker image [bold]{image_tag}[/bold]...[/yellow]" - ) - extract_cmd = [ - "docker", - "run", - "--rm", - "--platform", - platform_opt, - "-v", - f"{sdk_dst}:/out", - image_tag, - "/bin/bash", - "-c", - r"find /usr/lib -maxdepth 1 -name 'libscifi-peripheral-sdk.so*' -exec cp -a {} /out/ \;", - ] - try: - subprocess.run(extract_cmd, check=True) - except subprocess.CalledProcessError as exc: - console.print( - f"[bold red]Error:[/bold red] Failed to extract SDK runtime: {exc}" - ) - return False - - sdk_files = [ - f - for f in os.listdir(sdk_dst) - if f.startswith("libscifi-peripheral-sdk.so") - ] - if not sdk_files: - console.print( - "[bold red]Error:[/bold red] SDK runtime libraries not found in builder image. " - "Make sure your Dockerfile installs scifi-peripheral-sdk." - ) - return False - - # 3. Postinstall: nudge the user to restart scifi-server. - # Restarting automatically could interrupt an active recording session, - # so leave it manual. - postinstall_path = os.path.join(staging_dir, "postinstall.sh") - with open(postinstall_path, "w", encoding="utf-8") as fp: - fp.write( - "#!/bin/bash\n" - "set -e\n" - "echo 'Peripheral plugin installed. Restart scifi-server to load it.'\n" - "exit 0\n" + f"[bold red]Error:[/bold red] Failed to extract SDK runtime: {exc}" ) - # 0o644 is sufficient: fpm embeds this file's *contents* as the .deb's - # postinst maintainer script (via --after-install), and dpkg makes - # maintainer scripts executable itself at install time. The staging - # file's own exec bit never reaches the package. - os.chmod(postinstall_path, 0o644) - - # 4. Run fpm inside the cdrx/fpm-ubuntu image (matches apps' packaging path). - dist_dir = os.path.join(peripheral_dir, "dist") - os.makedirs(dist_dir, exist_ok=True) - - fpm_args = [ - "fpm", - "-s", - "dir", - "-t", - "deb", - "-n", - plugin_name, - "-f", - "-v", - version, - "-C", - "/pkg", - "--deb-no-default-config-files", - "--vendor", - "Science Corporation", - "--description", - "Synapse peripheral plugin", - "--architecture", - "arm64", - "--category", - SECTION_LABEL, - "--after-install", - "/pkg/postinstall.sh", - # Input is "usr" (not ".") so postinstall.sh is NOT packaged as a - # payload file — the -gateware deb installs alongside this one, - # and two packages shipping /postinstall.sh would dpkg-conflict. - "usr", + return False + + sdk_files = [ + f + for f in os.listdir(sdk_dst) + if f.startswith("libscifi-peripheral-sdk.so") ] + if not sdk_files: + console.print( + "[bold red]Error:[/bold red] SDK runtime libraries not found in builder image. " + "Make sure your Dockerfile installs scifi-peripheral-sdk." + ) + return False - console.print( - f"[yellow]Packaging plugin .deb (Docker image: {FPM_IMAGE}) ...[/yellow]" + # 3. Postinstall: nudge the user to restart scifi-server. + # Restarting automatically could interrupt an active recording session, + # so leave it manual. + postinstall_path = os.path.join(staging_dir, "postinstall.sh") + with open(postinstall_path, "w", encoding="utf-8") as fp: + fp.write( + "#!/bin/bash\n" + "set -e\n" + "echo 'Peripheral plugin installed. Restart scifi-server to load it.'\n" + "exit 0\n" ) - if not _run_fpm(staging_dir, dist_dir, fpm_args, plugin_name): - return False + # 0o644 is sufficient: fpm embeds this file's *contents* as the .deb's + # postinst maintainer script (via --after-install), and dpkg makes + # maintainer scripts executable itself at install time. The staging + # file's own exec bit never reaches the package. + os.chmod(postinstall_path, 0o644) - console.print("[green]Plugin .deb created successfully![/green]") - return True + # 4. Run fpm inside the cdrx/fpm-ubuntu image (matches apps' packaging path). + dist_dir = os.path.join(peripheral_dir, "dist") + os.makedirs(dist_dir, exist_ok=True) + + fpm_args = [ + "fpm", + "-s", + "dir", + "-t", + "deb", + "-n", + plugin_name, + "-f", + "-v", + version, + "-C", + "/pkg", + "--deb-no-default-config-files", + "--vendor", + "Science Corporation", + "--description", + "Synapse peripheral plugin", + "--architecture", + "arm64", + "--category", + SECTION_LABEL, + "--after-install", + "/pkg/postinstall.sh", + # Input is "usr" (not ".") so postinstall.sh is NOT packaged as a + # payload file — the -gateware deb installs alongside this one, + # and two packages shipping /postinstall.sh would dpkg-conflict. + "usr", + ] - except subprocess.CalledProcessError as exc: - console.print(f"[bold red]Error:[/bold red] fpm failed: {exc}") + console.print( + f"[yellow]Packaging plugin .deb (Docker image: {FPM_IMAGE}) ...[/yellow]" + ) + if not _run_fpm(staging_dir, dist_dir, fpm_args, plugin_name): return False - # Leave staging_dir on disk for inspection if something goes wrong; - # /tmp eventually cleans itself. + + console.print("[green]Plugin .deb created successfully![/green]") + return True # Suffix appended to the plugin name to form the gateware package name. @@ -610,83 +606,79 @@ def build_gateware_deb( return False staging_dir = tempfile.mkdtemp(prefix="synapse-gateware-package-") - try: - custom_dir = os.path.join(staging_dir, "opt", "scifi", "bitstreams", "custom") - os.makedirs(custom_dir, exist_ok=True) - shutil.copy2(bit_path, os.path.join(custom_dir, f"{plugin_name}.bit")) - - fragment = { - "name": plugin_name, - "usb_pid": usb_pid, - "artifact": f"custom/{plugin_name}.bit", - } - fragment_path = os.path.join(custom_dir, f"{plugin_name}.manifest.json") - with open(fragment_path, "w", encoding="utf-8") as fp: - json.dump(fragment, fp, indent=2) - fp.write("\n") - - postinstall_path = os.path.join(staging_dir, "postinstall.sh") - with open(postinstall_path, "w", encoding="utf-8") as fp: - fp.write( - "#!/bin/bash\n" - "set -e\n" - "echo 'Custom gateware installed. Flash probes from the device " - "UI (Probe Updates) or scifi-probe-updater.'\n" - "exit 0\n" - ) - # Contents are embedded as the deb's postinst (via --after-install); - # dpkg makes maintainer scripts executable itself. - os.chmod(postinstall_path, 0o644) - - dist_dir = os.path.join(peripheral_dir, "dist") - os.makedirs(dist_dir, exist_ok=True) - - # Input is "opt" (not ".") so postinstall.sh is NOT packaged as a - # payload file — the driver deb installs alongside this one, and two - # packages shipping /postinstall.sh would dpkg-conflict. - fpm_args = [ - "fpm", - "-s", - "dir", - "-t", - "deb", - "-n", - package_name, - "-f", - "-v", - version, - "-C", - "/pkg", - "--deb-no-default-config-files", - "--vendor", - "Science Corporation", - "--description", - "Synapse peripheral custom gateware", - "--architecture", - "arm64", - "--category", - SECTION_LABEL, - "--depends", - BITSTREAMS_PACKAGE, - "--after-install", - "/pkg/postinstall.sh", - "opt", - ] + # Leave staging_dir on disk for inspection if something goes wrong; + # /tmp eventually cleans itself. - console.print( - f"[yellow]Packaging gateware .deb (Docker image: {FPM_IMAGE}) ...[/yellow]" + custom_dir = os.path.join(staging_dir, "opt", "scifi", "bitstreams", "custom") + os.makedirs(custom_dir, exist_ok=True) + shutil.copy2(bit_path, os.path.join(custom_dir, f"{plugin_name}.bit")) + + fragment = { + "name": plugin_name, + "usb_pid": usb_pid, + "artifact": f"custom/{plugin_name}.bit", + } + fragment_path = os.path.join(custom_dir, f"{plugin_name}.manifest.json") + with open(fragment_path, "w", encoding="utf-8") as fp: + json.dump(fragment, fp, indent=2) + fp.write("\n") + + postinstall_path = os.path.join(staging_dir, "postinstall.sh") + with open(postinstall_path, "w", encoding="utf-8") as fp: + fp.write( + "#!/bin/bash\n" + "set -e\n" + "echo 'Custom gateware installed. Flash probes from the device " + "UI (Probe Updates) or scifi-probe-updater.'\n" + "exit 0\n" ) - if not _run_fpm(staging_dir, dist_dir, fpm_args, package_name): - return False + # Contents are embedded as the deb's postinst (via --after-install); + # dpkg makes maintainer scripts executable itself. + os.chmod(postinstall_path, 0o644) - console.print("[green]Gateware .deb created successfully![/green]") - return True + dist_dir = os.path.join(peripheral_dir, "dist") + os.makedirs(dist_dir, exist_ok=True) + + # Input is "opt" (not ".") so postinstall.sh is NOT packaged as a + # payload file — the driver deb installs alongside this one, and two + # packages shipping /postinstall.sh would dpkg-conflict. + fpm_args = [ + "fpm", + "-s", + "dir", + "-t", + "deb", + "-n", + package_name, + "-f", + "-v", + version, + "-C", + "/pkg", + "--deb-no-default-config-files", + "--vendor", + "Science Corporation", + "--description", + "Synapse peripheral custom gateware", + "--architecture", + "arm64", + "--category", + SECTION_LABEL, + "--depends", + BITSTREAMS_PACKAGE, + "--after-install", + "/pkg/postinstall.sh", + "opt", + ] - except subprocess.CalledProcessError as exc: - console.print(f"[bold red]Error:[/bold red] fpm failed: {exc}") + console.print( + f"[yellow]Packaging gateware .deb (Docker image: {FPM_IMAGE}) ...[/yellow]" + ) + if not _run_fpm(staging_dir, dist_dir, fpm_args, package_name): return False - # Leave staging_dir on disk for inspection if something goes wrong; - # /tmp eventually cleans itself. + + console.print("[green]Gateware .deb created successfully![/green]") + return True # --------------------------------------------------------------------------- @@ -913,7 +905,12 @@ def deploy_cmd(args) -> None: return for deb in deb_packages: - deploy_package(args.uri, deb) + if not deploy_package(args.uri, deb): + console.print( + f"[bold red]Error:[/bold red] Deploy failed for {deb}; " + "skipping any remaining packages." + ) + return # --------------------------------------------------------------------------- diff --git a/synapse/tests/cli/test_half_selectors.py b/synapse/tests/cli/test_half_selectors.py index e4ea02d5..2d50f79b 100644 --- a/synapse/tests/cli/test_half_selectors.py +++ b/synapse/tests/cli/test_half_selectors.py @@ -183,6 +183,7 @@ def fake_subprocess_run(argv, *args, **kwargs): def fake_deploy_package(uri, deb_path): recorders.deploy_calls.append((uri, deb_path)) + return True monkeypatch.setattr(peripherals, "build_peripheral_so", fake_build_so) monkeypatch.setattr(peripherals, "build_peripheral_deb", fake_build_deb) @@ -930,6 +931,27 @@ def test_case_Q_deploy_both_streams_two_debs(peripherals, tmp_path, monkeypatch) assert paths[1].endswith("intan_rhd2132-gateware_arm64.deb") +def test_case_Q2_deploy_stops_after_failed_driver_deploy( + peripherals, tmp_path, monkeypatch, capsys +): + """Q2: if the driver deb deploy fails, the gateware deb is NOT streamed.""" + pd = _make_peripheral_dir(tmp_path) + recorders = _install_common_stubs(peripherals, monkeypatch, tmp_path) + + def failing_deploy(uri, deb_path): + recorders.deploy_calls.append((uri, deb_path)) + return False + + monkeypatch.setattr(peripherals, "deploy_package", failing_deploy) + + peripherals.deploy_cmd(_deploy_args(pd, half="both", uri="10.0.0.1")) + + assert len(recorders.deploy_calls) == 1, ( + "gateware deb must not stream after the driver deploy failed" + ) + assert "deploy failed" in capsys.readouterr().out.lower() + + def test_case_R_gateware_build_aborts_without_usb_pid( peripherals, tmp_path, monkeypatch, capsys ): From 21d7bf4e9aace2946c4781225926a84577780da0 Mon Sep 17 00:00:00 2001 From: calvinleng-science Date: Wed, 10 Jun 2026 13:37:07 -0700 Subject: [PATCH 15/21] test(peripherals): drop dead gateware_target fixtures --- synapse/tests/cli/test_half_selectors.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/synapse/tests/cli/test_half_selectors.py b/synapse/tests/cli/test_half_selectors.py index 2d50f79b..7bd5eb82 100644 --- a/synapse/tests/cli/test_half_selectors.py +++ b/synapse/tests/cli/test_half_selectors.py @@ -70,7 +70,6 @@ def _make_peripheral_dir( name: str = "intan_rhd2132", with_gateware: bool = True, with_install_target: bool = True, - with_gateware_target: bool = False, ): """Create a fake peripheral directory tree. @@ -93,8 +92,6 @@ def _make_peripheral_dir( install: dict = {} if with_install_target: install["target"] = f"/usr/lib/scifi/plugins/{name}.so" - if with_gateware_target: - install["gateware_target"] = f"/usr/lib/scifi/gateware/{name}.bit" manifest = {"name": name, "version": "0.1.0"} if install: manifest["install"] = install @@ -983,6 +980,3 @@ def fake_run_gateware_no_summary(*args, **kwargs): assert "summary" in out -# --------------------------------------------------------------------------- -# (end of file — P1-P7 / _expected_bit_filename tests removed with that helper) -# --------------------------------------------------------------------------- From 283b2b74ca9bb1c1692e6e8ebc95436d507f9337 Mon Sep 17 00:00:00 2001 From: calvinleng-science Date: Wed, 10 Jun 2026 14:05:00 -0700 Subject: [PATCH 16/21] fix(peripherals): anchor deb selection on name and version to skip stale builds --- synapse/cli/peripherals.py | 4 ++-- .../tests/cli/test_custom_gateware_packaging.py | 9 +++++++++ synapse/tests/cli/test_half_selectors.py | 15 +++++++++++---- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/synapse/cli/peripherals.py b/synapse/cli/peripherals.py index 7a9a5dd3..baceab87 100644 --- a/synapse/cli/peripherals.py +++ b/synapse/cli/peripherals.py @@ -780,7 +780,7 @@ def _build_debs( peripheral_dir, manifest, so_path=so_path, version=version ): return None - deb = find_deb_package(dist_dir, plugin_name) + deb = find_deb_package(dist_dir, f"{plugin_name}_{version}") if deb is None: return None debs.append(deb) @@ -796,7 +796,7 @@ def _build_debs( peripheral_dir, manifest, bit_path=bit_path, usb_pid=usb_pid, version=version ): return None - deb = find_deb_package(dist_dir, f"{plugin_name}{GATEWARE_DEB_SUFFIX}") + deb = find_deb_package(dist_dir, f"{plugin_name}{GATEWARE_DEB_SUFFIX}_{version}") if deb is None: return None debs.append(deb) diff --git a/synapse/tests/cli/test_custom_gateware_packaging.py b/synapse/tests/cli/test_custom_gateware_packaging.py index a3d16c3d..06b6c5b3 100644 --- a/synapse/tests/cli/test_custom_gateware_packaging.py +++ b/synapse/tests/cli/test_custom_gateware_packaging.py @@ -136,6 +136,15 @@ def test_find_deb_package_no_match_returns_none(buildmod, tmp_path, capsys): assert "could not find" in capsys.readouterr().out.lower() +def test_find_deb_package_version_anchored_prefix_skips_stale(buildmod, tmp_path): + # dist/ accumulates old versions; callers anchor the prefix with the + # version so a stale 0.1.0 deb never shadows the fresh 0.2.0 build. + (tmp_path / "via_0.1.0_arm64.deb").write_text("deb") + (tmp_path / "via_0.2.0_arm64.deb").write_text("deb") + found = buildmod.find_deb_package(str(tmp_path), "via_0.2.0") + assert found is not None and found.endswith(os.sep + "via_0.2.0_arm64.deb") + + # --------------------------------------------------------------------------- # peripherals.build_gateware_deb # --------------------------------------------------------------------------- diff --git a/synapse/tests/cli/test_half_selectors.py b/synapse/tests/cli/test_half_selectors.py index 7bd5eb82..7bb37679 100644 --- a/synapse/tests/cli/test_half_selectors.py +++ b/synapse/tests/cli/test_half_selectors.py @@ -197,7 +197,8 @@ def fake_deploy_package(uri, deb_path): monkeypatch.setattr(peripherals.subprocess, "run", fake_subprocess_run) monkeypatch.setattr(peripherals, "deploy_package", fake_deploy_package) - # find_deb_package is called per deb with the package name. + # find_deb_package is called per deb with the version-anchored prefix + # (e.g. "intan_rhd2132_0.1.0"), so the returned path carries the version. monkeypatch.setattr( peripherals, "find_deb_package", @@ -914,7 +915,13 @@ def spy_mkdtemp(*args, **kwargs): def test_case_Q_deploy_both_streams_two_debs(peripherals, tmp_path, monkeypatch): - """Q: ``deploy both`` makes two DeployApp calls — driver deb first.""" + """Q: ``deploy both`` makes two DeployApp calls — driver deb first. + + _build_debs anchors the find_deb_package lookup on ``_`` + (not just ````) so stale debs accumulated in dist/ across version + bumps can never shadow the freshly built one. The fixture manifest version + is "0.1.0", so both paths must carry that version component. + """ pd = _make_peripheral_dir(tmp_path) recorders = _install_common_stubs(peripherals, monkeypatch, tmp_path) @@ -924,8 +931,8 @@ def test_case_Q_deploy_both_streams_two_debs(peripherals, tmp_path, monkeypatch) uris = [u for u, _ in recorders.deploy_calls] paths = [p for _, p in recorders.deploy_calls] assert uris == ["10.0.0.1", "10.0.0.1"] - assert paths[0].endswith("intan_rhd2132_arm64.deb") - assert paths[1].endswith("intan_rhd2132-gateware_arm64.deb") + assert paths[0].endswith("intan_rhd2132_0.1.0_arm64.deb") + assert paths[1].endswith("intan_rhd2132-gateware_0.1.0_arm64.deb") def test_case_Q2_deploy_stops_after_failed_driver_deploy( From 2905ec5c59007a5e5ff4f2a65061e5248ae19e7e Mon Sep 17 00:00:00 2001 From: calvinleng-science Date: Thu, 11 Jun 2026 13:59:46 -0700 Subject: [PATCH 17/21] fix(peripherals): accept SDK 1.0.2 top-level hex-string usb_pid in build summaries --- synapse/cli/gateware.py | 42 +++++-- .../cli/test_custom_gateware_packaging.py | 106 +++++++++++++++++- 2 files changed, 138 insertions(+), 10 deletions(-) diff --git a/synapse/cli/gateware.py b/synapse/cli/gateware.py index 9c1a3815..13a15c76 100644 --- a/synapse/cli/gateware.py +++ b/synapse/cli/gateware.py @@ -191,15 +191,25 @@ def summary_path_for(bit_path: str) -> str: def read_usb_pid(bit_path: str) -> int: - """Return ``['project']['usb_pid']`` from the bitstream's summary JSON. + """Return the USB product id from the bitstream's summary JSON, as an int. The custom-bitstream manifest fragment needs the probe USB product id the gateware targets; the gateware toolchain records it in the build summary. + Two summary shapes are accepted: + + - **SDK < 1.0.2** (legacy): ``{"project": {"usb_pid": 4, ...}, ...}`` + - **SDK 1.0.2+**: ``{"usb_pid": "0x000B", "project": {"name": "..."}, ...}`` + + In both cases ``project.usb_pid`` is preferred when present; the top-level + ``usb_pid`` is the fallback. Both integer and hex-string values (e.g. + ``"0x000B"`` or ``"11"``) are accepted and normalised to ``int``. + Raises: FileNotFoundError: no ``.summary.json`` exists next to *bit_path*. - ValueError: the summary is not valid JSON, or ``project.usb_pid`` is - missing or not an integer. + ValueError: the summary is not valid JSON; ``usb_pid`` is absent from + both locations; or the value cannot be interpreted as a uint16 in the + range 1..0xFFFF (0 and negatives are rejected; booleans are rejected). """ path = summary_path_for(bit_path) if not os.path.exists(path): @@ -213,12 +223,30 @@ def read_usb_pid(bit_path: str) -> int: summary = json.load(fp) except json.JSONDecodeError as exc: raise ValueError(f"Bitstream summary {path} is not valid JSON: {exc}") + project = summary.get("project") if isinstance(summary, dict) else None - usb_pid = project.get("usb_pid") if isinstance(project, dict) else None - if not isinstance(usb_pid, int) or isinstance(usb_pid, bool): + raw = None + if isinstance(project, dict) and "usb_pid" in project: + raw = project["usb_pid"] + elif isinstance(summary, dict) and "usb_pid" in summary: + # axon-peripheral-sdk 1.0.2 emits usb_pid at the top level (as a hex + # string, e.g. "0x000B"); prefer project.usb_pid when both exist. + raw = summary["usb_pid"] + + usb_pid = None + if isinstance(raw, int) and not isinstance(raw, bool): + usb_pid = raw + elif isinstance(raw, str): + try: + usb_pid = int(raw, 0) # accepts "0x000B" and "11" + except ValueError: + usb_pid = None + + if usb_pid is None or not (0 < usb_pid <= 0xFFFF): raise ValueError( - f"Bitstream summary {path} is missing ['project']['usb_pid'] " - "(expected an integer USB product id)" + f"Bitstream summary {path} has no usable usb_pid " + "(looked at ['project']['usb_pid'] and top-level ['usb_pid']; " + "expected an integer or hex-string USB product id in 1..0xFFFF)" ) return usb_pid diff --git a/synapse/tests/cli/test_custom_gateware_packaging.py b/synapse/tests/cli/test_custom_gateware_packaging.py index 06b6c5b3..c0ed8ace 100644 --- a/synapse/tests/cli/test_custom_gateware_packaging.py +++ b/synapse/tests/cli/test_custom_gateware_packaging.py @@ -92,12 +92,14 @@ def test_read_usb_pid_missing_key_raises(gateware, tmp_path): assert "usb_pid" in str(exc_info.value) -def test_read_usb_pid_rejects_non_int(gateware, tmp_path): +def test_read_usb_pid_rejects_bool(gateware, tmp_path): + """Booleans must be rejected even though bool is a subclass of int.""" bit = tmp_path / "sdk_x.bit" bit.write_text("bit") - _write_summary(bit, {"project": {"usb_pid": "4"}}) - with pytest.raises(ValueError): + _write_summary(bit, {"project": {"usb_pid": True}}) + with pytest.raises(ValueError) as exc_info: gateware.read_usb_pid(str(bit)) + assert "usb_pid" in str(exc_info.value) def test_read_usb_pid_non_object_summary_raises(gateware, tmp_path): @@ -108,6 +110,104 @@ def test_read_usb_pid_non_object_summary_raises(gateware, tmp_path): gateware.read_usb_pid(str(bit)) +# --------------------------------------------------------------------------- +# New shapes / validation — SDK 1.0.2 contract +# --------------------------------------------------------------------------- + + +def test_read_usb_pid_toplevel_hex_string(gateware, tmp_path): + """SDK 1.0.2 shape: usb_pid at top level as a hex string.""" + bit = tmp_path / "sdk_x.bit" + bit.write_text("bit") + _write_summary( + bit, + { + "schema_version": 1, + "sdk_version": "1.0.2", + "usb_pid": "0x000B", + "project": {"name": "gateware", "git_sha": "e6890a3"}, + }, + ) + assert gateware.read_usb_pid(str(bit)) == 11 + + +def test_read_usb_pid_toplevel_int(gateware, tmp_path): + """Top-level usb_pid as a plain integer is accepted.""" + bit = tmp_path / "sdk_x.bit" + bit.write_text("bit") + _write_summary(bit, {"usb_pid": 11, "project": {"name": "gateware"}}) + assert gateware.read_usb_pid(str(bit)) == 11 + + +def test_read_usb_pid_project_hex_string(gateware, tmp_path): + """project.usb_pid as a hex string is accepted.""" + bit = tmp_path / "sdk_x.bit" + bit.write_text("bit") + _write_summary(bit, {"project": {"usb_pid": "0x0004"}}) + assert gateware.read_usb_pid(str(bit)) == 4 + + +def test_read_usb_pid_project_wins_over_toplevel(gateware, tmp_path): + """When both locations are present, project.usb_pid takes precedence.""" + bit = tmp_path / "sdk_x.bit" + bit.write_text("bit") + _write_summary( + bit, + {"usb_pid": "0x000B", "project": {"usb_pid": 7}}, + ) + assert gateware.read_usb_pid(str(bit)) == 7 + + +def test_read_usb_pid_rejects_invalid_string(gateware, tmp_path): + """Non-numeric strings must raise ValueError mentioning usb_pid.""" + bit = tmp_path / "sdk_x.bit" + bit.write_text("bit") + _write_summary(bit, {"project": {"usb_pid": "abc"}}) + with pytest.raises(ValueError) as exc_info: + gateware.read_usb_pid(str(bit)) + assert "usb_pid" in str(exc_info.value) + + +def test_read_usb_pid_rejects_empty_string(gateware, tmp_path): + """Empty string must raise ValueError mentioning usb_pid.""" + bit = tmp_path / "sdk_x.bit" + bit.write_text("bit") + _write_summary(bit, {"project": {"usb_pid": ""}}) + with pytest.raises(ValueError) as exc_info: + gateware.read_usb_pid(str(bit)) + assert "usb_pid" in str(exc_info.value) + + +def test_read_usb_pid_rejects_zero(gateware, tmp_path): + """Zero is out of range for a USB PID (must be 1..0xFFFF).""" + bit = tmp_path / "sdk_x.bit" + bit.write_text("bit") + _write_summary(bit, {"project": {"usb_pid": 0}}) + with pytest.raises(ValueError) as exc_info: + gateware.read_usb_pid(str(bit)) + assert "usb_pid" in str(exc_info.value) + + +def test_read_usb_pid_rejects_negative(gateware, tmp_path): + """Negative values are out of range.""" + bit = tmp_path / "sdk_x.bit" + bit.write_text("bit") + _write_summary(bit, {"project": {"usb_pid": -1}}) + with pytest.raises(ValueError) as exc_info: + gateware.read_usb_pid(str(bit)) + assert "usb_pid" in str(exc_info.value) + + +def test_read_usb_pid_rejects_overflow(gateware, tmp_path): + """Values above 0xFFFF are out of uint16 range.""" + bit = tmp_path / "sdk_x.bit" + bit.write_text("bit") + _write_summary(bit, {"project": {"usb_pid": 0x10000}}) + with pytest.raises(ValueError) as exc_info: + gateware.read_usb_pid(str(bit)) + assert "usb_pid" in str(exc_info.value) + + # --------------------------------------------------------------------------- # build.find_deb_package package_name filtering # --------------------------------------------------------------------------- From 315db4503e155956c3a5115133ca052a4210d05a Mon Sep 17 00:00:00 2001 From: calvinleng-science Date: Thu, 11 Jun 2026 14:04:45 -0700 Subject: [PATCH 18/21] fix(peripherals): require top-level hex-string usb_pid in bitstream summaries --- synapse/cli/gateware.py | 67 ++++++----- .../cli/test_custom_gateware_packaging.py | 108 +++++++++--------- synapse/tests/cli/test_half_selectors.py | 2 +- 3 files changed, 95 insertions(+), 82 deletions(-) diff --git a/synapse/cli/gateware.py b/synapse/cli/gateware.py index 13a15c76..9775184b 100644 --- a/synapse/cli/gateware.py +++ b/synapse/cli/gateware.py @@ -196,27 +196,30 @@ def read_usb_pid(bit_path: str) -> int: The custom-bitstream manifest fragment needs the probe USB product id the gateware targets; the gateware toolchain records it in the build summary. - Two summary shapes are accepted: + Only the **axon-peripheral-sdk 1.0.2+** shape is accepted: - - **SDK < 1.0.2** (legacy): ``{"project": {"usb_pid": 4, ...}, ...}`` - - **SDK 1.0.2+**: ``{"usb_pid": "0x000B", "project": {"name": "..."}, ...}`` + .. code-block:: json - In both cases ``project.usb_pid`` is preferred when present; the top-level - ``usb_pid`` is the fallback. Both integer and hex-string values (e.g. - ``"0x000B"`` or ``"11"``) are accepted and normalised to ``int``. + {"usb_pid": "0x000B", "project": {"name": "..."}, ...} + + ``usb_pid`` MUST be at the **top level** of the summary object and MUST be + a **hex string** (e.g. ``"0x000B"`` or ``"000B"``). Any non-string value + (int, bool, null, object) is rejected. ``project.usb_pid`` is no longer + consulted. Raises: FileNotFoundError: no ``.summary.json`` exists next to *bit_path*. - ValueError: the summary is not valid JSON; ``usb_pid`` is absent from - both locations; or the value cannot be interpreted as a uint16 in the - range 1..0xFFFF (0 and negatives are rejected; booleans are rejected). + ValueError: the summary is not valid JSON; not a JSON object; top-level + ``usb_pid`` is absent or is not a hex string; or the parsed value is + outside the range 1..0xFFFF. """ path = summary_path_for(bit_path) if not os.path.exists(path): raise FileNotFoundError( f"Bitstream summary not found: {path}. The gateware build is " "expected to emit a .summary.json next to each .bit; rebuild " - "with an axon-peripheral-sdk that records project.usb_pid." + "with an axon-peripheral-sdk that emits a top-level usb_pid " + 'hex string (e.g. "0x000B").' ) with open(path, "r", encoding="utf-8") as fp: try: @@ -224,29 +227,33 @@ def read_usb_pid(bit_path: str) -> int: except json.JSONDecodeError as exc: raise ValueError(f"Bitstream summary {path} is not valid JSON: {exc}") - project = summary.get("project") if isinstance(summary, dict) else None - raw = None - if isinstance(project, dict) and "usb_pid" in project: - raw = project["usb_pid"] - elif isinstance(summary, dict) and "usb_pid" in summary: - # axon-peripheral-sdk 1.0.2 emits usb_pid at the top level (as a hex - # string, e.g. "0x000B"); prefer project.usb_pid when both exist. - raw = summary["usb_pid"] - - usb_pid = None - if isinstance(raw, int) and not isinstance(raw, bool): - usb_pid = raw - elif isinstance(raw, str): - try: - usb_pid = int(raw, 0) # accepts "0x000B" and "11" - except ValueError: - usb_pid = None + if not isinstance(summary, dict): + raise ValueError( + f"Bitstream summary {path} is not a JSON object; " + "expected a top-level usb_pid hex string (e.g. \"0x000B\")." + ) + + raw = summary.get("usb_pid") - if usb_pid is None or not (0 < usb_pid <= 0xFFFF): + if not isinstance(raw, str): raise ValueError( f"Bitstream summary {path} has no usable usb_pid " - "(looked at ['project']['usb_pid'] and top-level ['usb_pid']; " - "expected an integer or hex-string USB product id in 1..0xFFFF)" + "(top-level ['usb_pid'] must be a hex string, e.g. \"0x000B\"; " + f"got {raw!r})" + ) + + try: + usb_pid = int(raw, 16) + except ValueError: + raise ValueError( + f"Bitstream summary {path} usb_pid {raw!r} is not a valid hex " + "string (expected e.g. \"0x000B\")" + ) + + if not (0 < usb_pid <= 0xFFFF): + raise ValueError( + f"Bitstream summary {path} usb_pid {raw!r} ({usb_pid}) is out of " + "range; expected a hex string in 1..0xFFFF (e.g. \"0x000B\")" ) return usb_pid diff --git a/synapse/tests/cli/test_custom_gateware_packaging.py b/synapse/tests/cli/test_custom_gateware_packaging.py index c0ed8ace..bb978470 100644 --- a/synapse/tests/cli/test_custom_gateware_packaging.py +++ b/synapse/tests/cli/test_custom_gateware_packaging.py @@ -60,9 +60,26 @@ def test_summary_path_for_same_stem(gateware, tmp_path): def test_read_usb_pid_happy_path(gateware, tmp_path): + """SDK 1.0.2 shape: top-level hex string, project without usb_pid.""" bit = tmp_path / "sdk_x.bit" bit.write_text("bit") - _write_summary(bit, {"project": {"name": "gateware", "usb_pid": 4}}) + _write_summary( + bit, + { + "schema_version": 1, + "sdk_version": "1.0.2", + "usb_pid": "0x000B", + "project": {"name": "gateware", "git_sha": "e6890a3"}, + }, + ) + assert gateware.read_usb_pid(str(bit)) == 11 + + +def test_read_usb_pid_happy_path_0x0004(gateware, tmp_path): + """Top-level hex string "0x0004" -> 4.""" + bit = tmp_path / "sdk_x.bit" + bit.write_text("bit") + _write_summary(bit, {"usb_pid": "0x0004", "project": {"name": "gateware"}}) assert gateware.read_usb_pid(str(bit)) == 4 @@ -93,10 +110,10 @@ def test_read_usb_pid_missing_key_raises(gateware, tmp_path): def test_read_usb_pid_rejects_bool(gateware, tmp_path): - """Booleans must be rejected even though bool is a subclass of int.""" + """Booleans at top level must be rejected (bool is not a hex string).""" bit = tmp_path / "sdk_x.bit" bit.write_text("bit") - _write_summary(bit, {"project": {"usb_pid": True}}) + _write_summary(bit, {"usb_pid": True, "project": {"name": "gateware"}}) with pytest.raises(ValueError) as exc_info: gateware.read_usb_pid(str(bit)) assert "usb_pid" in str(exc_info.value) @@ -111,101 +128,90 @@ def test_read_usb_pid_non_object_summary_raises(gateware, tmp_path): # --------------------------------------------------------------------------- -# New shapes / validation — SDK 1.0.2 contract +# Strict contract: top-level hex string only (SDK 1.0.2 shape) # --------------------------------------------------------------------------- -def test_read_usb_pid_toplevel_hex_string(gateware, tmp_path): - """SDK 1.0.2 shape: usb_pid at top level as a hex string.""" - bit = tmp_path / "sdk_x.bit" - bit.write_text("bit") - _write_summary( - bit, - { - "schema_version": 1, - "sdk_version": "1.0.2", - "usb_pid": "0x000B", - "project": {"name": "gateware", "git_sha": "e6890a3"}, - }, - ) - assert gateware.read_usb_pid(str(bit)) == 11 - +def test_read_usb_pid_old_project_shape_raises(gateware, tmp_path): + """The OLD shape {"project": {"usb_pid": 4}} (no top-level) -> ValueError. -def test_read_usb_pid_toplevel_int(gateware, tmp_path): - """Top-level usb_pid as a plain integer is accepted.""" + Contract decision: project.usb_pid is no longer consulted; the top-level + hex-string is the only accepted form. + """ bit = tmp_path / "sdk_x.bit" bit.write_text("bit") - _write_summary(bit, {"usb_pid": 11, "project": {"name": "gateware"}}) - assert gateware.read_usb_pid(str(bit)) == 11 + _write_summary(bit, {"project": {"usb_pid": 4}}) + with pytest.raises(ValueError) as exc_info: + gateware.read_usb_pid(str(bit)) + assert "usb_pid" in str(exc_info.value) -def test_read_usb_pid_project_hex_string(gateware, tmp_path): - """project.usb_pid as a hex string is accepted.""" +def test_read_usb_pid_toplevel_int_raises(gateware, tmp_path): + """Top-level usb_pid as a plain integer -> ValueError (must be hex string).""" bit = tmp_path / "sdk_x.bit" bit.write_text("bit") - _write_summary(bit, {"project": {"usb_pid": "0x0004"}}) - assert gateware.read_usb_pid(str(bit)) == 4 + _write_summary(bit, {"usb_pid": 11, "project": {"name": "gateware"}}) + with pytest.raises(ValueError) as exc_info: + gateware.read_usb_pid(str(bit)) + assert "usb_pid" in str(exc_info.value) -def test_read_usb_pid_project_wins_over_toplevel(gateware, tmp_path): - """When both locations are present, project.usb_pid takes precedence.""" +def test_read_usb_pid_toplevel_bool_raises(gateware, tmp_path): + """Top-level bool -> ValueError (bool is not a hex string).""" bit = tmp_path / "sdk_x.bit" bit.write_text("bit") - _write_summary( - bit, - {"usb_pid": "0x000B", "project": {"usb_pid": 7}}, - ) - assert gateware.read_usb_pid(str(bit)) == 7 + _write_summary(bit, {"usb_pid": True, "project": {"name": "gateware"}}) + with pytest.raises(ValueError) as exc_info: + gateware.read_usb_pid(str(bit)) + assert "usb_pid" in str(exc_info.value) -def test_read_usb_pid_rejects_invalid_string(gateware, tmp_path): - """Non-numeric strings must raise ValueError mentioning usb_pid.""" +def test_read_usb_pid_rejects_unparseable_string(gateware, tmp_path): + """Non-hex string at top level -> ValueError mentioning usb_pid.""" bit = tmp_path / "sdk_x.bit" bit.write_text("bit") - _write_summary(bit, {"project": {"usb_pid": "abc"}}) + _write_summary(bit, {"usb_pid": "xyz", "project": {"name": "gateware"}}) with pytest.raises(ValueError) as exc_info: gateware.read_usb_pid(str(bit)) assert "usb_pid" in str(exc_info.value) def test_read_usb_pid_rejects_empty_string(gateware, tmp_path): - """Empty string must raise ValueError mentioning usb_pid.""" + """Empty string at top level -> ValueError mentioning usb_pid.""" bit = tmp_path / "sdk_x.bit" bit.write_text("bit") - _write_summary(bit, {"project": {"usb_pid": ""}}) + _write_summary(bit, {"usb_pid": "", "project": {"name": "gateware"}}) with pytest.raises(ValueError) as exc_info: gateware.read_usb_pid(str(bit)) assert "usb_pid" in str(exc_info.value) -def test_read_usb_pid_rejects_zero(gateware, tmp_path): - """Zero is out of range for a USB PID (must be 1..0xFFFF).""" +def test_read_usb_pid_rejects_zero_hex_string(gateware, tmp_path): + """\"0x0000\" is out of range (must be 1..0xFFFF).""" bit = tmp_path / "sdk_x.bit" bit.write_text("bit") - _write_summary(bit, {"project": {"usb_pid": 0}}) + _write_summary(bit, {"usb_pid": "0x0000", "project": {"name": "gateware"}}) with pytest.raises(ValueError) as exc_info: gateware.read_usb_pid(str(bit)) assert "usb_pid" in str(exc_info.value) -def test_read_usb_pid_rejects_negative(gateware, tmp_path): - """Negative values are out of range.""" +def test_read_usb_pid_rejects_overflow_hex_string(gateware, tmp_path): + """\"0x10000\" is above uint16 range.""" bit = tmp_path / "sdk_x.bit" bit.write_text("bit") - _write_summary(bit, {"project": {"usb_pid": -1}}) + _write_summary(bit, {"usb_pid": "0x10000", "project": {"name": "gateware"}}) with pytest.raises(ValueError) as exc_info: gateware.read_usb_pid(str(bit)) assert "usb_pid" in str(exc_info.value) -def test_read_usb_pid_rejects_overflow(gateware, tmp_path): - """Values above 0xFFFF are out of uint16 range.""" +def test_read_usb_pid_accepts_max_ffff(gateware, tmp_path): + """\"0xFFFF\" is the maximum valid value -> 65535.""" bit = tmp_path / "sdk_x.bit" bit.write_text("bit") - _write_summary(bit, {"project": {"usb_pid": 0x10000}}) - with pytest.raises(ValueError) as exc_info: - gateware.read_usb_pid(str(bit)) - assert "usb_pid" in str(exc_info.value) + _write_summary(bit, {"usb_pid": "0xFFFF", "project": {"name": "gateware"}}) + assert gateware.read_usb_pid(str(bit)) == 65535 # --------------------------------------------------------------------------- diff --git a/synapse/tests/cli/test_half_selectors.py b/synapse/tests/cli/test_half_selectors.py index 7bb37679..96d73257 100644 --- a/synapse/tests/cli/test_half_selectors.py +++ b/synapse/tests/cli/test_half_selectors.py @@ -169,7 +169,7 @@ def fake_run_gateware(*args, **kwargs): bit = str(path) stem, _ = os.path.splitext(bit) with open(f"{stem}.summary.json", "w", encoding="utf-8") as fh: - json.dump({"project": {"name": "gateware", "usb_pid": 4}}, fh) + json.dump({"usb_pid": "0x0004", "project": {"name": "gateware"}}, fh) return bit def fake_subprocess_run(argv, *args, **kwargs): From 287aeec0c79e5520f0924c9cc9aabc8a1f533767 Mon Sep 17 00:00:00 2001 From: calvinleng-science Date: Thu, 11 Jun 2026 14:41:43 -0700 Subject: [PATCH 19/21] feat(peripherals): carry the gateware project name as display_name in fragments --- synapse/cli/gateware.py | 26 ++++++ synapse/cli/peripherals.py | 12 ++- .../cli/test_custom_gateware_packaging.py | 86 ++++++++++++++++++- synapse/tests/cli/test_half_selectors.py | 1 + 4 files changed, 121 insertions(+), 4 deletions(-) diff --git a/synapse/cli/gateware.py b/synapse/cli/gateware.py index 9775184b..9c2e679e 100644 --- a/synapse/cli/gateware.py +++ b/synapse/cli/gateware.py @@ -258,6 +258,32 @@ def read_usb_pid(bit_path: str) -> int: return usb_pid +def read_project_name(bit_path: str) -> str | None: + """Return ``['project']['name']`` from the bitstream's summary JSON, or None. + + Used as the human-facing display name for custom gateware; unlike + :func:`read_usb_pid` this is best-effort — a missing or malformed value + degrades to None (callers fall back to the plugin name) rather than + failing the build. + """ + path = summary_path_for(bit_path) + try: + with open(path, "r", encoding="utf-8") as fp: + summary = json.load(fp) + except (OSError, json.JSONDecodeError): + return None + + if not isinstance(summary, dict): + return None + project = summary.get("project") + if not isinstance(project, dict): + return None + name = project.get("name") + if not isinstance(name, str) or not name: + return None + return name + + def _stdout_is_tty() -> bool: """Whether our stdout is a terminal (indirection kept for monkeypatching).""" return sys.stdout.isatty() diff --git a/synapse/cli/peripherals.py b/synapse/cli/peripherals.py index baceab87..48c8d94e 100644 --- a/synapse/cli/peripherals.py +++ b/synapse/cli/peripherals.py @@ -583,6 +583,7 @@ def build_gateware_deb( *, bit_path: str, usb_pid: int, + display_name: Optional[str] = None, version: str = "0.1.0", ) -> bool: """Stage the custom bitstream + manifest fragment, then fpm a .deb. @@ -591,8 +592,10 @@ def build_gateware_deb( /opt/scifi/bitstreams/custom/.bit /opt/scifi/bitstreams/custom/.manifest.json - The fragment carries ``{"name", "usb_pid", "artifact"}`` with ``artifact`` - relative to /opt/scifi/bitstreams (canonical-manifest convention); + The fragment carries ``{"name", "display_name", "usb_pid", "artifact"}`` + with ``artifact`` relative to /opt/scifi/bitstreams + (canonical-manifest convention); ``display_name`` is the human-facing + label from the gateware project (falls back to ``name`` when absent); scifi-probe-updater globs custom/*.manifest.json to list flashable custom gateware per probe. """ @@ -615,6 +618,7 @@ def build_gateware_deb( fragment = { "name": plugin_name, + "display_name": display_name or plugin_name, "usb_pid": usb_pid, "artifact": f"custom/{plugin_name}.bit", } @@ -792,8 +796,10 @@ def _build_debs( usb_pid = _gateware_usb_pid(bit_path) if usb_pid is None: return None + display_name = gateware.read_project_name(bit_path) if not build_gateware_deb( - peripheral_dir, manifest, bit_path=bit_path, usb_pid=usb_pid, version=version + peripheral_dir, manifest, bit_path=bit_path, usb_pid=usb_pid, + display_name=display_name, version=version ): return None deb = find_deb_package(dist_dir, f"{plugin_name}{GATEWARE_DEB_SUFFIX}_{version}") diff --git a/synapse/tests/cli/test_custom_gateware_packaging.py b/synapse/tests/cli/test_custom_gateware_packaging.py index bb978470..fa3e3509 100644 --- a/synapse/tests/cli/test_custom_gateware_packaging.py +++ b/synapse/tests/cli/test_custom_gateware_packaging.py @@ -214,6 +214,59 @@ def test_read_usb_pid_accepts_max_ffff(gateware, tmp_path): assert gateware.read_usb_pid(str(bit)) == 65535 +# --------------------------------------------------------------------------- +# gateware.read_project_name +# --------------------------------------------------------------------------- + + +def test_read_project_name_happy_path(gateware, tmp_path): + """SDK 1.0.2 shape: project.name is a non-empty string.""" + bit = tmp_path / "sdk_x.bit" + bit.write_text("bit") + _write_summary( + bit, + { + "usb_pid": "0x000B", + "project": {"name": "gateware", "git_sha": "e6890a3"}, + }, + ) + assert gateware.read_project_name(str(bit)) == "gateware" + + +def test_read_project_name_missing_summary_returns_none(gateware, tmp_path): + bit = tmp_path / "sdk_x.bit" + bit.write_text("bit") + assert gateware.read_project_name(str(bit)) is None + + +def test_read_project_name_missing_project_returns_none(gateware, tmp_path): + bit = tmp_path / "sdk_x.bit" + bit.write_text("bit") + _write_summary(bit, {"usb_pid": "0x000B"}) + assert gateware.read_project_name(str(bit)) is None + + +def test_read_project_name_missing_name_returns_none(gateware, tmp_path): + bit = tmp_path / "sdk_x.bit" + bit.write_text("bit") + _write_summary(bit, {"usb_pid": "0x000B", "project": {"git_sha": "abc123"}}) + assert gateware.read_project_name(str(bit)) is None + + +def test_read_project_name_empty_string_returns_none(gateware, tmp_path): + bit = tmp_path / "sdk_x.bit" + bit.write_text("bit") + _write_summary(bit, {"usb_pid": "0x000B", "project": {"name": ""}}) + assert gateware.read_project_name(str(bit)) is None + + +def test_read_project_name_invalid_json_returns_none(gateware, tmp_path): + bit = tmp_path / "sdk_x.bit" + bit.write_text("bit") + _write_summary(bit, "{not json") + assert gateware.read_project_name(str(bit)) is None + + # --------------------------------------------------------------------------- # build.find_deb_package package_name filtering # --------------------------------------------------------------------------- @@ -287,7 +340,8 @@ def test_build_gateware_deb_stages_bit_fragment_and_depends( monkeypatch.setattr(peripherals.subprocess, "run", fake_fpm_run(dist_dir, calls)) ok = peripherals.build_gateware_deb( - str(pd), manifest, bit_path=str(bit), usb_pid=4, version="0.2.0" + str(pd), manifest, bit_path=str(bit), usb_pid=4, display_name="my-gateware", + version="0.2.0" ) assert ok is True assert len(staging) == 1 @@ -304,6 +358,7 @@ def test_build_gateware_deb_stages_bit_fragment_and_depends( frag = json.load(fh) assert frag == { "name": "scifi-my-chip", + "display_name": "my-gateware", "usb_pid": 4, "artifact": "custom/scifi-my-chip.bit", } @@ -317,6 +372,35 @@ def test_build_gateware_deb_stages_bit_fragment_and_depends( assert fpm_call[-1] == "opt" +def test_build_gateware_deb_omit_display_name_falls_back_to_plugin_name( + peripherals, tmp_path, monkeypatch +): + """Omitting display_name causes the fragment to use the plugin name as display_name.""" + pd = tmp_path / "plugin" + pd.mkdir() + bit = tmp_path / "sdk_x.bit" + bit.write_text("BITSTREAM") + manifest = {"name": "scifi-my-chip", "version": "0.2.0"} + + staging: list = [] + _spy_mkdtemp(peripherals, monkeypatch, staging) + calls: list = [] + dist_dir = os.path.join(str(pd), "dist") + monkeypatch.setattr(peripherals.subprocess, "run", fake_fpm_run(dist_dir, calls)) + + ok = peripherals.build_gateware_deb( + str(pd), manifest, bit_path=str(bit), usb_pid=4, version="0.2.0" + ) + assert ok is True + frag_dst = os.path.join( + staging[0], "opt", "scifi", "bitstreams", "custom", + "scifi-my-chip.manifest.json", + ) + with open(frag_dst, "r", encoding="utf-8") as fh: + frag = json.load(fh) + assert frag["display_name"] == "scifi-my-chip" + + def test_build_gateware_deb_missing_bit_errors(peripherals, tmp_path, capsys): pd = tmp_path / "plugin" pd.mkdir() diff --git a/synapse/tests/cli/test_half_selectors.py b/synapse/tests/cli/test_half_selectors.py index 96d73257..a24efdc2 100644 --- a/synapse/tests/cli/test_half_selectors.py +++ b/synapse/tests/cli/test_half_selectors.py @@ -902,6 +902,7 @@ def spy_mkdtemp(*args, **kwargs): frag = json.load(fh) assert frag == { "name": "intan_rhd2132", + "display_name": "gateware", "usb_pid": 4, "artifact": "custom/intan_rhd2132.bit", } From eef1a5c338b8c2ab2d839bf0a8ed0f1d8c69131e Mon Sep 17 00:00:00 2001 From: calvinleng-science Date: Thu, 11 Jun 2026 14:44:58 -0700 Subject: [PATCH 20/21] feat(peripherals): name custom bitstreams after the gateware project, not the plugin --- synapse/cli/gateware.py | 11 ++++--- synapse/cli/peripherals.py | 32 +++++++++++-------- .../cli/test_custom_gateware_packaging.py | 13 ++++---- synapse/tests/cli/test_half_selectors.py | 3 +- 4 files changed, 32 insertions(+), 27 deletions(-) diff --git a/synapse/cli/gateware.py b/synapse/cli/gateware.py index 9c2e679e..6359a903 100644 --- a/synapse/cli/gateware.py +++ b/synapse/cli/gateware.py @@ -261,10 +261,13 @@ def read_usb_pid(bit_path: str) -> int: def read_project_name(bit_path: str) -> str | None: """Return ``['project']['name']`` from the bitstream's summary JSON, or None. - Used as the human-facing display name for custom gateware; unlike - :func:`read_usb_pid` this is best-effort — a missing or malformed value - degrades to None (callers fall back to the plugin name) rather than - failing the build. + This IS the custom bitstream's identity — the name placed in the manifest + fragment's ``name`` field and passed to ``scifi-probe-updater update + --name``. Unlike :func:`read_usb_pid` this is best-effort: a missing or + malformed value degrades to None and the call site falls back to the plugin + name. The name should be unique per device and systemd-instance-safe + (``[A-Za-z0-9._-]``) since it rides in + ``scifi-probe-update-custom@:``. """ path = summary_path_for(bit_path) try: diff --git a/synapse/cli/peripherals.py b/synapse/cli/peripherals.py index 48c8d94e..75cfd607 100644 --- a/synapse/cli/peripherals.py +++ b/synapse/cli/peripherals.py @@ -583,21 +583,23 @@ def build_gateware_deb( *, bit_path: str, usb_pid: int, - display_name: Optional[str] = None, + bitstream_name: Optional[str] = None, version: str = "0.1.0", ) -> bool: """Stage the custom bitstream + manifest fragment, then fpm a .deb. Layout inside the ``-gateware`` .deb: - /opt/scifi/bitstreams/custom/.bit - /opt/scifi/bitstreams/custom/.manifest.json - - The fragment carries ``{"name", "display_name", "usb_pid", "artifact"}`` - with ``artifact`` relative to /opt/scifi/bitstreams - (canonical-manifest convention); ``display_name`` is the human-facing - label from the gateware project (falls back to ``name`` when absent); - scifi-probe-updater globs custom/*.manifest.json to list flashable - custom gateware per probe. + /opt/scifi/bitstreams/custom/.bit + /opt/scifi/bitstreams/custom/.manifest.json + + The fragment carries ``{"name", "usb_pid", "artifact"}`` with ``artifact`` + relative to /opt/scifi/bitstreams (canonical-manifest convention). Files + and the deb package are keyed on the plugin name for dpkg uniqueness; + ``name`` in the fragment is the flashable identity shown in the UI and + passed to ``scifi-probe-updater update --name`` — taken from the gateware + project (``bitstream_name``) and falling back to the plugin name when no + project name was available. scifi-probe-updater globs + custom/*.manifest.json to list flashable custom gateware per probe. """ plugin_name = manifest["name"] package_name = f"{plugin_name}{GATEWARE_DEB_SUFFIX}" @@ -617,8 +619,11 @@ def build_gateware_deb( shutil.copy2(bit_path, os.path.join(custom_dir, f"{plugin_name}.bit")) fragment = { - "name": plugin_name, - "display_name": display_name or plugin_name, + # The flashable identity shown in the UI and passed to + # `scifi-probe-updater update --name`: the gateware project's name + # (from the build summary), not the plugin's. Files and the deb + # stay keyed on the plugin name for dpkg uniqueness. + "name": bitstream_name or plugin_name, "usb_pid": usb_pid, "artifact": f"custom/{plugin_name}.bit", } @@ -796,10 +801,9 @@ def _build_debs( usb_pid = _gateware_usb_pid(bit_path) if usb_pid is None: return None - display_name = gateware.read_project_name(bit_path) if not build_gateware_deb( peripheral_dir, manifest, bit_path=bit_path, usb_pid=usb_pid, - display_name=display_name, version=version + bitstream_name=gateware.read_project_name(bit_path), version=version ): return None deb = find_deb_package(dist_dir, f"{plugin_name}{GATEWARE_DEB_SUFFIX}_{version}") diff --git a/synapse/tests/cli/test_custom_gateware_packaging.py b/synapse/tests/cli/test_custom_gateware_packaging.py index fa3e3509..860ca2de 100644 --- a/synapse/tests/cli/test_custom_gateware_packaging.py +++ b/synapse/tests/cli/test_custom_gateware_packaging.py @@ -340,7 +340,7 @@ def test_build_gateware_deb_stages_bit_fragment_and_depends( monkeypatch.setattr(peripherals.subprocess, "run", fake_fpm_run(dist_dir, calls)) ok = peripherals.build_gateware_deb( - str(pd), manifest, bit_path=str(bit), usb_pid=4, display_name="my-gateware", + str(pd), manifest, bit_path=str(bit), usb_pid=4, bitstream_name="my-gateware", version="0.2.0" ) assert ok is True @@ -353,12 +353,11 @@ def test_build_gateware_deb_stages_bit_fragment_and_depends( staging[0], "opt", "scifi", "bitstreams", "custom", "scifi-my-chip.manifest.json", ) - assert os.path.exists(bit_dst), "bitstream staged under custom/ as .bit" + assert os.path.exists(bit_dst), "bitstream staged under custom/ as .bit" with open(frag_dst, "r", encoding="utf-8") as fh: frag = json.load(fh) assert frag == { - "name": "scifi-my-chip", - "display_name": "my-gateware", + "name": "my-gateware", "usb_pid": 4, "artifact": "custom/scifi-my-chip.bit", } @@ -372,10 +371,10 @@ def test_build_gateware_deb_stages_bit_fragment_and_depends( assert fpm_call[-1] == "opt" -def test_build_gateware_deb_omit_display_name_falls_back_to_plugin_name( +def test_build_gateware_deb_omit_bitstream_name_falls_back_to_plugin_name( peripherals, tmp_path, monkeypatch ): - """Omitting display_name causes the fragment to use the plugin name as display_name.""" + """Omitting bitstream_name causes the fragment's 'name' to use the plugin name.""" pd = tmp_path / "plugin" pd.mkdir() bit = tmp_path / "sdk_x.bit" @@ -398,7 +397,7 @@ def test_build_gateware_deb_omit_display_name_falls_back_to_plugin_name( ) with open(frag_dst, "r", encoding="utf-8") as fh: frag = json.load(fh) - assert frag["display_name"] == "scifi-my-chip" + assert frag == {"name": "scifi-my-chip", "usb_pid": 4, "artifact": "custom/scifi-my-chip.bit"} def test_build_gateware_deb_missing_bit_errors(peripherals, tmp_path, capsys): diff --git a/synapse/tests/cli/test_half_selectors.py b/synapse/tests/cli/test_half_selectors.py index a24efdc2..c5730b81 100644 --- a/synapse/tests/cli/test_half_selectors.py +++ b/synapse/tests/cli/test_half_selectors.py @@ -901,8 +901,7 @@ def spy_mkdtemp(*args, **kwargs): ) as fh: frag = json.load(fh) assert frag == { - "name": "intan_rhd2132", - "display_name": "gateware", + "name": "gateware", "usb_pid": 4, "artifact": "custom/intan_rhd2132.bit", } From dc985f6b596acef8a5a8fcbe836049e53e27a748 Mon Sep 17 00:00:00 2001 From: calvinleng-science Date: Thu, 11 Jun 2026 15:07:51 -0700 Subject: [PATCH 21/21] feat(peripherals): key custom gateware on target-profile_project identity with git hash --- synapse/cli/gateware.py | 52 ++++++++--- synapse/cli/peripherals.py | 70 ++++++++------ .../cli/test_custom_gateware_packaging.py | 92 +++++++++++++------ synapse/tests/cli/test_half_selectors.py | 19 ++-- 4 files changed, 158 insertions(+), 75 deletions(-) diff --git a/synapse/cli/gateware.py b/synapse/cli/gateware.py index 6359a903..28d8145a 100644 --- a/synapse/cli/gateware.py +++ b/synapse/cli/gateware.py @@ -258,15 +258,16 @@ def read_usb_pid(bit_path: str) -> int: return usb_pid -def read_project_name(bit_path: str) -> str | None: - """Return ``['project']['name']`` from the bitstream's summary JSON, or None. - - This IS the custom bitstream's identity — the name placed in the manifest - fragment's ``name`` field and passed to ``scifi-probe-updater update - --name``. Unlike :func:`read_usb_pid` this is best-effort: a missing or - malformed value degrades to None and the call site falls back to the plugin - name. The name should be unique per device and systemd-instance-safe - (``[A-Za-z0-9._-]``) since it rides in +def read_identifier(bit_path: str) -> str | None: + """Return ``_`` from the bitstream's summary. + + This is the custom bitstream's identity: the on-device file names, the + manifest fragment's ``name``, the ``axon-gateware-*`` deb name, + ``scifi-probe-updater update --name``, and the systemd instance all key + on it, and deploying the same identifier again overrides the previous + install. Best-effort: returns None (callers fall back to the plugin + name) when the summary or either field is missing/malformed. Values + should stay within ``[A-Za-z0-9._-]`` — the identifier rides in ``scifi-probe-update-custom@:``. """ path = summary_path_for(bit_path) @@ -276,15 +277,42 @@ def read_project_name(bit_path: str) -> str | None: except (OSError, json.JSONDecodeError): return None + if not isinstance(summary, dict): + return None + target_profile = summary.get("target_profile") + if not isinstance(target_profile, str) or not target_profile: + return None + project = summary.get("project") + if not isinstance(project, dict): + return None + project_name = project.get("name") + if not isinstance(project_name, str) or not project_name: + return None + return f"{target_profile}_{project_name}" + + +def read_git_sha(bit_path: str) -> str | None: + """Return ``['project']['git_sha']`` from the bitstream's summary JSON, or None. + + Best-effort: returns None when the summary or the field is + missing/malformed. + """ + path = summary_path_for(bit_path) + try: + with open(path, "r", encoding="utf-8") as fp: + summary = json.load(fp) + except (OSError, json.JSONDecodeError): + return None + if not isinstance(summary, dict): return None project = summary.get("project") if not isinstance(project, dict): return None - name = project.get("name") - if not isinstance(name, str) or not name: + git_sha = project.get("git_sha") + if not isinstance(git_sha, str) or not git_sha: return None - return name + return git_sha def _stdout_is_tty() -> bool: diff --git a/synapse/cli/peripherals.py b/synapse/cli/peripherals.py index 75cfd607..2edeacec 100644 --- a/synapse/cli/peripherals.py +++ b/synapse/cli/peripherals.py @@ -570,13 +570,20 @@ def build_peripheral_deb( return True -# Suffix appended to the plugin name to form the gateware package name. -GATEWARE_DEB_SUFFIX = "-gateware" +# Gateware debs are keyed by the bitstream identifier (not the plugin), so +# redeploying the same identifier from any repo replaces the previous +# install instead of dpkg-conflicting with it. +GATEWARE_DEB_PREFIX = "axon-gateware-" # Owns /opt/scifi/bitstreams and the canonical manifest the fragment's # relative `artifact` resolves against. BITSTREAMS_PACKAGE = "axonprobe-bitstreams" +def _gateware_package_name(identifier: str) -> str: + """Debianize the bitstream identifier into the gateware package name.""" + return f"{GATEWARE_DEB_PREFIX}{identifier.lower().replace('_', '-')}" + + def build_gateware_deb( peripheral_dir: str, manifest: dict, @@ -584,25 +591,28 @@ def build_gateware_deb( bit_path: str, usb_pid: int, bitstream_name: Optional[str] = None, + git_hash: Optional[str] = None, version: str = "0.1.0", ) -> bool: """Stage the custom bitstream + manifest fragment, then fpm a .deb. - Layout inside the ``-gateware`` .deb: - /opt/scifi/bitstreams/custom/.bit - /opt/scifi/bitstreams/custom/.manifest.json - - The fragment carries ``{"name", "usb_pid", "artifact"}`` with ``artifact`` - relative to /opt/scifi/bitstreams (canonical-manifest convention). Files - and the deb package are keyed on the plugin name for dpkg uniqueness; - ``name`` in the fragment is the flashable identity shown in the UI and - passed to ``scifi-probe-updater update --name`` — taken from the gateware - project (``bitstream_name``) and falling back to the plugin name when no - project name was available. scifi-probe-updater globs - custom/*.manifest.json to list flashable custom gateware per probe. + The deb package is named ``axon-gateware-`` where + the identifier is ``bitstream_name`` (falling back to the plugin name). + On-device files land at: + /opt/scifi/bitstreams/custom/.bit + /opt/scifi/bitstreams/custom/.manifest.json + + The fragment carries ``{"name", "usb_pid", "artifact"}`` (plus + ``"git_hash"`` when provided) with ``artifact`` relative to + /opt/scifi/bitstreams (canonical-manifest convention). Redeploying the + same identifier from any repo replaces the previous install (dpkg + override semantics) instead of conflicting with a plugin-name-keyed + package. scifi-probe-updater globs custom/*.manifest.json to list + flashable custom gateware per probe. """ plugin_name = manifest["name"] - package_name = f"{plugin_name}{GATEWARE_DEB_SUFFIX}" + identifier = bitstream_name or plugin_name + package_name = _gateware_package_name(identifier) if not os.path.exists(bit_path): console.print( @@ -616,18 +626,16 @@ def build_gateware_deb( custom_dir = os.path.join(staging_dir, "opt", "scifi", "bitstreams", "custom") os.makedirs(custom_dir, exist_ok=True) - shutil.copy2(bit_path, os.path.join(custom_dir, f"{plugin_name}.bit")) - - fragment = { - # The flashable identity shown in the UI and passed to - # `scifi-probe-updater update --name`: the gateware project's name - # (from the build summary), not the plugin's. Files and the deb - # stay keyed on the plugin name for dpkg uniqueness. - "name": bitstream_name or plugin_name, + shutil.copy2(bit_path, os.path.join(custom_dir, f"{identifier}.bit")) + + fragment: dict = { + "name": identifier, "usb_pid": usb_pid, - "artifact": f"custom/{plugin_name}.bit", + "artifact": f"custom/{identifier}.bit", } - fragment_path = os.path.join(custom_dir, f"{plugin_name}.manifest.json") + if git_hash: + fragment["git_hash"] = git_hash + fragment_path = os.path.join(custom_dir, f"{identifier}.manifest.json") with open(fragment_path, "w", encoding="utf-8") as fp: json.dump(fragment, fp, indent=2) fp.write("\n") @@ -801,12 +809,18 @@ def _build_debs( usb_pid = _gateware_usb_pid(bit_path) if usb_pid is None: return None + identifier = gateware.read_identifier(bit_path) or plugin_name if not build_gateware_deb( - peripheral_dir, manifest, bit_path=bit_path, usb_pid=usb_pid, - bitstream_name=gateware.read_project_name(bit_path), version=version + peripheral_dir, + manifest, + bit_path=bit_path, + usb_pid=usb_pid, + version=version, + bitstream_name=identifier, + git_hash=gateware.read_git_sha(bit_path), ): return None - deb = find_deb_package(dist_dir, f"{plugin_name}{GATEWARE_DEB_SUFFIX}_{version}") + deb = find_deb_package(dist_dir, f"{_gateware_package_name(identifier)}_{version}") if deb is None: return None debs.append(deb) diff --git a/synapse/tests/cli/test_custom_gateware_packaging.py b/synapse/tests/cli/test_custom_gateware_packaging.py index 860ca2de..3dec7e54 100644 --- a/synapse/tests/cli/test_custom_gateware_packaging.py +++ b/synapse/tests/cli/test_custom_gateware_packaging.py @@ -215,56 +215,84 @@ def test_read_usb_pid_accepts_max_ffff(gateware, tmp_path): # --------------------------------------------------------------------------- -# gateware.read_project_name +# gateware.read_identifier # --------------------------------------------------------------------------- -def test_read_project_name_happy_path(gateware, tmp_path): - """SDK 1.0.2 shape: project.name is a non-empty string.""" +def test_read_identifier_happy_path(gateware, tmp_path): + """SDK 1.0.2 shape: target_profile + project.name -> '_'.""" bit = tmp_path / "sdk_x.bit" bit.write_text("bit") _write_summary( bit, { "usb_pid": "0x000B", + "target_profile": "via-devkit", "project": {"name": "gateware", "git_sha": "e6890a3"}, }, ) - assert gateware.read_project_name(str(bit)) == "gateware" + assert gateware.read_identifier(str(bit)) == "via-devkit_gateware" -def test_read_project_name_missing_summary_returns_none(gateware, tmp_path): +def test_read_identifier_missing_target_profile_returns_none(gateware, tmp_path): bit = tmp_path / "sdk_x.bit" bit.write_text("bit") - assert gateware.read_project_name(str(bit)) is None + _write_summary(bit, {"usb_pid": "0x000B", "project": {"name": "gateware"}}) + assert gateware.read_identifier(str(bit)) is None -def test_read_project_name_missing_project_returns_none(gateware, tmp_path): +def test_read_identifier_missing_project_name_returns_none(gateware, tmp_path): bit = tmp_path / "sdk_x.bit" bit.write_text("bit") - _write_summary(bit, {"usb_pid": "0x000B"}) - assert gateware.read_project_name(str(bit)) is None + _write_summary(bit, {"usb_pid": "0x000B", "target_profile": "via-devkit", "project": {}}) + assert gateware.read_identifier(str(bit)) is None -def test_read_project_name_missing_name_returns_none(gateware, tmp_path): +def test_read_identifier_missing_summary_returns_none(gateware, tmp_path): bit = tmp_path / "sdk_x.bit" bit.write_text("bit") - _write_summary(bit, {"usb_pid": "0x000B", "project": {"git_sha": "abc123"}}) - assert gateware.read_project_name(str(bit)) is None + assert gateware.read_identifier(str(bit)) is None -def test_read_project_name_empty_string_returns_none(gateware, tmp_path): +def test_read_identifier_invalid_json_returns_none(gateware, tmp_path): bit = tmp_path / "sdk_x.bit" bit.write_text("bit") - _write_summary(bit, {"usb_pid": "0x000B", "project": {"name": ""}}) - assert gateware.read_project_name(str(bit)) is None + _write_summary(bit, "{not json") + assert gateware.read_identifier(str(bit)) is None + + +# --------------------------------------------------------------------------- +# gateware.read_git_sha +# --------------------------------------------------------------------------- -def test_read_project_name_invalid_json_returns_none(gateware, tmp_path): +def test_read_git_sha_happy_path(gateware, tmp_path): + """SDK 1.0.2 shape: project.git_sha is a non-empty string.""" bit = tmp_path / "sdk_x.bit" bit.write_text("bit") - _write_summary(bit, "{not json") - assert gateware.read_project_name(str(bit)) is None + _write_summary( + bit, + { + "usb_pid": "0x000B", + "target_profile": "via-devkit", + "project": {"name": "gateware", "git_sha": "e6890a3"}, + }, + ) + assert gateware.read_git_sha(str(bit)) == "e6890a3" + + +def test_read_git_sha_missing_git_sha_returns_none(gateware, tmp_path): + bit = tmp_path / "sdk_x.bit" + bit.write_text("bit") + _write_summary(bit, {"usb_pid": "0x000B", "project": {"name": "gateware"}}) + assert gateware.read_git_sha(str(bit)) is None + + +def test_read_git_sha_empty_string_returns_none(gateware, tmp_path): + bit = tmp_path / "sdk_x.bit" + bit.write_text("bit") + _write_summary(bit, {"usb_pid": "0x000B", "project": {"name": "gateware", "git_sha": ""}}) + assert gateware.read_git_sha(str(bit)) is None # --------------------------------------------------------------------------- @@ -327,6 +355,7 @@ def spy(*args, **kwargs): def test_build_gateware_deb_stages_bit_fragment_and_depends( peripherals, tmp_path, monkeypatch ): + """With bitstream_name and git_hash: files/fragment keyed on identifier.""" pd = tmp_path / "plugin" pd.mkdir() bit = tmp_path / "sdk_x.bit" @@ -340,30 +369,32 @@ def test_build_gateware_deb_stages_bit_fragment_and_depends( monkeypatch.setattr(peripherals.subprocess, "run", fake_fpm_run(dist_dir, calls)) ok = peripherals.build_gateware_deb( - str(pd), manifest, bit_path=str(bit), usb_pid=4, bitstream_name="my-gateware", + str(pd), manifest, bit_path=str(bit), usb_pid=4, + bitstream_name="via-devkit_gateware", git_hash="e6890a3", version="0.2.0" ) assert ok is True assert len(staging) == 1 bit_dst = os.path.join( - staging[0], "opt", "scifi", "bitstreams", "custom", "scifi-my-chip.bit" + staging[0], "opt", "scifi", "bitstreams", "custom", "via-devkit_gateware.bit" ) frag_dst = os.path.join( staging[0], "opt", "scifi", "bitstreams", "custom", - "scifi-my-chip.manifest.json", + "via-devkit_gateware.manifest.json", ) - assert os.path.exists(bit_dst), "bitstream staged under custom/ as .bit" + assert os.path.exists(bit_dst), "bitstream staged under custom/ as .bit" with open(frag_dst, "r", encoding="utf-8") as fh: frag = json.load(fh) assert frag == { - "name": "my-gateware", + "name": "via-devkit_gateware", "usb_pid": 4, - "artifact": "custom/scifi-my-chip.bit", + "artifact": "custom/via-devkit_gateware.bit", + "git_hash": "e6890a3", } fpm_call = next(c for c in calls if "fpm" in c) - assert fpm_call[fpm_call.index("-n") + 1] == "scifi-my-chip-gateware" + assert fpm_call[fpm_call.index("-n") + 1] == "axon-gateware-via-devkit-gateware" assert fpm_call[fpm_call.index("--depends") + 1] == "axonprobe-bitstreams" # fpm input must be "opt" (not "."): postinstall.sh must NOT ship in the # payload, or the driver and gateware debs would dpkg-conflict on @@ -374,7 +405,7 @@ def test_build_gateware_deb_stages_bit_fragment_and_depends( def test_build_gateware_deb_omit_bitstream_name_falls_back_to_plugin_name( peripherals, tmp_path, monkeypatch ): - """Omitting bitstream_name causes the fragment's 'name' to use the plugin name.""" + """Omitting bitstream_name: files/fragment keyed on plugin name, no git_hash key.""" pd = tmp_path / "plugin" pd.mkdir() bit = tmp_path / "sdk_x.bit" @@ -391,14 +422,23 @@ def test_build_gateware_deb_omit_bitstream_name_falls_back_to_plugin_name( str(pd), manifest, bit_path=str(bit), usb_pid=4, version="0.2.0" ) assert ok is True + + bit_dst = os.path.join( + staging[0], "opt", "scifi", "bitstreams", "custom", "scifi-my-chip.bit" + ) frag_dst = os.path.join( staging[0], "opt", "scifi", "bitstreams", "custom", "scifi-my-chip.manifest.json", ) + assert os.path.exists(bit_dst), "fallback: bit staged as .bit" with open(frag_dst, "r", encoding="utf-8") as fh: frag = json.load(fh) + # 3-key fragment, NO git_hash key assert frag == {"name": "scifi-my-chip", "usb_pid": 4, "artifact": "custom/scifi-my-chip.bit"} + fpm_call = next(c for c in calls if "fpm" in c) + assert fpm_call[fpm_call.index("-n") + 1] == "axon-gateware-scifi-my-chip" + def test_build_gateware_deb_missing_bit_errors(peripherals, tmp_path, capsys): pd = tmp_path / "plugin" diff --git a/synapse/tests/cli/test_half_selectors.py b/synapse/tests/cli/test_half_selectors.py index c5730b81..9050ff96 100644 --- a/synapse/tests/cli/test_half_selectors.py +++ b/synapse/tests/cli/test_half_selectors.py @@ -169,7 +169,7 @@ def fake_run_gateware(*args, **kwargs): bit = str(path) stem, _ = os.path.splitext(bit) with open(f"{stem}.summary.json", "w", encoding="utf-8") as fh: - json.dump({"usb_pid": "0x0004", "project": {"name": "gateware"}}, fh) + json.dump({"usb_pid": "0x0004", "target_profile": "via-devkit", "project": {"name": "gateware", "git_sha": "e6890a3"}}, fh) return bit def fake_subprocess_run(argv, *args, **kwargs): @@ -804,12 +804,12 @@ def spy_mkdtemp(*args, **kwargs): ) assert any( - f.endswith(os.path.join("opt/scifi/bitstreams/custom", "intan_rhd2132.bit")) + f.endswith(os.path.join("opt/scifi/bitstreams/custom", "via-devkit_gateware.bit")) for f in gateware_files ), f"gateware deb stages the bit under custom/; got: {gateware_files!r}" assert any( f.endswith( - os.path.join("opt/scifi/bitstreams/custom", "intan_rhd2132.manifest.json") + os.path.join("opt/scifi/bitstreams/custom", "via-devkit_gateware.manifest.json") ) for f in gateware_files ), f"gateware deb stages the manifest fragment; got: {gateware_files!r}" @@ -823,7 +823,7 @@ def spy_mkdtemp(*args, **kwargs): if "fpm" in c: fpm_argv = c[c.index("fpm"):] fpm_names.append(fpm_argv[fpm_argv.index("-n") + 1]) - assert fpm_names == ["intan_rhd2132", "intan_rhd2132-gateware"] + assert fpm_names == ["intan_rhd2132", "axon-gateware-via-devkit-gateware"] def test_case_N_driver_deb_fpm_input_excludes_postinstall( @@ -888,22 +888,23 @@ def spy_mkdtemp(*args, **kwargs): assert len(staging_dirs) == 1 files = _captured_staging_files(staging_dirs[0]) assert any( - f.endswith(os.path.join("opt/scifi/bitstreams/custom", "intan_rhd2132.bit")) + f.endswith(os.path.join("opt/scifi/bitstreams/custom", "via-devkit_gateware.bit")) for f in files ), f"gateware deb stages the bit; got: {files!r}" with open( os.path.join( staging_dirs[0], - "opt", "scifi", "bitstreams", "custom", "intan_rhd2132.manifest.json", + "opt", "scifi", "bitstreams", "custom", "via-devkit_gateware.manifest.json", ), "r", encoding="utf-8", ) as fh: frag = json.load(fh) assert frag == { - "name": "gateware", + "name": "via-devkit_gateware", "usb_pid": 4, - "artifact": "custom/intan_rhd2132.bit", + "artifact": "custom/via-devkit_gateware.bit", + "git_hash": "e6890a3", } assert not any(f.endswith(".so") for f in files) assert not any("libscifi-peripheral-sdk" in f for f in files) @@ -932,7 +933,7 @@ def test_case_Q_deploy_both_streams_two_debs(peripherals, tmp_path, monkeypatch) paths = [p for _, p in recorders.deploy_calls] assert uris == ["10.0.0.1", "10.0.0.1"] assert paths[0].endswith("intan_rhd2132_0.1.0_arm64.deb") - assert paths[1].endswith("intan_rhd2132-gateware_0.1.0_arm64.deb") + assert paths[1].endswith("axon-gateware-via-devkit-gateware_0.1.0_arm64.deb") def test_case_Q2_deploy_stops_after_failed_driver_deploy(