diff --git a/.github/workflows/dev-stack.yml b/.github/workflows/dev-stack.yml new file mode 100644 index 000000000..06f83b855 --- /dev/null +++ b/.github/workflows/dev-stack.yml @@ -0,0 +1,59 @@ +name: Dev stack + +# npm run dev:stack (#65): the whole app from source on a clean runner, without a TTY. +on: + pull_request: + paths: + - "scripts/dev-stack*.mjs" + - "scripts/tests/dev-stack.test.mjs" + - "apps/**" + - "packages/**" + - "deploy/docker/**" + - "Dockerfile" + - "package.json" + - "package-lock.json" + - ".github/workflows/dev-stack.yml" + push: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: dev-stack-${{ github.ref }} + cancel-in-progress: true + +jobs: + dev-stack: + name: dev:stack up, owner setup, OpenCode pane, reset + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version-file: .nvmrc + cache: npm + + - name: Install dependencies + env: + ELECTRON_SKIP_BINARY_DOWNLOAD: "1" + run: npm ci + + - name: Dev-stack unit tests + run: node --test scripts/tests/dev-stack.test.mjs + + - name: Run the stack, set up the owner, open an OpenCode pane, reset + run: node scripts/dev-stack-ci.mjs < /dev/null + + - name: Nothing of the dev stack is left + if: always() + run: | + test -z "$(docker ps -aq --filter label=com.oll4com.spaceapp.dev-stack)" + test -z "$(docker volume ls -q --filter label=com.oll4com.spaceapp.dev-stack)" + test ! -e .dev-stack diff --git a/.gitignore b/.gitignore index 81e68b26a..13e8e29e3 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ coverage/ playwright-report/ test-results/ scratch/ +# npm run dev:stack state: secrets, provider homes, OpenCode install (reset removes it). +/.dev-stack/ .npm/ apps/desktop/release/*-unpacked/ apps/desktop/release/*.7z diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 290739cb7..e41c41113 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,6 +50,96 @@ select a short temporary directory you own and set `TMPDIR` for the command. On Windows run this particular smoke check inside WSL or a Linux development container. It does not certify the Windows/macOS installer. +## Run the full app from source + +`npm run dev:stack` starts a working SpaceApp from this checkout with hot +reload. It never touches an installed SpaceApp: it has its own compose project +(`spaceapp-dev-stack`), its own git-ignored state directory (`.dev-stack/`) +and its own ports, and every port it opens binds to `127.0.0.1` only. + +Prerequisites: the Node and npm versions above, `npm ci` done, and Docker with +the Compose plugin (`docker compose version`). Docker runs only PostgreSQL +(pgvector) and Temporal, with the images and digests pinned in +`packages/run-spaceapp/templates/compose.yml` (read from that file at start). + +```sh +npm run dev:stack # start; prints the URL and the one-time owner setup token +npm run dev:stack -- down # stop the processes and containers, keep the data +npm run dev:stack -- reset # remove the containers, volumes and .dev-stack/ it created +npm run dev:stack -- rebuild # rebuild packages/* and restart the API, worker and CLI host +npm run dev:stack -- status +``` + +In order, `dev:stack` starts PostgreSQL and Temporal and waits for their +health checks, builds the workspace packages, runs the migrations, installs the +OpenCode version pinned in the `Dockerfile` into `.dev-stack/cli/vendor/` (not +globally), and then runs from source, each with its own prefix in the output: + +| Process | What runs | Reload | +| --- | --- | --- | +| `[api]` | `tsx watch apps/api/src/server.ts` | restarts on a change in `apps/api/src` | +| `[worker-build]`, `[worker]` | `tsc --watch` and `node --watch dist/worker.js` | restarts on a change in `apps/worker/src` | +| `[web]` | Vite dev server | hot reload in the browser for `apps/web/src` | +| `[cli-host]` | `packages/cli-host/dist/main.js` on `.dev-stack/run/cli-host.sock` | `rebuild` | + +A change in `packages/*` is picked up with `npm run dev:stack -- rebuild` in a +second terminal: it builds the packages and the running stack restarts the +processes that load them. + +Open the printed URL, enter the setup token, create the owner and sign in +(the dev-login shortcut is off). The Getting Started room opens; an OpenCode +pane starts through the from-source CLI host with the same wrapper as the +image (`deploy/docker/public-cli-wrapper.sh`, pointed at `.dev-stack/cli/`). +OpenCode's free model needs no provider account. The browser service and the +optional companions stay disabled. + +Ports (environment variables; the defaults are never 4910/4911): + +| Variable | Default | +| --- | --- | +| `SPACE_DEV_STACK_WEB_PORT` (open this one) | 4921 | +| `SPACE_DEV_STACK_API_PORT` | 4920 | +| `SPACE_DEV_STACK_POSTGRES_PORT` | 4922 | +| `SPACE_DEV_STACK_TEMPORAL_PORT` | 4923 | + +`SPACE_DEV_STACK_DIR` moves the state directory and `SPACE_DEV_STACK_PROJECT` +renames the compose project. The database password, session secret and setup +token are generated once per state directory in `.dev-stack/secrets/` with +mode 0600; only the setup token is printed. `SPACE_*` variables of your shell +are not passed to the processes. + +If a process exits, `dev:stack` names it (`dev-stack: worker exited +unexpectedly ...`) and stops the stack. While it runs it checks `/readyz` and +prints the dependency that fails (for example `worker=ERROR`). + +Troubleshooting: + +- *Port ... is in use*: set the matching `SPACE_DEV_STACK_*_PORT`. +- *The dev stack is already running*: `npm run dev:stack -- down` first. +- */readyz did not report ok*: the last line names the failing dependency; + `docker compose --project-name spaceapp-dev-stack --file .dev-stack/compose.yml logs` + shows the PostgreSQL and Temporal side. +- *was not created by dev:stack*: `SPACE_DEV_STACK_DIR` names a directory that + already holds other files; `dev:stack` never uses or removes it. +- A deep checkout path puts the CLI host socket in a private directory under + `TMPDIR` (Unix sockets are limited to about 100 bytes). + +Platform notes: + +- **Linux**: Docker Engine with the Compose plugin. +- **macOS**: Docker Desktop, or Colima (`colima start`, then `docker context use colima`). + The ports are published on the Mac's `127.0.0.1`. +- **Windows 11**: run everything inside WSL2 (Ubuntu), with the checkout on the + Linux file system (`~/src/spaceapp`, not `/mnt/c/...`, for file watching and + speed) and Docker Desktop's WSL integration or Docker Engine inside WSL. + Open the URL in a Windows browser; WSL forwards `127.0.0.1`. Native Windows is + refused with a message. + +The `Dev stack` CI workflow runs `scripts/dev-stack-ci.mjs` on `ubuntu-latest`: +it starts the stack without a TTY, waits for `/readyz`, creates the owner over +HTTP, opens an OpenCode pane through the API, checks that it runs, then runs +`reset` and checks that no container, volume or process is left. + ## Make and verify a small change Work on a branch, not on `main`, so the change can become a pull request: diff --git a/apps/web/tests/vite-config.test.ts b/apps/web/tests/vite-config.test.ts index f33db6382..d4b006827 100644 --- a/apps/web/tests/vite-config.test.ts +++ b/apps/web/tests/vite-config.test.ts @@ -1,7 +1,7 @@ // @vitest-environment node import { describe, expect, it } from "vitest"; -import { manualChunkForModule } from "../vite.config.js"; +import { devApiOrigin, manualChunkForModule } from "../vite.config.js"; describe("web bundle chunk policy", () => { it("isolates React and xterm while leaving feature modules available for lazy chunks", () => { @@ -16,3 +16,12 @@ describe("web bundle chunk policy", () => { expect(manualChunkForModule("/workspace/src/features/browser-pane/BrowserPane.tsx")).toBeUndefined(); }); }); + +describe("dev server API proxy", () => { + it("targets the installed API port by default and the dev-stack API when SPACE_WEB_API_ORIGIN names it", () => { + expect(devApiOrigin({})).toBe("http://127.0.0.1:4910"); + expect(devApiOrigin({ SPACE_WEB_API_ORIGIN: "http://127.0.0.1:4920" })).toBe("http://127.0.0.1:4920"); + expect(() => devApiOrigin({ SPACE_WEB_API_ORIGIN: "http://0.0.0.0:4920" })).toThrow(/127\.0\.0\.1/); + expect(() => devApiOrigin({ SPACE_WEB_API_ORIGIN: "https://example.com" })).toThrow(); + }); +}); diff --git a/apps/web/vite.config.ts b/apps/web/vite.config.ts index cd7612d98..682051936 100644 --- a/apps/web/vite.config.ts +++ b/apps/web/vite.config.ts @@ -17,6 +17,13 @@ export function manualChunkForModule(moduleId: string): string | undefined { return undefined; } +export function devApiOrigin(env: NodeJS.ProcessEnv): string { + const origin = env.SPACE_WEB_API_ORIGIN?.trim(); + if (!origin) return "http://127.0.0.1:4910"; + if (!/^http:\/\/(?:127\.0\.0\.1|localhost):\d{2,5}$/.test(origin)) throw new Error("SPACE_WEB_API_ORIGIN must be http://127.0.0.1:."); + return origin; +} + export default defineConfig({ plugins: [react(), demoBundleBoundaryPlugin(), retainOpenTabAssets()], // The icon package ships as source; the alias keeps worktrees (which share live node_modules) resolving this checkout. @@ -33,13 +40,9 @@ export default defineConfig({ server: { host: process.env.SPACE_WEB_HOST ?? "0.0.0.0", port: Number.parseInt(process.env.SPACE_WEB_PORT ?? "4911", 10), - proxy: { - "/api": "http://127.0.0.1:4910", - "/plugins": "http://127.0.0.1:4910", - "/healthz": "http://127.0.0.1:4910", - "/readyz": "http://127.0.0.1:4910", - "/version": "http://127.0.0.1:4910" - } + // npm run dev:stack runs its API on its own port (SPACE_WEB_API_ORIGIN) so it never reaches an installed SpaceApp on 4910. + // /api also carries the pane terminal WebSockets. + proxy: Object.fromEntries(["/api", "/plugins", "/healthz", "/readyz", "/version"].map((path) => [path, { target: devApiOrigin(process.env), ws: path === "/api" }])) }, test: { environment: "jsdom", diff --git a/deploy/docker/opencode-first-run.mjs b/deploy/docker/opencode-first-run.mjs index 2ff0a3bdc..b2a45991d 100644 --- a/deploy/docker/opencode-first-run.mjs +++ b/deploy/docker/opencode-first-run.mjs @@ -40,7 +40,8 @@ export function toolCycleSucceeded(result,token) { const answer=list.findIndex((e,i)=>i>tool&&e.type==='text'&&e.part?.text?.includes(token)); return answer>tool; } -const MEMORY_DIRECTORY='/var/lib/spaceapp/memory'; +// npm run dev:stack runs the wrapper outside the container and names its own memory directory. +const MEMORY_DIRECTORY=process.env.SPACE_MEMORY_ROOT||'/var/lib/spaceapp/memory'; // The 1.1.x default. Configs that still carry exactly this value get the new // default; any other value is an owner policy and is preserved. const LEGACY_EXTERNAL_DIRECTORY={'*':'ask',[MEMORY_DIRECTORY]:'allow',[`${MEMORY_DIRECTORY}/*`]:'allow'}; diff --git a/package.json b/package.json index 00e4c39c6..dbfc73c3d 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ "dev:api": "npm run dev -w @space/api", "dev:web": "npm run dev -w @space/web", "dev:worker": "npm run dev -w @space/worker", + "dev:stack": "node scripts/dev-stack.mjs", "db:migrate": "npm run migrate -w @space/db", "build": "npm run build -w @space/contracts && npm run build -w @space/icons && npm run build -w @space/streaming && npm run build -w @space/opencode-control && npm run build -w @space/memory-graph && npm run build -w @space/browser-host && npm run build -w @space/cli-host && npm run build -w @space/codex-app-server && npm run build -w @space/runtime && npm run build -w @space/db && npm run build -w @space/gateway && node scripts/build-demo-projects.mjs && npm run build -w @space/api && npm run build -w @space/worker && npm run build -w @space/web && node scripts/fix-build-permissions.mjs", "public:export:verify": "node scripts/public-export.mjs --verify", @@ -58,7 +59,7 @@ "build:demos": "node scripts/build-demo-projects.mjs", "test:architecture": "node --test scripts/tests/domain-boundaries.test.mjs scripts/tests/composition-root-size.test.mjs scripts/tests/quality-known-limitations.test.mjs && node scripts/domain-boundaries.mjs", "smoke:offline": "npm run build -w @space/cli-host && node scripts/contributor-smoke.mjs", - "test:contributor": "node --test scripts/tests/contributor-smoke.test.mjs", + "test:contributor": "node --test scripts/tests/contributor-smoke.test.mjs scripts/tests/dev-stack.test.mjs", "lint": "eslint --max-warnings=-1 apps packages scripts" }, "devDependencies": { diff --git a/scripts/dev-stack-ci.mjs b/scripts/dev-stack-ci.mjs new file mode 100644 index 000000000..f6e50bbcd --- /dev/null +++ b/scripts/dev-stack-ci.mjs @@ -0,0 +1,98 @@ +#!/usr/bin/env node +// CI check of npm run dev:stack (#65), without a TTY: start the stack, wait for /readyz, create the owner over HTTP with +// the setup token, open an OpenCode pane through the API and check that its CLI session runs, then reset and check that +// no container, volume or process of the dev stack is left. +import { spawn, spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import process from "node:process"; +import { repoRoot, resolveConfig, statePaths } from "./dev-stack.mjs"; + +const config = resolveConfig(process.env); +const paths = statePaths(config); +const api = `http://${config.host}:${config.ports.api}`; +const web = `http://${config.host}:${config.ports.web}`; +const label = `com.oll4com.spaceapp.dev-stack=${config.project}`; +const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const step = (text) => process.stdout.write(`dev-stack-ci: ${text}\n`); + +async function until(what, check, timeoutMs, isAlive = () => true) { + const deadline = Date.now() + timeoutMs; + let last; + while (Date.now() < deadline) { + if (!isAlive()) throw new Error(`dev:stack exited while waiting for ${what}.`); + try { const value = await check(); if (value) return value; } catch (error) { last = error; } + await delay(2_000); + } + throw new Error(`Timed out waiting for ${what}${last ? ` (${last.message})` : ""}.`); +} + +function leftovers() { + const containers = spawnSync("docker", ["ps", "-a", "-q", "--filter", `label=${label}`], { encoding: "utf8" }).stdout?.trim() ?? ""; + const volumes = spawnSync("docker", ["volume", "ls", "-q", "--filter", `label=${label}`], { encoding: "utf8" }).stdout?.trim() ?? ""; + const processes = (spawnSync("ps", ["-eo", "pid=,args="], { encoding: "utf8" }).stdout ?? "").split("\n") + .filter((line) => line.includes(config.stateDir) || line.includes(paths.socket) || /scripts\/dev-stack\.mjs up/.test(line)) + .filter((line) => !line.includes("dev-stack-ci.mjs") && !line.includes(" ps -eo")); + return { containers, volumes, processes, stateDir: existsSync(config.stateDir) }; +} + +async function main() { + step("starting npm run dev:stack"); + const stack = spawn(process.execPath, [join(repoRoot, "scripts", "dev-stack.mjs"), "up"], { cwd: repoRoot, stdio: ["ignore", "inherit", "inherit"] }); + let stackExited = false; + stack.once("exit", () => { stackExited = true; }); + const alive = () => !stackExited; + let failure; + try { + await until("/readyz ok: true", async () => (await (await fetch(`${api}/readyz`)).json()).ok === true, 20 * 60_000, alive); + step("/readyz reports ok: true"); + const webIndex = await fetch(web); + if (!webIndex.ok) throw new Error(`The Vite dev server answered ${webIndex.status}.`); + const token = (await readFile(join(paths.secrets, "setup-token"), "utf8")).trim(); + // Through the Vite dev server, as a browser would, so the proxy is part of the check. + const claim = await fetch(`${web}/api/setup/claim`, { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ token, email: "owner@dev-stack.invalid", password: `dev-stack-${token.slice(0, 16)}` }) + }); + if (!claim.ok) throw new Error(`Owner setup failed: HTTP ${claim.status}.`); + const { starterRoomId } = await claim.json(); + const cookie = claim.headers.getSetCookie().map((value) => value.split(";")[0]).join("; "); + step("owner created with the setup token"); + const csrf = await (await fetch(`${web}/api/auth/csrf`, { headers: { cookie } })).json(); + const headers = { cookie, "content-type": "application/json", [csrf.headerName]: csrf.csrfToken }; + const created = await fetch(`${web}/api/rooms/${encodeURIComponent(starterRoomId)}/panes`, { + method: "POST", headers, body: JSON.stringify({ panes: [{ mode: "TERMINAL", terminalRuntimeId: "cli:opencode" }] }) + }); + if (!created.ok) throw new Error(`Opening the OpenCode pane failed: HTTP ${created.status} ${await created.text()}`); + const paneId = (await created.json()).data[0].id; + const session = await fetch(`${web}/api/panes/${encodeURIComponent(paneId)}/cli/session`, { method: "POST", headers, body: JSON.stringify({ runtimeId: "cli:opencode" }) }); + if (!session.ok) throw new Error(`Starting the OpenCode CLI session failed: HTTP ${session.status} ${await session.text()}`); + const running = await until("the OpenCode pane to run", async () => { + const current = await (await fetch(`${web}/api/panes/${encodeURIComponent(paneId)}/cli/session`, { headers: { cookie } })).json(); + if (current?.status === "ERROR" || current?.status === "EXITED") throw new Error(`OpenCode pane ${current.status}: ${current.statusReason}`); + return current?.status === "RUNNING" ? current : null; + }, 3 * 60_000, alive); + step(`OpenCode pane ${paneId} is ${running.status}`); + } catch (error) { + failure = error; + } finally { + step("npm run dev:stack -- reset"); + const reset = spawnSync(process.execPath, [join(repoRoot, "scripts", "dev-stack.mjs"), "reset"], { cwd: repoRoot, stdio: "inherit" }); + await until("dev:stack to stop", () => stackExited, 60_000).catch((error) => { failure ??= error; stack.kill("SIGKILL"); }); + if (reset.status !== 0) failure ??= new Error(`reset failed with exit ${reset.status}.`); + } + const left = leftovers(); + step(`after reset: containers=[${left.containers}] volumes=[${left.volumes}] processes=${left.processes.length} stateDir=${left.stateDir}`); + if (left.containers || left.volumes || left.processes.length || left.stateDir) { + for (const line of left.processes) step(`left: ${line}`); + failure ??= new Error("reset left containers, volumes, processes or the state directory behind."); + } + if (failure) throw failure; + step("passed"); +} + +main().catch((error) => { + process.stderr.write(`dev-stack-ci: ${error.message}\n`); + process.exitCode = 1; +}); diff --git a/scripts/dev-stack.mjs b/scripts/dev-stack.mjs new file mode 100644 index 000000000..359dae7f9 --- /dev/null +++ b/scripts/dev-stack.mjs @@ -0,0 +1,588 @@ +#!/usr/bin/env node +// npm run dev:stack [-- up|down|reset|rebuild|status] +// +// Runs the whole SpaceApp from source for contributors (ONB-001, #65): PostgreSQL (pgvector) and Temporal in Docker with +// the images pinned in packages/run-spaceapp/templates/compose.yml, the migrations, then the API (tsx watch), the Vite +// web dev server, the Temporal worker (tsc --watch + node --watch) and the CLI host, all from this checkout. It is +// isolated from an installed SpaceApp: its own compose project, its own git-ignored state directory and its own ports, +// all bound to 127.0.0.1. Secrets live in the state directory with mode 0600 and only the setup token is printed. +import { spawn as nodeSpawn, spawnSync } from "node:child_process"; +import { createHash, randomBytes } from "node:crypto"; +import { existsSync, unwatchFile, watchFile } from "node:fs"; +import { chmod, lstat, mkdir, readdir, readFile, rm, stat, symlink, writeFile } from "node:fs/promises"; +import { createServer } from "node:net"; +import { homedir, tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; + +export const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +export const DEFAULT_PROJECT = "spaceapp-dev-stack"; +export const STATE_MARKER = ".spaceapp-dev-stack"; +export const INSTALL_PORTS = Object.freeze([4910, 4911]); +export const DEFAULT_PORTS = Object.freeze({ api: 4920, web: 4921, postgres: 4922, temporal: 4923 }); +const PORT_ENV = Object.freeze({ + api: "SPACE_DEV_STACK_API_PORT", + web: "SPACE_DEV_STACK_WEB_PORT", + postgres: "SPACE_DEV_STACK_POSTGRES_PORT", + temporal: "SPACE_DEV_STACK_TEMPORAL_PORT" +}); +// Library packages the apps import from their dist/ (the order of the root "build" script). +export const LIBRARY_PACKAGES = Object.freeze([ + "@space/contracts", "@space/icons", "@space/streaming", "@space/opencode-control", "@space/memory-graph", + "@space/browser-host", "@space/cli-host", "@space/codex-app-server", "@space/runtime", "@space/db", "@space/gateway" +]); +const SECRET_NAMES = Object.freeze(["postgres-password", "session-secret", "setup-token"]); +const UNIX_SOCKET_MAX_BYTES = 100; + +/** The pinned postgres and temporal images, read from the launcher's compose template (never copied by hand). */ +export function readPinnedImages(composeText) { + const images = {}; + for (const service of ["postgres", "temporal"]) { + const block = composeText.match(new RegExp(`^ {2}${service}:\\n((?: {4}.*\\n|\\s*\\n)*)`, "m"))?.[1] ?? ""; + const image = block.match(/^ {4}image:\s*(\S+)\s*$/m)?.[1]; + if (!image || !/@sha256:[0-9a-f]{64}$/.test(image)) { + throw new Error(`packages/run-spaceapp/templates/compose.yml has no digest-pinned image for ${service}.`); + } + images[service] = image; + } + return images; +} + +/** The OpenCode package and version the CLI image installs (Dockerfile), e.g. "@opencode/cli@2.0.32". */ +export function readOpenCodePackage(dockerfileText) { + const match = dockerfileText.match(/npm install --global[^\n]*?\s((?:@opencode\/cli|opencode-ai)@\d[\w.+-]*)/); + if (!match) throw new Error("Dockerfile does not pin an OpenCode package version."); + const spec = match[1]; + const at = spec.lastIndexOf("@"); + return { spec, name: spec.slice(0, at), version: spec.slice(at + 1) }; +} + +function parsePort(name, raw) { + if (!/^\d{1,5}$/.test(raw)) throw new Error(`${name} must be a port number, got "${raw}".`); + const port = Number(raw); + if (port < 1024 || port > 65535) throw new Error(`${name} must be between 1024 and 65535.`); + return port; +} + +export function resolveConfig(env = process.env, root = repoRoot) { + const ports = {}; + for (const [key, envName] of Object.entries(PORT_ENV)) { + ports[key] = env[envName] ? parsePort(envName, env[envName]) : DEFAULT_PORTS[key]; + if (INSTALL_PORTS.includes(ports[key])) { + throw new Error(`${envName}=${ports[key]} collides with an installed SpaceApp (4910/4911). Choose another port.`); + } + } + if (new Set(Object.values(ports)).size !== 4) throw new Error("The dev-stack ports must all be different."); + const project = env.SPACE_DEV_STACK_PROJECT || DEFAULT_PROJECT; + if (!/^[a-z0-9][a-z0-9_-]{0,62}$/.test(project)) throw new Error("SPACE_DEV_STACK_PROJECT must be a lowercase compose project name."); + if (project === "spaceapp") throw new Error("SPACE_DEV_STACK_PROJECT must not be the installed SpaceApp's project name."); + const stateDir = resolve(root, env.SPACE_DEV_STACK_DIR || ".dev-stack"); + // reset removes this directory: never the checkout, an ancestor of it, the home directory or /. + if (stateDir === "/" || stateDir === homedir() || `${resolve(root)}/`.startsWith(`${stateDir}/`)) { + throw new Error(`SPACE_DEV_STACK_DIR=${stateDir} must be a directory of its own, not the checkout, its parent or your home.`); + } + return { root, project, ports, stateDir, host: "127.0.0.1" }; +} + +export function statePaths(config) { + const s = config.stateDir; + const socketCandidate = join(s, "run", "cli-host.sock"); + // Unix socket paths are limited to about 104-108 bytes; long checkouts put the socket in a private temp directory. + const socketDir = Buffer.byteLength(socketCandidate) > UNIX_SOCKET_MAX_BYTES + ? join(tmpdir(), `spaceapp-dev-${createHash("sha256").update(s).digest("hex").slice(0, 12)}`) + : join(s, "run"); + return { + marker: join(s, STATE_MARKER), + secrets: join(s, "secrets"), + compose: join(s, "compose.yml"), + pid: join(s, "dev-stack.pid"), + rebuildStamp: join(s, "rebuild.stamp"), + bin: join(s, "bin"), + cliState: join(s, "cli"), + data: join(s, "data"), + memory: join(s, "data", "memory"), + workspaces: join(s, "workspaces"), + socketDir, + socket: join(socketDir, "cli-host.sock") + }; +} + +/** The compose file of the dev stack: only PostgreSQL and Temporal, every port on 127.0.0.1. */ +export function renderCompose({ images, ports, host = "127.0.0.1", secretsDir, project }) { + const label = ` com.oll4com.spaceapp.dev-stack: "${project}"`; + return `# Generated by scripts/dev-stack.mjs. Do not edit; images come from packages/run-spaceapp/templates/compose.yml. +name: ${project} +services: + postgres: + image: ${images.postgres} + labels: +${label} + environment: + POSTGRES_DB: spaceapp + POSTGRES_USER: spaceapp + POSTGRES_PASSWORD_FILE: /run/secrets/postgres-password + ports: + - "${host}:${ports.postgres}:5432" + volumes: + - postgres-data:/var/lib/postgresql/data + secrets: + - postgres-password + healthcheck: + test: ["CMD-SHELL", "pg_isready -U spaceapp -d spaceapp"] + interval: 3s + timeout: 5s + retries: 40 + temporal: + image: ${images.temporal} + labels: +${label} + depends_on: + postgres: + condition: service_healthy + environment: + DB: postgres12 + DB_PORT: 5432 + POSTGRES_USER: spaceapp + POSTGRES_SEEDS: postgres + DYNAMIC_CONFIG_FILE_PATH: config/dynamicconfig/docker.yaml + entrypoint: ["/bin/sh", "-ec"] + command: + - export POSTGRES_PWD="$$(cat /run/secrets/postgres-password)"; exec /etc/temporal/entrypoint.sh autosetup + ports: + - "${host}:${ports.temporal}:7233" + secrets: + - postgres-password + healthcheck: + test: ["CMD-SHELL", "temporal operator cluster health --address temporal:7233 >/dev/null && temporal operator namespace describe --namespace default --address temporal:7233 >/dev/null"] + interval: 5s + timeout: 10s + retries: 60 +volumes: + postgres-data: + labels: +${label} +secrets: + postgres-password: + file: ${JSON.stringify(join(secretsDir, "postgres-password"))} +`; +} + +/** Creates the state directory and its secrets once (mode 0600, directory 0700); later runs reuse them. */ +export async function ensureState(config, { random = (bytes) => randomBytes(bytes).toString("base64url") } = {}) { + const paths = statePaths(config); + if (existsSync(config.stateDir) && !existsSync(paths.marker)) { + const entries = await readdir(config.stateDir); + if (entries.length > 0) throw new Error(`${config.stateDir} exists and was not created by dev:stack; refusing to use it.`); + } + await mkdir(config.stateDir, { recursive: true, mode: 0o700 }); + await chmod(config.stateDir, 0o700); + if (!existsSync(paths.marker)) await writeFile(paths.marker, `${config.project}\n`, { mode: 0o600 }); + for (const dir of [paths.secrets, paths.bin, paths.cliState, join(paths.cliState, "providers"), join(paths.cliState, "imported-credentials"), + join(paths.cliState, "vendor"), paths.data, join(paths.data, "artifacts"), paths.memory, paths.workspaces, paths.socketDir]) { + await mkdir(dir, { recursive: true, mode: 0o700 }); + } + const secrets = {}; + for (const name of SECRET_NAMES) { + const file = join(paths.secrets, name); + if (!existsSync(file)) { + // The setup token schema needs at least 32 characters; 32 random bytes give 43. + await writeFile(file, `${random(32)}\n`, { mode: 0o600, flag: "wx" }); + } + await chmod(file, 0o600); + secrets[name] = (await readFile(file, "utf8")).trim(); + } + return { paths, secrets }; +} + +function shellQuote(value) { + return `'${String(value).replaceAll("'", "'\\''")}'`; +} + +/** The installed CLI wrapper, pointed at the state directory and this checkout instead of the container paths. */ +export function renderCliWrapper(wrapperText, { cliStateRoot, deployDockerDir, memoryRoot }) { + if (!/^set -euo pipefail$/m.test(wrapperText)) throw new Error("CLI wrapper has an unexpected header."); + const rendered = wrapperText + .replace(/^set -euo pipefail$/m, `set -euo pipefail\nexport SPACE_MEMORY_ROOT=${shellQuote(memoryRoot)}`) + .replaceAll("/var/lib/spaceapp-cli", cliStateRoot) + .replaceAll("/app/deploy/docker", deployDockerDir); + if (/\/var\/lib\/spaceapp-cli|\/app\/deploy\//.test(rendered)) throw new Error("CLI wrapper still names a container path."); + return rendered; +} + +/** Drops SPACE_* and SPACEAPP_* variables of the caller's shell so an installed SpaceApp's settings never leak in. */ +export function baseEnv(env = process.env) { + return Object.fromEntries(Object.entries(env).filter(([key]) => !/^SPACE(?:APP)?_/.test(key))); +} + +export function serviceEnvironments(config, { paths, secrets }, env = process.env) { + const { host, ports } = config; + const databaseUrl = `postgres://spaceapp:${encodeURIComponent(secrets["postgres-password"])}@${host}:${ports.postgres}/spaceapp`; + const vendorBin = join(paths.cliState, "vendor", "opencode", "node_modules", ".bin"); + const base = baseEnv(env); + const shared = { + SPACE_RUNTIME_STORE: "postgres", + SPACE_DATABASE_URL: databaseUrl, + SPACE_TEMPORAL_ADDRESS: `${host}:${ports.temporal}`, + SPACEAPP_WORKFLOWS_ENABLED: "true", + SPACE_MEMORY_ROOT: paths.memory, + SPACE_STARTER_MEMORY_ROOT: join(config.root, "starter-memory"), + SPACE_MEMORY_GRAPH_ROOT: join(paths.data, "memory-graph"), + SPACE_MEMORY_INDEX_PATH: join(paths.memory, "space_memory.md"), + SPACE_MEMORY_MONTHLY_PATH: join(paths.memory, "space_memory_monthly.md"), + SPACE_MEMORY_LOCK_PATH: join(paths.memory, ".memory.lock"), + SPACE_ARTIFACT_ROOT: join(paths.data, "artifacts"), + SPACE_CODEX_HOME: join(paths.cliState, "providers", "codex"), + SPACE_TELEGRAM_SECRET_ROOT: join(paths.data, "telegram"), + SPACE_TELEGRAM_OWNERSHIP_MANIFEST: join(paths.data, "integrations", "telegram", "thread-ownership.json") + }; + const api = { + ...base, + ...shared, + NODE_ENV: "development", + SPACE_PUBLIC_DISTRIBUTION: "true", + SPACE_DEV_LOGIN: "false", + SPACE_AGENT_TOOLS_WRITER: "disabled", + SPACE_SESSION_SECRET: secrets["session-secret"], + SPACE_SETUP_TOKEN_FILE: join(paths.secrets, "setup-token"), + SPACE_API_HOST: host, + SPACE_API_PORT: String(ports.api), + SPACE_API_ORIGIN: `http://${host}:${ports.api}`, + SPACE_INSTALL_ROOT: config.stateDir, + SPACE_PROJECTS_ROOT: join(paths.data, "projects"), + SPACE_DEMO_PROJECTS_VAR_ROOT: join(paths.data, "demo-projects"), + SPACE_DEMO_PROJECTS_ROOT: join(config.root, "demos"), + SPACE_CLI_ENABLED: "true", + SPACE_CLI_HOST_SOCKET: paths.socket, + SPACE_CLI_COMMAND_PATH: paths.bin, + SPACE_CLI_WORKSPACE_ROOT: paths.workspaces, + SPACE_CODEX_APP_SERVER_HOME: join(paths.cliState, "providers", "codex"), + SPACE_CODEX_GOALS_DB_PATH: join(paths.data, "codex-goals.sqlite"), + SPACE_CODEX_CLI_MODE_DEFAULTS_PROJECTION_PATH: join(paths.data, "codex-cli-mode-defaults-v1.json"), + SPACE_BROWSER_SESSIONS_ENABLED: "false" + }; + const worker = { ...base, ...shared, NODE_ENV: "development" }; + const web = { + ...base, + SPACE_WEB_HOST: host, + SPACE_WEB_PORT: String(ports.web), + SPACE_WEB_API_ORIGIN: `http://${host}:${ports.api}` + }; + const cliHost = { + ...base, + SPACE_PUBLIC_DISTRIBUTION: "true", + SPACE_MEMORY_ROOT: paths.memory, + SPACE_CLI_HOST_SOCKET: paths.socket, + PATH: [paths.bin, vendorBin, base.PATH].filter(Boolean).join(":") + }; + return { api, worker, web, cliHost, databaseUrl }; +} + +/** The long-running from-source processes, in start order. */ +export function processSpecs(config, envs) { + const bin = (name) => join(config.root, "node_modules", ".bin", name); + const node = process.execPath; + return [ + { name: "cli-host", command: node, args: [join(config.root, "packages", "cli-host", "dist", "main.js")], cwd: config.root, env: envs.cliHost, restartOnRebuild: true }, + { name: "api", command: bin("tsx"), args: ["watch", "--clear-screen=false", "src/server.ts"], cwd: join(config.root, "apps", "api"), env: envs.api, restartOnRebuild: true }, + { name: "worker-build", command: bin("tsc"), args: ["-p", "tsconfig.json", "--watch", "--preserveWatchOutput"], cwd: join(config.root, "apps", "worker"), env: envs.worker }, + { name: "worker", command: node, args: ["--watch-path=dist", "dist/worker.js"], cwd: join(config.root, "apps", "worker"), env: envs.worker, restartOnRebuild: true }, + { name: "web", command: bin("vite"), args: ["--host", config.host, "--port", String(config.ports.web), "--strictPort"], cwd: join(config.root, "apps", "web"), env: envs.web } + ]; +} + +function prefixLines(name, stream, write) { + let pending = ""; + stream?.setEncoding?.("utf8"); + stream?.on("data", (chunk) => { + pending += chunk; + const lines = pending.split(/\r?\n/); + pending = lines.pop() ?? ""; + for (const line of lines) write(`[${name}] ${line}\n`); + }); + stream?.on("end", () => { if (pending) write(`[${name}] ${pending}\n`); }); +} + +/** + * Starts the processes and stops all of them when one exits on its own, with a message naming it: a crashed worker is a + * visible failure, never a silent hang. Resolves with the exit code once everything has stopped. + */ +export function superviseProcesses(specs, { spawn = nodeSpawn, write = (text) => process.stdout.write(text), onStart } = {}) { + const children = new Map(); + let stopping = false; + let finished = false; + let exitCode = 0; + let resolveDone; + const done = new Promise((resolveFn) => { resolveDone = resolveFn; }); + const finishIfIdle = () => { if (children.size === 0 && stopping && !finished) { finished = true; resolveDone(exitCode); } }; + const stop = (reason, code = 0) => { + if (stopping) return; + stopping = true; + if (code) exitCode = code; + if (reason) write(`dev-stack: ${reason}\n`); + for (const child of children.values()) child.kill("SIGTERM"); + finishIfIdle(); + }; + const start = (spec) => { + const child = spawn(spec.command, spec.args, { cwd: spec.cwd, env: spec.env, stdio: ["ignore", "pipe", "pipe"] }); + children.set(spec.name, child); + prefixLines(spec.name, child.stdout, write); + prefixLines(spec.name, child.stderr, write); + child.once("error", (error) => { + if (children.get(spec.name) !== child) return; + children.delete(spec.name); + stop(`${spec.name} failed to start (${error.code ?? error.message}). Stopping the dev stack.`, 1); + finishIfIdle(); + }); + child.once("exit", (code, signal) => { + if (children.get(spec.name) !== child) return; + children.delete(spec.name); + if (!stopping && !child.spaceRestarting) { + stop(`${spec.name} exited unexpectedly (${code ?? signal}). Stopping the dev stack; see the [${spec.name}] lines above.`, 1); + } + finishIfIdle(); + }); + onStart?.(spec, child); + return child; + }; + for (const spec of specs) start(spec); + const restart = (names) => { + for (const spec of specs.filter((item) => names.includes(item.name))) { + const child = children.get(spec.name); + if (!child || stopping) continue; + child.spaceRestarting = true; + child.once("exit", () => { if (!stopping) start(spec); }); + child.kill("SIGTERM"); + } + }; + return { done, stop, restart, children, isRunning: () => !stopping }; +} + +async function portFree(port, host) { + return new Promise((resolveFn) => { + const server = createServer(); + server.once("error", () => resolveFn(false)); + server.listen({ port, host, exclusive: true }, () => server.close(() => resolveFn(true))); + }); +} + +function runChecked(command, args, options = {}) { + const result = spawnSync(command, args, { stdio: "inherit", ...options }); + if (result.error) throw new Error(`${command} could not start: ${result.error.message}`); + if (result.status !== 0) throw new Error(`${command} ${args.join(" ")} failed (exit ${result.status}).`); +} + +export function composeArgs(config, paths, ...rest) { + return ["compose", "--project-name", config.project, "--file", paths.compose, ...rest]; +} + +async function waitForReady(url, { timeoutMs = 240_000, isAlive = () => true, fetchImpl = fetch } = {}) { + const deadline = Date.now() + timeoutMs; + let last = "no answer"; + while (Date.now() < deadline) { + if (!isAlive()) throw new Error("A dev-stack process stopped before SpaceApp was ready."); + try { + const response = await fetchImpl(url, { signal: AbortSignal.timeout(5_000) }); + const body = await response.json().catch(() => ({})); + if (response.ok && body.ok === true) return body; + last = summarizeReadiness(body); + } catch (error) { + last = error.cause?.code ?? error.name; + } + await new Promise((resolveFn) => setTimeout(resolveFn, 2_000)); + } + throw new Error(`/readyz did not report ok: true within ${Math.round(timeoutMs / 1000)} s (last: ${last}).`); +} + +/** The failing dependencies of a /readyz answer (names and states only, never a value that could hold a secret). */ +export function summarizeReadiness(body) { + const deps = body && typeof body === "object" ? body.dependencies : null; + if (!deps || typeof deps !== "object") return "no readiness answer"; + const failing = []; + if (deps.database === "UNAVAILABLE") failing.push("database=UNAVAILABLE (PostgreSQL)"); + if (deps.sessionRevocations && deps.sessionRevocations !== "RUNNING") failing.push(`sessionRevocations=${deps.sessionRevocations}`); + if (deps.worker && deps.worker !== "disabled" && deps.worker !== "RUNNING") failing.push(`worker=${deps.worker} (Temporal worker or Temporal)`); + if (deps.cliHost === "UNAVAILABLE") failing.push("cliHost=UNAVAILABLE (CLI host)"); + if (deps.cliAdminHost === "UNAVAILABLE") failing.push("cliAdminHost=UNAVAILABLE"); + if (deps.browserHost && !["in-process", "RUNNING", "DISABLED"].includes(deps.browserHost)) failing.push(`browserHost=${deps.browserHost}`); + return failing.length ? failing.join(", ") : `ok=${String(body.ok)}`; +} + +async function readPid(paths) { + try { + const pid = Number((await readFile(paths.pid, "utf8")).trim()); + if (!Number.isInteger(pid) || pid <= 1) return null; + process.kill(pid, 0); + return pid; + } catch { return null; } +} + +async function installOpenCode(config, paths) { + const pin = readOpenCodePackage(await readFile(join(config.root, "Dockerfile"), "utf8")); + const prefix = join(paths.cliState, "vendor", "opencode"); + const installed = join(prefix, "node_modules", ...pin.name.split("/"), "package.json"); + if (existsSync(installed) && JSON.parse(await readFile(installed, "utf8")).version === pin.version) return pin; + process.stdout.write(`dev-stack: installing ${pin.spec} into ${prefix} (not globally)\n`); + await mkdir(prefix, { recursive: true, mode: 0o700 }); + runChecked("npm", ["install", "--prefix", prefix, "--no-audit", "--no-fund", "--no-save", "--package-lock=false", pin.spec]); + return pin; +} + +async function writeCliWrapper(config, paths) { + const wrapper = renderCliWrapper(await readFile(join(config.root, "deploy", "docker", "public-cli-wrapper.sh"), "utf8"), { + cliStateRoot: paths.cliState, + deployDockerDir: join(config.root, "deploy", "docker"), + memoryRoot: paths.memory + }); + const target = join(paths.bin, "spaceapp-cli-wrapper"); + await writeFile(target, wrapper, { mode: 0o700 }); + await chmod(target, 0o700); + // Only OpenCode for now (#65); other providers can be added through the same wrapper. + const link = join(paths.bin, "opencode-vscode-parity"); + if (!await lstat(link).catch(() => null)) await symlink("spaceapp-cli-wrapper", link); +} + +export function buildLibraries(root = repoRoot, run = runChecked) { + for (const name of LIBRARY_PACKAGES) run("npm", ["run", "build", "-w", name], { cwd: root }); +} + +/** + * The OpenCode panes share one detached `opencode serve` per provider store (deploy/docker/opencode-shared-server.mjs). + * In a container it ends with the container; here down/reset stop it, but only a pid that still is `opencode serve` on + * the recorded port. + */ +export async function stopSharedServers(paths, { run = spawnSync, kill = process.kill.bind(process) } = {}) { + const stopped = []; + const providers = await readdir(join(paths.cliState, "providers")).catch(() => []); + for (const provider of providers) { + const record = await readFile(join(paths.cliState, "providers", provider, "state", "opencode", "spaceapp-shared-server.json"), "utf8") + .then(JSON.parse).catch(() => null); + if (!Number.isSafeInteger(record?.pid) || record.pid <= 1 || !Number.isSafeInteger(record?.port)) continue; + const args = run("ps", ["-o", "args=", "-p", String(record.pid)], { encoding: "utf8" }).stdout?.trim() ?? ""; + if (!/(?:^|\/)opencode serve\b/.test(args) || !args.includes(`--port ${record.port}`)) continue; + try { kill(record.pid, "SIGTERM"); stopped.push(record.pid); } catch { /* already gone */ } + } + return stopped; +} + +async function up(config) { + if (process.platform === "win32") throw new Error("Run dev:stack inside WSL2 on Windows 11 (see CONTRIBUTING.md)."); + if (await readPid(statePaths(config))) throw new Error("The dev stack is already running. Stop it with: npm run dev:stack -- down"); + runChecked("docker", ["compose", "version"], { stdio: "ignore" }); + for (const [name, port] of Object.entries(config.ports)) { + if (!await portFree(port, config.host)) throw new Error(`Port ${port} (${name}) on ${config.host} is in use. Set ${PORT_ENV[name]} to another port.`); + } + const state = await ensureState(config); + const { paths } = state; + const images = readPinnedImages(await readFile(join(config.root, "packages", "run-spaceapp", "templates", "compose.yml"), "utf8")); + await writeFile(paths.compose, renderCompose({ images, ports: config.ports, host: config.host, secretsDir: paths.secrets, project: config.project }), { mode: 0o600 }); + process.stdout.write(`dev-stack: starting PostgreSQL and Temporal (compose project ${config.project})\n`); + runChecked("docker", composeArgs(config, paths, "up", "--detach", "--wait")); + process.stdout.write("dev-stack: building the workspace packages\n"); + buildLibraries(config.root); + runChecked("npm", ["run", "build", "-w", "@space/worker"], { cwd: config.root }); + const envs = serviceEnvironments(config, state); + process.stdout.write("dev-stack: running the migrations\n"); + runChecked(process.execPath, [join(config.root, "packages", "db", "dist", "migrate.js")], { cwd: config.root, env: envs.api }); + await installOpenCode(config, paths); + await writeCliWrapper(config, paths); + await writeFile(paths.pid, `${process.pid}\n`, { mode: 0o600 }); + const supervisor = superviseProcesses(processSpecs(config, envs)); + let rebuildSeen = (await stat(paths.rebuildStamp).catch(() => null))?.mtimeMs ?? 0; + watchFile(paths.rebuildStamp, { interval: 1000 }, (current) => { + if (current.mtimeMs > rebuildSeen) { + rebuildSeen = current.mtimeMs; + process.stdout.write("dev-stack: packages rebuilt, restarting cli-host, api and worker\n"); + supervisor.restart(["cli-host", "api", "worker"]); + } + }); + for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) process.once(signal, () => supervisor.stop("stopping")); + const cleanup = async () => { unwatchFile(paths.rebuildStamp); await stopSharedServers(paths); await rm(paths.pid, { force: true }); }; + let readyWatch; + try { + await waitForReady(`http://${config.host}:${config.ports.api}/readyz`, { isAlive: supervisor.isRunning }); + process.stdout.write([ + "", + "SpaceApp is running from source.", + ` Open: http://${config.host}:${config.ports.web}`, + ` Owner setup token: ${state.secrets["setup-token"]} (one-time: create the owner with it, then sign in)`, + " Stop: npm run dev:stack -- down Remove everything: npm run dev:stack -- reset", + "" + ].join("\n") + "\n"); + // Keep checking: a dependency that goes away later is reported instead of failing silently in the browser. + let lastReport = ""; + readyWatch = setInterval(async () => { + try { + const response = await fetch(`http://${config.host}:${config.ports.api}/readyz`, { signal: AbortSignal.timeout(5_000) }); + const body = await response.json().catch(() => ({})); + const report = response.ok && body.ok === true ? "" : summarizeReadiness(body); + if (report !== lastReport) process.stdout.write(report ? `dev-stack: /readyz is failing: ${report}\n` : "dev-stack: /readyz is ok again\n"); + lastReport = report; + } catch { /* the API restarting under tsx watch; the process supervisor reports real exits */ } + }, 15_000); + } catch (error) { + supervisor.stop(error.message, 1); + } + const code = await supervisor.done; + clearInterval(readyWatch); + await cleanup(); + process.exitCode = code; +} + +async function stopSupervisor(paths) { + const pid = await readPid(paths); + if (pid) { + process.kill(pid, "SIGTERM"); + for (let i = 0; i < 100 && await readPid(paths); i += 1) await new Promise((r) => setTimeout(r, 100)); + } + await stopSharedServers(paths); +} + +async function down(config) { + const paths = statePaths(config); + await stopSupervisor(paths); + if (existsSync(paths.compose)) runChecked("docker", composeArgs(config, paths, "down", "--remove-orphans")); + process.stdout.write("dev-stack: stopped. Data is kept; `npm run dev:stack -- reset` removes it.\n"); +} + +async function reset(config) { + const paths = statePaths(config); + if (existsSync(config.stateDir) && !existsSync(paths.marker)) { + throw new Error(`${config.stateDir} was not created by dev:stack; refusing to remove it.`); + } + await stopSupervisor(paths); + if (existsSync(paths.compose)) { + runChecked("docker", composeArgs(config, paths, "down", "--volumes", "--remove-orphans")); + } + if (paths.socketDir !== join(config.stateDir, "run")) await rm(paths.socketDir, { recursive: true, force: true }); + await rm(config.stateDir, { recursive: true, force: true }); + process.stdout.write(`dev-stack: removed the ${config.project} containers, volumes and ${config.stateDir}.\n`); +} + +async function rebuild(config) { + const paths = statePaths(config); + buildLibraries(config.root); + await writeFile(paths.rebuildStamp, `${Date.now()}\n`, { mode: 0o600 }); + process.stdout.write((await readPid(paths)) ? "dev-stack: rebuilt; the running stack restarts cli-host, api and worker.\n" : "dev-stack: rebuilt.\n"); +} + +async function status(config) { + const paths = statePaths(config); + const pid = await readPid(paths); + process.stdout.write(`dev-stack: ${pid ? `running (pid ${pid})` : "not running"}; state ${existsSync(config.stateDir) ? config.stateDir : "absent"}; web http://${config.host}:${config.ports.web}\n`); +} + +export async function main(argv = process.argv.slice(2), env = process.env) { + const command = argv[0] ?? "up"; + const config = resolveConfig(env); + const commands = { up, down, reset, rebuild, status }; + if (!commands[command]) throw new Error(`Unknown dev:stack command "${command}". Use up, down, reset, rebuild or status.`); + await commands[command](config); +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch((error) => { + process.stderr.write(`dev-stack: ${error.message}\n`); + process.exitCode = 1; + }); +} diff --git a/scripts/tests/dev-stack.test.mjs b/scripts/tests/dev-stack.test.mjs new file mode 100644 index 000000000..c0ad4662d --- /dev/null +++ b/scripts/tests/dev-stack.test.mjs @@ -0,0 +1,254 @@ +import assert from "node:assert/strict"; +import { spawn, spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { homedir, tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import test from "node:test"; +import { + DEFAULT_PORTS, DEFAULT_PROJECT, STATE_MARKER, baseEnv, ensureState, main, processSpecs, readOpenCodePackage, readPinnedImages, + renderCliWrapper, renderCompose, repoRoot, resolveConfig, serviceEnvironments, statePaths, stopSharedServers, summarizeReadiness, + superviseProcesses +} from "../dev-stack.mjs"; + +const root = resolve(import.meta.dirname, "../.."); +const read = (path) => readFile(join(root, path), "utf8"); +const mode = async (path) => (await stat(path)).mode & 0o777; + +async function scratch(t) { + const dir = await mkdtemp(join(tmpdir(), "dev-stack-test-")); + t.after(() => rm(dir, { recursive: true, force: true })); + return dir; +} + +test("the dev stack uses the launcher's digest-pinned postgres and temporal images", async () => { + const template = await read("packages/run-spaceapp/templates/compose.yml"); + const images = readPinnedImages(template); + for (const image of Object.values(images)) { + assert.match(image, /@sha256:[0-9a-f]{64}$/); + assert.ok(template.includes(`image: ${image}`)); + } + assert.match(images.postgres, /^pgvector\/pgvector:/); + assert.match(images.temporal, /^temporalio\/auto-setup:/); + assert.throws(() => readPinnedImages(template.replace(/(pgvector\/pgvector:[^@\s]+)@sha256:[0-9a-f]+/, "$1")), /digest-pinned image for postgres/); +}); + +test("OpenCode is pinned to the version the CLI image installs", async () => { + const pin = readOpenCodePackage(await read("Dockerfile")); + assert.match(pin.spec, /^(?:@opencode\/cli|opencode-ai)@\d/); + assert.equal(`${pin.name}@${pin.version}`, pin.spec); + assert.deepEqual(readOpenCodePackage("RUN npm install --global --no-fund opencode-ai@1.2.3 \\\n"), { spec: "opencode-ai@1.2.3", name: "opencode-ai", version: "1.2.3" }); + assert.throws(() => readOpenCodePackage("RUN npm install --global opencode-ai"), /does not pin/); +}); + +test("ports, project and state directory are isolated from an installed SpaceApp", () => { + const config = resolveConfig({}, root); + assert.deepEqual(config.ports, DEFAULT_PORTS); + assert.equal(config.project, DEFAULT_PROJECT); + assert.equal(config.stateDir, join(root, ".dev-stack")); + assert.equal(config.host, "127.0.0.1"); + for (const port of Object.values(config.ports)) assert.ok(![4910, 4911].includes(port)); + assert.equal(resolveConfig({ SPACE_DEV_STACK_WEB_PORT: "5173" }, root).ports.web, 5173); + assert.throws(() => resolveConfig({ SPACE_DEV_STACK_API_PORT: "4910" }, root), /installed SpaceApp/); + assert.throws(() => resolveConfig({ SPACE_DEV_STACK_WEB_PORT: "4911" }, root), /installed SpaceApp/); + assert.throws(() => resolveConfig({ SPACE_DEV_STACK_WEB_PORT: "4920" }, root), /all be different/); + assert.throws(() => resolveConfig({ SPACE_DEV_STACK_API_PORT: "80" }, root), /between 1024/); + assert.throws(() => resolveConfig({ SPACE_DEV_STACK_API_PORT: "49x" }, root), /port number/); + assert.throws(() => resolveConfig({ SPACE_DEV_STACK_PROJECT: "spaceapp" }, root), /installed SpaceApp/); + for (const dir of [root, resolve(root, ".."), "/", homedir()]) { + assert.throws(() => resolveConfig({ SPACE_DEV_STACK_DIR: dir }, root), /directory of its own/); + } +}); + +test(".dev-stack is git-ignored and npm run dev:stack runs the script", async () => { + assert.match(await read(".gitignore"), /^\/\.dev-stack\/$/m); + const pkg = JSON.parse(await read("package.json")); + assert.equal(pkg.scripts["dev:stack"], "node scripts/dev-stack.mjs"); + const ignored = spawnSync("git", ["check-ignore", "-q", ".dev-stack/secrets/setup-token"], { cwd: root }); + if (ignored.error?.code !== "ENOENT") assert.equal(ignored.status, 0); +}); + +test("the compose file binds every port to 127.0.0.1 and holds no secret value", async (t) => { + const dir = await scratch(t); + const config = resolveConfig({ SPACE_DEV_STACK_DIR: dir }, root); + const { paths, secrets } = await ensureState(config); + const images = readPinnedImages(await read("packages/run-spaceapp/templates/compose.yml")); + const compose = renderCompose({ images, ports: config.ports, host: config.host, secretsDir: paths.secrets, project: config.project }); + assert.match(compose, new RegExp(`^name: ${DEFAULT_PROJECT}$`, "m")); + const portLines = compose.match(/^ {6}- "[^"]*:\d+:\d+"$/gm); + assert.equal(portLines.length, 2); + for (const line of portLines) assert.match(line, /^ {6}- "127\.0\.0\.1:\d+:\d+"$/); + assert.ok(compose.includes(`"127.0.0.1:${config.ports.postgres}:5432"`)); + assert.ok(compose.includes(`"127.0.0.1:${config.ports.temporal}:7233"`)); + assert.ok(compose.includes(images.postgres) && compose.includes(images.temporal)); + for (const value of Object.values(secrets)) assert.ok(!compose.includes(value)); + assert.match(compose, /com\.oll4com\.spaceapp\.dev-stack: "spaceapp-dev-stack"/); +}); + +test("secrets are generated once per state directory with mode 0600", async (t) => { + const dir = join(await scratch(t), "state"); + const config = resolveConfig({ SPACE_DEV_STACK_DIR: dir }, root); + const first = await ensureState(config); + assert.equal(await mode(dir), 0o700); + assert.equal(await mode(first.paths.secrets), 0o700); + for (const name of ["postgres-password", "session-secret", "setup-token"]) { + assert.equal(await mode(join(first.paths.secrets, name)), 0o600); + assert.ok(first.secrets[name].length >= 32, name); + } + assert.equal(new Set(Object.values(first.secrets)).size, 3); + const second = await ensureState(config); + assert.deepEqual(second.secrets, first.secrets); + assert.equal((await readFile(join(dir, STATE_MARKER), "utf8")).trim(), DEFAULT_PROJECT); +}); + +test("a directory dev:stack did not create is never used or removed", async (t) => { + const dir = await scratch(t); + await writeFile(join(dir, "keep.txt"), "owner data\n"); + const config = resolveConfig({ SPACE_DEV_STACK_DIR: dir }, root); + await assert.rejects(ensureState(config), /not created by dev:stack/); + await assert.rejects(main(["reset"], { SPACE_DEV_STACK_DIR: dir }), /refusing to remove/); + assert.ok(existsSync(join(dir, "keep.txt"))); +}); + +test("reset removes only the state directory it created", async (t) => { + const parent = await scratch(t); + const dir = join(parent, "state"); + const sibling = join(parent, "sibling.txt"); + await writeFile(sibling, "stays\n"); + await ensureState(resolveConfig({ SPACE_DEV_STACK_DIR: dir }, root)); + const log = t.mock.method(process.stdout, "write", () => true); + await main(["reset"], { SPACE_DEV_STACK_DIR: dir }); + log.mock.restore(); + assert.equal(existsSync(dir), false); + assert.ok(existsSync(sibling)); + await assert.rejects(main(["bogus"], { SPACE_DEV_STACK_DIR: dir }), /Unknown dev:stack command/); +}); + +test("long checkouts put the CLI host socket in a short private directory", () => { + const short = statePaths(resolveConfig({ SPACE_DEV_STACK_DIR: "/tmp/ds" }, root)); + assert.equal(short.socket, "/tmp/ds/run/cli-host.sock"); + const long = statePaths(resolveConfig({ SPACE_DEV_STACK_DIR: `/tmp/${"x".repeat(120)}` }, root)); + assert.ok(Buffer.byteLength(long.socket) <= 100); + assert.match(long.socket, /spaceapp-dev-[0-9a-f]{12}\/cli-host\.sock$/); +}); + +test("the CLI wrapper runs from the state directory and this checkout, not the container paths", async (t) => { + const dir = await scratch(t); + const wrapper = renderCliWrapper(await read("deploy/docker/public-cli-wrapper.sh"), { + cliStateRoot: join(dir, "cli"), deployDockerDir: join(root, "deploy", "docker"), memoryRoot: join(dir, "it's memory") + }); + assert.match(wrapper, /^set -euo pipefail\nexport SPACE_MEMORY_ROOT='.*it'\\''s memory'$/m); + assert.doesNotMatch(wrapper, /\/var\/lib\/spaceapp-cli|\/app\/deploy/); + assert.ok(wrapper.includes(`state_root="${join(dir, "cli")}/providers/$runtime_name"`)); + assert.ok(wrapper.includes(`node ${join(root, "deploy", "docker")}/opencode-shared-server.mjs`)); + assert.ok(wrapper.includes(`export PATH="${join(dir, "cli")}/vendor/$runtime_name/node_modules/.bin:`)); + const file = join(dir, "wrapper.sh"); + await writeFile(file, wrapper); + const syntax = spawnSync("bash", ["-n", file]); + assert.equal(syntax.status, 0, syntax.stderr?.toString()); +}); + +test("process environments use owner setup, loopback addresses and none of the caller's SpaceApp settings", async (t) => { + const dir = await scratch(t); + const config = resolveConfig({ SPACE_DEV_STACK_DIR: dir }, root); + const state = await ensureState(config); + const envs = serviceEnvironments(config, state, { PATH: "/usr/bin", HOME: "/home/x", SPACE_DEV_LOGIN: "true", SPACEAPP_PORT: "4911", SPACE_DATABASE_URL: "postgres://installed" }); + assert.deepEqual(baseEnv({ SPACE_X: "1", SPACEAPP_Y: "2", SPACEX: "3", PATH: "/bin" }), { SPACEX: "3", PATH: "/bin" }); + assert.equal(envs.api.SPACE_DEV_LOGIN, "false"); + assert.equal(envs.api.SPACE_SESSION_SECRET, state.secrets["session-secret"]); + assert.equal(envs.api.SPACE_SETUP_TOKEN_FILE, join(state.paths.secrets, "setup-token")); + assert.equal(envs.api.SPACE_API_HOST, "127.0.0.1"); + assert.equal(envs.api.SPACE_API_PORT, String(config.ports.api)); + assert.equal(envs.api.SPACE_TEMPORAL_ADDRESS, `127.0.0.1:${config.ports.temporal}`); + assert.equal(envs.api.SPACE_DATABASE_URL, `postgres://spaceapp:${encodeURIComponent(state.secrets["postgres-password"])}@127.0.0.1:${config.ports.postgres}/spaceapp`); + assert.equal(envs.worker.SPACE_DATABASE_URL, envs.api.SPACE_DATABASE_URL); + assert.equal(envs.api.SPACE_CLI_COMMAND_PATH, state.paths.bin); + assert.equal(envs.api.SPACE_CLI_HOST_SOCKET, envs.cliHost.SPACE_CLI_HOST_SOCKET); + assert.equal(envs.web.SPACE_WEB_HOST, "127.0.0.1"); + assert.equal(envs.web.SPACE_WEB_API_ORIGIN, `http://127.0.0.1:${config.ports.api}`); + assert.ok(envs.cliHost.PATH.startsWith(`${state.paths.bin}:${join(state.paths.cliState, "vendor", "opencode", "node_modules", ".bin")}:`)); + for (const env of [envs.api, envs.worker, envs.web, envs.cliHost]) { + for (const [key, value] of Object.entries(env)) { + if (/^SPACE_/.test(key) && value.startsWith("/")) { + assert.ok(value.startsWith(dir) || value.startsWith(root) || value.startsWith(tmpdir()), `${key}=${value}`); + } + } + assert.notEqual(env.SPACEAPP_PORT, "4911"); + } + const specs = processSpecs(config, envs); + assert.deepEqual(specs.map((spec) => spec.name), ["cli-host", "api", "worker-build", "worker", "web"]); + const web = specs.find((spec) => spec.name === "web"); + assert.deepEqual(web.args, ["--host", "127.0.0.1", "--port", String(config.ports.web), "--strictPort"]); + assert.deepEqual(specs.find((spec) => spec.name === "api").args.slice(0, 1), ["watch"]); +}); + +test("down and reset stop the detached OpenCode server only when the recorded pid still is it", async (t) => { + const dir = await scratch(t); + const { paths } = await ensureState(resolveConfig({ SPACE_DEV_STACK_DIR: dir }, root)); + const record = join(paths.cliState, "providers", "opencode", "state", "opencode"); + await mkdir(record, { recursive: true }); + await writeFile(join(record, "spaceapp-shared-server.json"), JSON.stringify({ pid: 4242, port: 47047 })); + const killed = []; + const kill = (pid, signal) => killed.push([pid, signal]); + const ps = (args) => () => ({ stdout: `${args}\n` }); + assert.deepEqual(await stopSharedServers(paths, { run: ps("/x/node_modules/.bin/opencode serve --hostname 127.0.0.1 --port 47047"), kill }), [4242]); + assert.deepEqual(killed, [[4242, "SIGTERM"]]); + assert.deepEqual(await stopSharedServers(paths, { run: ps("vim notes.txt"), kill }), []); + assert.deepEqual(await stopSharedServers(paths, { run: ps("opencode serve --hostname 127.0.0.1 --port 5000"), kill }), []); + assert.equal(killed.length, 1); +}); + +test("a /readyz failure names the dependency that is down", () => { + assert.equal(summarizeReadiness({ ok: true, dependencies: { database: "RUNNING", sessionRevocations: "RUNNING", worker: "RUNNING", cliHost: "RUNNING", browserHost: "DISABLED" } }), "ok=true"); + assert.equal(summarizeReadiness({ ok: false, dependencies: { database: "RUNNING", sessionRevocations: "RUNNING", worker: "ERROR", cliHost: "UNAVAILABLE", browserHost: "DISABLED" } }), + "worker=ERROR (Temporal worker or Temporal), cliHost=UNAVAILABLE (CLI host)"); + assert.equal(summarizeReadiness("nope"), "no readiness answer"); +}); + +function nodeChild(code, delayMs) { + return { command: process.execPath, args: ["-e", `console.log("up"); setTimeout(() => process.exit(${code}), ${delayMs})`], cwd: root, env: { PATH: process.env.PATH } }; +} + +test("one process exiting stops the whole stack with a message naming it, instead of hanging", async () => { + const lines = []; + const supervisor = superviseProcesses([ + { name: "api", ...nodeChild(0, 60_000) }, + { name: "worker", ...nodeChild(3, 300) }, + { name: "web", ...nodeChild(0, 60_000) } + ], { spawn, write: (text) => lines.push(text) }); + const code = await supervisor.done; + assert.equal(code, 1); + assert.equal(supervisor.children.size, 0); + const output = lines.join(""); + assert.match(output, /dev-stack: worker exited unexpectedly \(3\)\. Stopping the dev stack/); + assert.match(output, /\[api\] up\n/); +}); + +test("a missing executable is a clear start failure", async () => { + const lines = []; + const supervisor = superviseProcesses([{ name: "cli-host", command: join(root, "does-not-exist"), args: [], cwd: root, env: {} }], { spawn, write: (text) => lines.push(text) }); + assert.equal(await supervisor.done, 1); + assert.match(lines.join(""), /cli-host failed to start \(ENOENT\)/); +}); + +test("rebuild restarts the named processes without stopping the stack, and stop ends everything", async () => { + const lines = []; + let starts = 0; + const supervisor = superviseProcesses([ + { name: "api", ...nodeChild(0, 60_000) }, + { name: "web", ...nodeChild(0, 60_000) } + ], { spawn, write: (text) => lines.push(text), onStart: (spec) => { if (spec.name === "api") starts += 1; } }); + const firstApi = supervisor.children.get("api"); + supervisor.restart(["api"]); + for (let i = 0; i < 100 && starts < 2; i += 1) await new Promise((r) => setTimeout(r, 20)); + assert.equal(starts, 2); + assert.notEqual(supervisor.children.get("api"), firstApi); + assert.ok(supervisor.isRunning()); + supervisor.stop("stopping"); + assert.equal(await supervisor.done, 0); + assert.doesNotMatch(lines.join(""), /exited unexpectedly/); +}); + +test("the dev-stack script is importable without starting anything", () => { + assert.equal(repoRoot, root); +});