diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b8ff06e..3391a7bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,6 +55,7 @@ jobs: libwebkit2gtk-4.1-dev \ libayatana-appindicator3-dev \ librsvg2-dev \ + expect \ patchelf \ sqlite3 - name: Report Linux FUSE coverage @@ -75,6 +76,8 @@ jobs: run: cargo test --workspace --all-targets - name: Run real Linux FUSE smoke test run: LOCALITY_FUSE_SMOKE=1 LOCALITY_FUSE_SMOKE_REQUIRED=1 tests/linux_fuse_smoke.sh + - name: Run Amika snapshot shell test + run: make test-init-amika-locality-snapshot - name: Install desktop dependencies run: npm ci working-directory: apps/desktop diff --git a/Makefile b/Makefile index 0fee28d2..10c26768 100644 --- a/Makefile +++ b/Makefile @@ -189,7 +189,7 @@ audit-oauth-service: $(OAUTH_SERVICE_NODE_MODULES_STAMP) ## Audit OAuth service $(OAUTH_SERVICE_NPM) audit .PHONY: test -test: test-rust ## Run the default test suite. +test: test-rust test-init-amika-locality-snapshot ## Run the default test suite. .PHONY: test-rust test-rust: ## Run all Rust workspace tests. @@ -239,6 +239,10 @@ test-launch-readiness-wrappers: ## Validate launch-readiness wrapper defaults. tests/launch_readiness_aws_wrapper.sh tests/init_amika_locality_snapshot.sh +.PHONY: test-init-amika-locality-snapshot +test-init-amika-locality-snapshot: ## Validate secure Amika snapshot initialization and interruption cleanup. + tests/init_amika_locality_snapshot.sh + .PHONY: fmt fmt: ## Format Rust code. $(CARGO) fmt --all diff --git a/docs/cli.md b/docs/cli.md index 69942d14..fb899f0b 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -73,14 +73,20 @@ Library callers cannot opt out accidentally: convenience sandbox, profile-key, session-credential, and generation-2 materialization APIs inspect the default state root, while `*_at_state_root` variants accept an explicit state root. -For a fresh remote Amika environment, the repository helper creates a sandbox, +For a remote Amika environment, the repository helper creates a fresh sandbox, checks out the current repository revision for scenario evidence, downloads the versioned Locality `v0.3.7` Linux package, verifies its pinned SHA-256, extracts the released `loc` binary without installing desktop dependencies, and streams -a one-time bootstrap token to `loc sandbox init` over standard input. After +a reusable Workspace Profile key from the Admin UI to `loc sandbox init` over +standard input. After materialization, the helper runs its inline Notion-only launch-gate scenario, prints the prompt, and prints `/home/amika/final_report.md`. +The local machine needs `amika`, `git`, Python 3.11 or newer, and `expect` on `PATH`. +Install Expect with `brew install expect` on macOS or `sudo apt-get install +expect` on Debian/Ubuntu. Expect provides the PTY transport used to stream the +credentials without putting them in arguments or logs. + The helper uses the existing Azure Codex setup. Export the Azure key locally; the script streams it directly to the remote Codex process without placing it in command arguments or writing it to a sandbox file: @@ -89,22 +95,41 @@ in command arguments or writing it to a sandbox file: export AZURE_OPENAI_API_KEY= ``` -Then read a reusable Workspace Profile key from the administration portal without -adding it to shell history -and run the helper from this repository: +In `https://api.dev.locality.dev/admin/access`, select a ready Workspace Profile +and create a key. The plaintext is shown once. Read it without adding it to +shell history and run the helper from this repository: ```bash -read -rs LOCALITY_BOOTSTRAP_TOKEN +read -rs LOCALITY_PROFILE_KEY printf '%s\n' "$LOCALITY_PROFILE_KEY" | \ scripts/init-amika-locality-snapshot.sh \ --api-url https://api.dev.locality.dev unset LOCALITY_PROFILE_KEY ``` -The key can launch multiple short-lived sandboxes until revoked. The helper creates a uniquely named sandbox, publishes -the workspace at `/home/amika/locality-snapshot`, uses only `/home/amika` paths -in the prompt, and leaves the sandbox running for inspection. Use `--name` when -a stable sandbox name is needed. +The key remains reusable until it expires or is revoked. The helper creates a +uniquely named sandbox, publishes the workspace at +`/home/amika/locality-snapshot`, uses only `/home/amika` paths in the prompt, +and leaves the sandbox running for inspection. It refuses `--reuse`: repository +cleanliness cannot make an existing sandbox a trusted boundary for a Workspace +Profile key or Azure credential. A name collision fails before either +credential is sent. The helper never deletes or replaces the colliding sandbox. +If replacement is intended, make that destructive action separately and +explicitly, then rerun the helper: + +```bash +amika sandbox delete --remote --force --delete-volumes saga-locality-snapshot +printf '%s\n' "$LOCALITY_PROFILE_KEY" | \ + scripts/init-amika-locality-snapshot.sh \ + --api-url https://api.dev.locality.dev \ + --name saga-locality-snapshot +``` + +Revoke the temporary Workspace Profile key in Admin after testing. +An existing generation-2 root is owned by the exact Profile key bytes used to +create it. Creating a new key for the same Workspace Profile does not authorize +refresh of that old root. After revocation, use a fresh sandbox and materialize +a new root with the new key. ## Provider Connections diff --git a/experiment/locality-mcp-comparison/setup-codex-azure.sh b/experiment/locality-mcp-comparison/setup-codex-azure.sh index 45fe1dff..29297ed1 100755 --- a/experiment/locality-mcp-comparison/setup-codex-azure.sh +++ b/experiment/locality-mcp-comparison/setup-codex-azure.sh @@ -8,26 +8,418 @@ CODEX_HOME="${CODEX_HOME:-$HOME/.codex}" mkdir -p "$CODEX_HOME" -cat > "$CODEX_HOME/config.toml" < value_start and text[value_end - 1] in " \t": + value_end -= 1 + return key, value_start, value_end, line_end(index) + + +assignments = [] +headers = [] +current_table = () +current_array_table = False +index = 0 +while index < len(text): + statement_start = index + while statement_start < len(text) and text[statement_start] in " \t": + statement_start += 1 + if statement_start == len(text): + break + if text[statement_start] in "\r\n": + index = line_end(statement_start) + continue + if text[statement_start] == "#": + index = line_end(statement_start) + continue + if text[statement_start] == "[": + if headers: + headers[-1]["section_end"] = statement_start + table_path, current_array_table, index = scan_header(statement_start) + current_table = table_path + headers.append({ + "path": table_path, + "array": current_array_table, + "start": statement_start, + "section_end": len(text), + }) + continue + key_path, value_start, value_end, index = scan_assignment(statement_start) + assignments.append({ + "path": current_table + key_path, + "context": current_table, + "array": current_array_table, + "value_start": value_start, + "value_end": value_end, + }) +if headers: + headers[-1]["section_end"] = len(text) + +first_header = headers[0]["start"] if headers else len(text) +azure_path = ("model_providers", "azure") +replacements = [] +insertions = {} + + +def replacement_for(target_path, value): + matches = [item for item in assignments if item["path"] == target_path] + if len(matches) > 1 or (matches and matches[0]["array"]): + raise SystemExit(f"refusing to modify ambiguous TOML setting {'.'.join(target_path)} in {path}") + if not matches: + return False + item = matches[0] + replacements.append((item["value_start"], item["value_end"], json.dumps(value, ensure_ascii=False))) + return True + + +def add_group(position, lines): + insertions.setdefault(position, []).append(lines) + + +missing_root = [] +for key, value in root_values.items(): + if not replacement_for((key,), value): + if key in parsed_existing: + raise SystemExit(f"refusing to modify non-source TOML setting {key} in {path}") + missing_root.append(f"{key} = {json.dumps(value, ensure_ascii=False)}") +if missing_root: + add_group(first_header, missing_root) + +model_providers = parsed_existing.get("model_providers", {}) +if not isinstance(model_providers, dict): + raise SystemExit(f"refusing to replace non-table model_providers in {path}") +provider = model_providers.get("azure") +if provider is not None and not isinstance(provider, dict): + raise SystemExit(f"refusing to replace non-table Azure provider in {path}") + +missing_provider = [] +for key, value in provider_values.items(): + if not replacement_for(azure_path + (key,), value): + if isinstance(provider, dict) and key in provider: + raise SystemExit(f"refusing to modify inline Azure provider setting {key} in {path}") + missing_provider.append(key) + +if missing_provider: + explicit_provider_tables = [ + header for header in headers + if header["path"] == azure_path and not header["array"] + ] + if len(explicit_provider_tables) > 1: + raise SystemExit(f"refusing to modify duplicate Azure provider tables in {path}") + if explicit_provider_tables: + header = explicit_provider_tables[0] + add_group(header["section_end"], [ + f"{key} = {json.dumps(provider_values[key], ensure_ascii=False)}" + for key in missing_provider + ]) + elif provider is None: + add_group(len(text), [ + "[model_providers.azure]", + *( + f"{key} = {json.dumps(provider_values[key], ensure_ascii=False)}" + for key in missing_provider + ), + ]) + else: + provider_children = [ + item for item in assignments + if not item["array"] + and len(item["path"]) > len(azure_path) + and item["path"][:len(azure_path)] == azure_path + and azure_path[:len(item["context"])] == item["context"] + ] + if not provider_children: + nested_provider_tables = [ + header for header in headers + if not header["array"] + and len(header["path"]) > len(azure_path) + and header["path"][:len(azure_path)] == azure_path + ] + if not nested_provider_tables: + raise SystemExit(f"refusing to extend inline Azure provider table in {path}") + first_nested_table = min(nested_provider_tables, key=lambda header: header["start"]) + add_group(first_nested_table["start"], [ + "[model_providers.azure]", + *( + f"{key} = {json.dumps(provider_values[key], ensure_ascii=False)}" + for key in missing_provider + ), + ]) + else: + context = max((item["context"] for item in provider_children), key=len) + if context: + context_headers = [ + header for header in headers + if header["path"] == context and not header["array"] + ] + if len(context_headers) != 1: + raise SystemExit(f"refusing to extend ambiguous Azure provider context in {path}") + insertion_position = context_headers[0]["section_end"] + else: + insertion_position = first_header + relative_provider = azure_path[len(context):] + add_group(insertion_position, [ + f"{'.'.join((*relative_provider, key))} = {json.dumps(provider_values[key], ensure_ascii=False)}" + for key in missing_provider + ]) + +newline = "\r\n" if "\r\n" in text else "\n" +for position, groups in insertions.items(): + payload = (newline * 2).join(newline.join(group) for group in groups) + if position > 0 and text[position - 1] not in "\r\n": + payload = newline + payload + if position < len(text) or text.endswith(("\n", "\r")): + payload += newline + replacements.append((position, position, payload)) + +ordered = sorted(replacements, key=lambda item: (item[0], item[1])) +for previous, current in zip(ordered, ordered[1:]): + if previous[1] > current[0]: + raise SystemExit(f"refusing to apply overlapping TOML edits to {path}") +rendered = text +for start, end, replacement in reversed(ordered): + rendered = rendered[:start] + replacement + rendered[end:] + +try: + parsed = tomllib.loads(rendered) +except tomllib.TOMLDecodeError as error: + raise SystemExit(f"refusing to write invalid merged Codex config {path}: {error}") + +provider = parsed.get("model_providers", {}).get("azure", {}) +expected_root = root_values +expected_provider = provider_values +if any(parsed.get(key) != value for key, value in expected_root.items()): + raise SystemExit(f"merged Codex root settings failed validation for {path}") +if any(provider.get(key) != value for key, value in expected_provider.items()): + raise SystemExit(f"merged Azure provider settings failed validation for {path}") + +expected = copy.deepcopy(parsed_existing) +expected.update(root_values) +expected.setdefault("model_providers", {}).setdefault("azure", {}).update(provider_values) +if parsed != expected: + raise SystemExit(f"refusing to write Codex config after unrelated TOML values changed in {path}") + +directory = os.path.dirname(path) +os.makedirs(directory, mode=0o700, exist_ok=True) +fd, temporary = tempfile.mkstemp(prefix=".config.toml.", dir=directory) +try: + with os.fdopen(fd, "w", encoding="utf-8", newline="") as destination: + destination.write(rendered) + os.chmod(temporary, 0o600) + os.replace(temporary, path) +finally: + try: + os.unlink(temporary) + except FileNotFoundError: + pass +PY +} + +merge_codex_config "$CODEX_HOME/config.toml" chmod 700 "$CODEX_HOME" chmod 600 "$CODEX_HOME/config.toml" if [ -n "${AMIKA_AGENT_CWD:-}" ] && [ -d "$AMIKA_AGENT_CWD" ]; then mkdir -p "$AMIKA_AGENT_CWD/.codex" - cat > "$AMIKA_AGENT_CWD/.codex/config.toml" <<'TOML' -sandbox_mode = "workspace-write" -TOML + merge_codex_config "$AMIKA_AGENT_CWD/.codex/config.toml" "workspace-write" fi codex --version || true diff --git a/scripts/init-amika-locality-snapshot.sh b/scripts/init-amika-locality-snapshot.sh index 25994c0e..90d3d574 100755 --- a/scripts/init-amika-locality-snapshot.sh +++ b/scripts/init-amika-locality-snapshot.sh @@ -1,19 +1,27 @@ #!/usr/bin/env bash set -euo pipefail +set +a +AZURE_OPENAI_API_KEY_CAPTURE="${AZURE_OPENAI_API_KEY:-}" +unset PROFILE_KEY LOCALITY_PROFILE_KEY AMIKA_SECRET_LINE \ + AZURE_OPENAI_API_KEY AZURE_OPENAI_API_KEY_VALUE +AZURE_OPENAI_API_KEY_VALUE="$AZURE_OPENAI_API_KEY_CAPTURE" +unset AZURE_OPENAI_API_KEY_CAPTURE +export -n AZURE_OPENAI_API_KEY_VALUE usage() { cat <<'EOF' Usage: init-amika-locality-snapshot.sh --api-url [options] -Creates a fresh remote Amika sandbox, installs the verified Locality v0.3.7 CLI, -and materializes a scoped workspace snapshot. It then runs one inline Notion-only -scenario and prints both the prompt and generated report. The reusable Workspace -Profile key is read from standard input and is never passed in a command-line -argument. +Creates a fresh remote Amika sandbox, installs the verified Locality v0.3.7 +CLI, and materializes a scoped workspace snapshot. It then runs one inline +Notion-only scenario and prints both the prompt and generated report. The +Workspace Profile key created in Admin is read from standard input and is never +passed in a command-line argument. Options: --api-url Locality backend API origin (required). --name Amika sandbox name. Default: locality-snapshot-. + --reuse Refused: credential-bearing sandboxes must be fresh. --model Model passed to codex exec. Default: CODEX_MODEL or gpt-5.6-sol. --reasoning Reasoning effort passed to codex exec. @@ -32,12 +40,239 @@ fail() { exit 2 } +ACTIVE_CHILD_PID="" +PENDING_CHILD_SIGNAL="" +TERMINAL_STATE="" +TERMINAL_STATE_ACTIVE=false +ACTIVE_DELIVERY_DIR="" + +restore_terminal_state() { + local saved_state="${TERMINAL_STATE:-}" + + if [ "${TERMINAL_STATE_ACTIVE:-false}" != true ]; then + return 0 + fi + TERMINAL_STATE_ACTIVE=false + TERMINAL_STATE="" + stty "$saved_state" < /dev/tty 2>/dev/null || stty echo < /dev/tty 2>/dev/null || true +} + +read_profile_key_from_terminal() { + local chunk + local status + + PROFILE_KEY="" + while true; do + chunk="" + if IFS= read -r -t 1 chunk; then + PROFILE_KEY+="$chunk" + return 0 + else + status=$? + fi + PROFILE_KEY+="$chunk" + if [ "$status" -le 128 ]; then + return "$status" + fi + done +} + +cleanup_delivery_marker() { + local delivery_dir="${ACTIVE_DELIVERY_DIR:-}" + + if [ -z "$delivery_dir" ]; then + return + fi + ACTIVE_DELIVERY_DIR="" + rm -f -- "$delivery_dir/delivered" 2>/dev/null || true + rmdir "$delivery_dir" 2>/dev/null || true +} + +forward_and_reap_active_child() { + local signal="$1" + local child_pid="${ACTIVE_CHILD_PID:-}" + + if [ -z "$child_pid" ]; then + return + fi + ACTIVE_CHILD_PID="" + kill -s "$signal" "$child_pid" 2>/dev/null || true + if wait_for_child_exit "$child_pid" 200; then + return + fi + if [ "$signal" != TERM ]; then + kill -TERM "$child_pid" 2>/dev/null || true + if wait_for_child_exit "$child_pid" 100; then + return + fi + fi + kill -KILL "$child_pid" 2>/dev/null || true + if ! wait_for_child_exit "$child_pid" 200; then + printf 'init Amika Locality snapshot: could not reap child %s after SIGKILL\n' "$child_pid" >&2 + fi +} + +wait_for_child_exit() { + local child_pid="$1" + local attempts="$2" + local state + + while [ "$attempts" -gt 0 ]; do + if ! kill -0 "$child_pid" 2>/dev/null; then + wait "$child_pid" 2>/dev/null || true + return 0 + fi + state="$(ps -o stat= -p "$child_pid" 2>/dev/null || true)" + case "$state" in + ''|*Z*) + wait "$child_pid" 2>/dev/null || true + return 0 + ;; + esac + attempts=$((attempts - 1)) + sleep 0.01 + done + return 1 +} + +cleanup_on_exit() { + local status="$1" + + trap - EXIT + trap '' HUP INT TERM + restore_terminal_state + forward_and_reap_active_child TERM + cleanup_delivery_marker + exit "$status" +} + +terminate_for_signal() { + local signal="$1" + local signal_number="$2" + + trap '' HUP INT TERM + PROFILE_KEY="" + AZURE_OPENAI_API_KEY_VALUE="" + restore_terminal_state + forward_and_reap_active_child "$signal" + cleanup_delivery_marker + exit $((128 + signal_number)) +} + +trap 'cleanup_on_exit $?' EXIT +trap 'terminate_for_signal HUP 1' HUP +trap 'terminate_for_signal INT 2' INT +trap 'terminate_for_signal TERM 15' TERM + +prepare_child_launch() { + PENDING_CHILD_SIGNAL="" + trap 'PENDING_CHILD_SIGNAL="HUP 1"' HUP + trap 'PENDING_CHILD_SIGNAL="INT 2"' INT + trap 'PENDING_CHILD_SIGNAL="TERM 15"' TERM +} + +activate_child() { + ACTIVE_CHILD_PID="$1" + trap 'terminate_for_signal HUP 1' HUP + trap 'terminate_for_signal INT 2' INT + trap 'terminate_for_signal TERM 15' TERM + case "$PENDING_CHILD_SIGNAL" in + "HUP 1") terminate_for_signal HUP 1 ;; + "INT 2") terminate_for_signal INT 2 ;; + "TERM 15") terminate_for_signal TERM 15 ;; + esac +} + +wait_for_active_child() { + local child_pid="$1" + local status + + if wait "$child_pid"; then + status=0 + else + status=$? + fi + if [ "${ACTIVE_CHILD_PID:-}" = "$child_pid" ]; then + ACTIVE_CHILD_PID="" + fi + return "$status" +} + encode_remote_argv() { local payload payload="$(printf '%s\0' "$@" | base64 | tr -d '\n')" printf 'python3 -c '\''import base64, os, sys; argv = [os.fsdecode(item) for item in base64.b64decode(sys.argv[1]).split(b"\\0")[:-1]]; os.execvp(argv[0], argv)'\'' %s' "$payload" } +EXPECT_TRANSPORT_PROCS=' + proc child_status {result} { + if {[llength $result] >= 6 && [lindex $result 4] eq "CHILDKILLED"} { + array set signal_number { + SIGHUP 1 SIGINT 2 SIGQUIT 3 SIGKILL 9 SIGPIPE 13 SIGTERM 15 + } + set signal [lindex $result 5] + if {[info exists signal_number($signal)]} { + return [expr {128 + $signal_number($signal)}] + } + return 125 + } + if {[lindex $result 2] != 0} { + return 125 + } + return [lindex $result 3] + } + + proc restore_terminal {} { + global terminal_state + if {$terminal_state ne ""} { + catch {exec stty $terminal_state < /dev/tty} + } else { + catch {exec stty sane < /dev/tty} + } + } + + proc reap_child {{signal ""}} { + global child_active spawn_id + if {!$child_active} { + return + } + set child_pid "" + catch {set child_pid [exp_pid -i $spawn_id]} + if {$signal ne "" && [string is integer -strict $child_pid]} { + catch {exec kill -$signal -- -$child_pid} + after 1000 + catch {exec kill -KILL -- -$child_pid} + } + catch {close -i $spawn_id} + catch {wait -i $spawn_id} + set child_active 0 + } + + proc terminate_for_signal {signal number} { + catch {set ::secret ""} + reap_child $signal + restore_terminal + exit [expr {128 + $number}] + } + + proc initialize_transport {} { + global child_active terminal_state + set child_active 0 + set terminal_state "" + catch {set terminal_state [exec stty -g < /dev/tty]} + trap {terminate_for_signal SIGHUP 1} SIGHUP + trap {terminate_for_signal SIGINT 2} SIGINT + trap {terminate_for_signal SIGTERM 15} SIGTERM + } + + proc wait_for_child {} { + global child_active spawn_id + set result [wait -i $spawn_id] + set child_active 0 + return $result + } +' + amika_ssh() { local sandbox="$1" local remote_command @@ -45,43 +280,162 @@ amika_ssh() { if [ "${1:-}" = "--" ]; then shift fi - if [ -t 2 ] && [ -r /dev/tty ]; then - remote_command="$(encode_remote_argv "$@")" - amika sandbox ssh -t "$sandbox" -- "$remote_command" < /dev/tty - else - amika sandbox ssh "$sandbox" "$@" - fi + remote_command="$(encode_remote_argv "$@")" + command -v expect >/dev/null 2>&1 || fail "expect is required for Amika PTY transport" + prepare_child_launch + ( + trap - HUP INT TERM + AMIKA_EXPECT_COMMON="$EXPECT_TRANSPORT_PROCS" \ + AMIKA_SANDBOX_NAME="$sandbox" AMIKA_REMOTE_COMMAND="$remote_command" \ + exec expect -c ' + eval $env(AMIKA_EXPECT_COMMON) + initialize_transport + set timeout 1800 + spawn -noecho amika sandbox ssh -t $env(AMIKA_SANDBOX_NAME) -- $env(AMIKA_REMOTE_COMMAND) + set child_active 1 + expect { + eof { + set result [wait_for_child] + restore_terminal + exit [child_status $result] + } + timeout { + reap_child SIGTERM + restore_terminal + puts stderr "Amika operation did not finish within 30 minutes" + exit 124 + } + } + ' + ) & + activate_child "$!" + wait_for_active_child "$ACTIVE_CHILD_PID" } amika_ssh_secret_line() { local sandbox="$1" local secret="$2" + local attempt=1 + local delivered + local delivery_marker + local status local remote_command shift 2 if [ "${1:-}" = "--" ]; then shift fi - if [ -t 2 ] && [ -r /dev/tty ]; then - command -v expect >/dev/null 2>&1 || fail "expect is required for Amika TTY credential transfer" - remote_command="$(encode_remote_argv "$@")" - AMIKA_SECRET_LINE="$secret" AMIKA_SANDBOX_NAME="$sandbox" \ - AMIKA_REMOTE_COMMAND="$remote_command" \ - expect -f /dev/stdin <<'EXPECT' -set timeout -1 -spawn -noecho amika sandbox ssh -t $env(AMIKA_SANDBOX_NAME) -- $env(AMIKA_REMOTE_COMMAND) -expect "__LOCALITY_STDIN_READY__" -send -- "$env(AMIKA_SECRET_LINE)\n" -expect eof -set result [wait] -exit [lindex $result 3] -EXPECT - else - printf '%s\n' "$secret" | amika sandbox ssh "$sandbox" "$@" - fi + command -v expect >/dev/null 2>&1 || fail "expect is required for Amika credential transfer" + remote_command="$(encode_remote_argv "$@")" + ACTIVE_DELIVERY_DIR="$(mktemp -d "${TMPDIR:-/tmp}/amika-secret-delivery.XXXXXX")" || \ + fail "could not create credential delivery state directory" + delivery_marker="$ACTIVE_DELIVERY_DIR/delivered" + + while [ "$attempt" -le 3 ]; do + prepare_child_launch + ( + trap - HUP INT TERM + AMIKA_EXPECT_COMMON="$EXPECT_TRANSPORT_PROCS" \ + AMIKA_SANDBOX_NAME="$sandbox" AMIKA_REMOTE_COMMAND="$remote_command" \ + AMIKA_DELIVERY_MARKER="$delivery_marker" \ + exec expect -c ' + eval $env(AMIKA_EXPECT_COMMON) + initialize_transport + set timeout 30 + if {[gets stdin secret] < 0} { + restore_terminal + puts stderr "credential input closed before a secret was read" + exit 65 + } + spawn -noecho amika sandbox ssh -t $env(AMIKA_SANDBOX_NAME) -- $env(AMIKA_REMOTE_COMMAND) + set child_active 1 + expect { + "__LOCALITY_STDIN_READY__" {} + eof { + catch {wait_for_child} + restore_terminal + puts stderr "Amika SSH closed before requesting credential input" + exit 75 + } + timeout { + reap_child SIGTERM + restore_terminal + puts stderr "Amika SSH did not request credential input within 30 seconds" + exit 75 + } + } + send -- "$secret\n" + if {[catch { + set marker [open $env(AMIKA_DELIVERY_MARKER) {WRONLY CREAT EXCL}] + close $marker + }]} { + set secret "" + reap_child SIGTERM + restore_terminal + puts stderr "could not record credential delivery; it was not retried" + exit 125 + } + set secret "" + set timeout 1800 + expect { + eof { + set result [wait_for_child] + restore_terminal + exit [child_status $result] + } + timeout { + set secret "" + reap_child SIGTERM + restore_terminal + puts stderr "Amika credential operation did not finish within 30 minutes; it was not retried" + exit 124 + } + } + ' + ) <<<"$secret" & + activate_child "$!" + if wait_for_active_child "$ACTIVE_CHILD_PID"; then + secret="" + cleanup_delivery_marker + return 0 + else + status=$? + fi + + delivered=false + if [ -f "$delivery_marker" ]; then + delivered=true + fi + if [ "$status" -ne 75 ] || [ "$delivered" = true ] || [ "$attempt" -eq 3 ]; then + secret="" + cleanup_delivery_marker + return "$status" + fi + printf 'Amika credential transport closed before secret delivery; retrying (%s/3)...\n' "$attempt" >&2 + attempt=$((attempt + 1)) + sleep 1 + done +} + +create_amika_sandbox() { + local sandbox="$1" + + prepare_child_launch + ( + trap - HUP INT TERM + cd "$REPO_ROOT" + exec amika sandbox create \ + --remote \ + --name "$sandbox" \ + --no-git \ + --yes + ) >/dev/null & + activate_child "$!" + wait_for_active_child "$ACTIVE_CHILD_PID" } API_URL="" SANDBOX_NAME="locality-snapshot-$(date -u +%Y%m%d-%H%M%S)" +REUSE_SANDBOX=false REMOTE_ROOT="/home/amika/locality-snapshot" LOC_RELEASE_VERSION="0.3.7" LOC_RELEASE_DEB_SHA256="692b05460839ba44b85cd1e6b3b6969ad4a3f62f3e81f420c4651159ad7ef195" @@ -103,6 +457,10 @@ while [ "$#" -gt 0 ]; do SANDBOX_NAME="$2" shift 2 ;; + --reuse) + REUSE_SANDBOX=true + shift + ;; --model) [ "$#" -ge 2 ] || fail "--model requires a value" CODEX_MODEL="$2" @@ -131,8 +489,10 @@ esac case "$SANDBOX_NAME" in *[!a-zA-Z0-9._-]*|'') fail "--name contains unsupported characters" ;; esac +[ "$REUSE_SANDBOX" != true ] || \ + fail "--reuse is refused because an existing sandbox is not a trusted credential boundary; choose a fresh --name" [ -n "$CODEX_MODEL" ] || fail "--model must not be empty" -[ -n "${AZURE_OPENAI_API_KEY:-}" ] || fail "AZURE_OPENAI_API_KEY is required" +[ -n "$AZURE_OPENAI_API_KEY_VALUE" ] || fail "AZURE_OPENAI_API_KEY is required" case "$AZURE_OPENAI_BASE_URL" in https://*) ;; *) fail "AZURE_OPENAI_BASE_URL must use https" ;; @@ -186,39 +546,33 @@ command -v git >/dev/null 2>&1 || fail "git is not available on PATH" SOURCE_REVISION="$(git -C "$REPO_ROOT" rev-parse HEAD)" || fail "could not resolve source revision" if [ -t 0 ]; then - stty -echo - IFS= read -r PROFILE_KEY || { - stty echo + TERMINAL_STATE="$(stty -g < /dev/tty)" || fail "could not read terminal state" + TERMINAL_STATE_ACTIVE=true + stty -echo < /dev/tty || fail "could not disable terminal echo" + read_profile_key_from_terminal || { + restore_terminal_state fail "read the Workspace Profile key from standard input" } - stty echo + restore_terminal_state printf '\n' else IFS= read -r PROFILE_KEY || fail "read the Workspace Profile key from standard input" fi -[ -n "$PROFILE_KEY" ] || fail "Workspace Profile key must not be empty" +export -n PROFILE_KEY +[ "${#PROFILE_KEY}" -eq 64 ] || fail "Workspace Profile key must be 64 lowercase hexadecimal characters" +case "$PROFILE_KEY" in + *[!0-9a-f]*) fail "Workspace Profile key must be 64 lowercase hexadecimal characters" ;; +esac -printf 'Creating Amika sandbox %s...\n' "$SANDBOX_NAME" -(cd "$REPO_ROOT" && amika sandbox create \ - --remote \ - --name "$SANDBOX_NAME" \ - --yes >/dev/null) +printf 'Creating fresh Amika sandbox %s (existing sandboxes are never reused or replaced)...\n' "$SANDBOX_NAME" +create_amika_sandbox "$SANDBOX_NAME" || \ + fail "could not create fresh sandbox ${SANDBOX_NAME}; no credentials were transferred (choose a new name or explicitly delete the old sandbox)" printf 'Installing released loc CLI v%s in %s...\n' "$LOC_RELEASE_VERSION" "$SANDBOX_NAME" amika_ssh "$SANDBOX_NAME" -- sh -c ' set -eu - revision=$1 - loc_version=$2 - expected_sha256=$3 - manifest=$(find "$HOME/workspace" -mindepth 2 -maxdepth 2 -type f -name Cargo.toml -print -quit) - test -n "$manifest" - repo_dir=${manifest%/Cargo.toml} - git -C "$repo_dir" cat-file -e "$revision^{commit}" - git -C "$repo_dir" checkout --detach "$revision" - if [ "$repo_dir" != /home/amika/workspace/locality ]; then - test ! -e /home/amika/workspace/locality - ln -s "$repo_dir" /home/amika/workspace/locality - fi + loc_version=$1 + expected_sha256=$2 work_dir=$(mktemp -d) trap '\''rm -rf "$work_dir"'\'' EXIT package="$work_dir/Locality_Linux_v${loc_version}.deb" @@ -231,7 +585,7 @@ amika_ssh "$SANDBOX_NAME" -- sh -c ' mkdir -p "$HOME/.local/bin" install -m 0755 "$work_dir/package/usr/bin/loc" "$HOME/.local/bin/loc" "$HOME/.local/bin/loc" sandbox init --help >/dev/null -' sh "$SOURCE_REVISION" "$LOC_RELEASE_VERSION" "$LOC_RELEASE_DEB_SHA256" +' sh "$LOC_RELEASE_VERSION" "$LOC_RELEASE_DEB_SHA256" printf 'Materializing scoped workspace at %s:%s...\n' "$SANDBOX_NAME" "$REMOTE_ROOT" amika_ssh_secret_line "$SANDBOX_NAME" "$PROFILE_KEY" -- sh -c ' @@ -249,11 +603,36 @@ amika_ssh_secret_line "$SANDBOX_NAME" "$PROFILE_KEY" -- sh -c ' --api-url "$API_URL" \ --root "$REMOTE_ROOT" \ --profile-key-stdin \ + --profile \ --json unset PROFILE_KEY printf 'Snapshot ready in Amika sandbox %s at %s\n' "$SANDBOX_NAME" "$REMOTE_ROOT" +printf 'Preparing clean Locality evidence checkout at revision %s...\n' "$SOURCE_REVISION" +amika_ssh "$SANDBOX_NAME" -- sh -c ' + set -eu + revision=$1 + repo_dir=/home/amika/workspace/locality + mkdir -p /home/amika/workspace + if [ ! -e "$repo_dir" ]; then + git clone https://github.com/codeflash-ai/locality.git "$repo_dir" + fi + test "$(git -C "$repo_dir" rev-parse --is-inside-work-tree)" = true + origin_url=$(git -C "$repo_dir" remote get-url origin) + case "$origin_url" in + https://github.com/codeflash-ai/locality|https://github.com/codeflash-ai/locality.git|git@github.com:codeflash-ai/locality.git) ;; + *) printf "unexpected Locality repository origin: %s\n" "$origin_url" >&2; exit 65 ;; + esac + test -z "$(git -C "$repo_dir" status --porcelain --untracked-files=all)" || { + printf "Locality evidence checkout is dirty; use a clean sandbox or remove the changes explicitly\n" >&2 + exit 65 + } + git -C "$repo_dir" fetch origin --prune + git -C "$repo_dir" cat-file -e "$revision^{commit}" + git -C "$repo_dir" checkout --detach "$revision" +' sh "$SOURCE_REVISION" + amika_ssh "$SANDBOX_NAME" -- sh -c ' set -eu azure_base_url=$1 @@ -262,7 +641,12 @@ amika_ssh "$SANDBOX_NAME" -- sh -c ' AZURE_OPENAI_BASE_URL="$azure_base_url" \ CODEX_MODEL="$model" \ CODEX_REASONING_EFFORT="$reasoning" \ + AMIKA_AGENT_CWD="$HOME" \ bash /home/amika/workspace/locality/experiment/locality-mcp-comparison/setup-codex-azure.sh + test -z "$(git -C /home/amika/workspace/locality status --porcelain --untracked-files=all)" || { + printf "Codex setup dirtied the Locality evidence checkout\n" >&2 + exit 65 + } ' sh "$AZURE_OPENAI_BASE_URL" "$CODEX_MODEL" "$CODEX_REASONING_EFFORT" PROMPT_BASE64="$(printf '%s\n' "$EFFECTIVE_PROMPT" | base64 | tr -d '\n')" @@ -274,7 +658,7 @@ unset PROMPT_BASE64 printf '\n===== Inline scenario prompt =====\n%s\n' "$EFFECTIVE_PROMPT" printf '\n===== Running scenario in %s =====\n' "$SANDBOX_NAME" -amika_ssh_secret_line "$SANDBOX_NAME" "$AZURE_OPENAI_API_KEY" -- sh -c ' +amika_ssh_secret_line "$SANDBOX_NAME" "$AZURE_OPENAI_API_KEY_VALUE" -- sh -c ' set -eu model=$1 reasoning=$2 @@ -309,6 +693,15 @@ amika_ssh_secret_line "$SANDBOX_NAME" "$AZURE_OPENAI_API_KEY" -- sh -c ' exit 1 } ' sh "$CODEX_MODEL" "$CODEX_REASONING_EFFORT" "$REMOTE_ROOT" +AZURE_OPENAI_API_KEY_VALUE="" +unset AZURE_OPENAI_API_KEY_VALUE + +amika_ssh "$SANDBOX_NAME" -- sh -c ' + test -z "$(git -C /home/amika/workspace/locality status --porcelain --untracked-files=all)" || { + printf "scenario modified the Locality evidence checkout\n" >&2 + exit 65 + } +' sh printf '\n===== /home/amika/final_report.md =====\n' amika_ssh "$SANDBOX_NAME" -- cat /home/amika/final_report.md diff --git a/tests/init_amika_locality_snapshot.sh b/tests/init_amika_locality_snapshot.sh index 9eedd4e0..c7086d62 100755 --- a/tests/init_amika_locality_snapshot.sh +++ b/tests/init_amika_locality_snapshot.sh @@ -3,6 +3,7 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" SCRIPT="${ROOT}/scripts/init-amika-locality-snapshot.sh" +AZURE_SETUP_SCRIPT="${ROOT}/experiment/locality-mcp-comparison/setup-codex-azure.sh" fail() { printf 'init Amika Locality snapshot test: %s\n' "$*" >&2 @@ -15,6 +16,14 @@ assert_contains() { grep -F -q -- "$needle" "$path" || fail "missing ${needle} in ${path}" } +assert_not_contains() { + local path="$1" + local needle="$2" + if grep -F -q -- "$needle" "$path"; then + fail "unexpected ${needle} in ${path}" + fi +} + tmp_root="$(mktemp -d "${TMPDIR:-/tmp}/loc-init-amika-snapshot-test.XXXXXX")" trap 'rm -rf "$tmp_root"' EXIT fake_bin="${tmp_root}/bin" @@ -25,10 +34,188 @@ prompt_input="${tmp_root}/scenario-prompt.input" fake_report="${tmp_root}/final_report.md" mkdir -p "$fake_bin" +cat > "${fake_bin}/codex" <<'SH' +#!/usr/bin/env bash +exit 0 +SH +chmod +x "${fake_bin}/codex" + +setup_agent_root="${tmp_root}/setup-agent" +setup_config="${setup_agent_root}/.codex/config.toml" +mkdir -p "$(dirname "$setup_config")" +setup_config_text="$(cat <<'TOML' +# Existing Codex settings must survive Azure setup. +"approval_policy" = "never" # keep quoted root key +"model" = "old-model" # keep target comment +"literal.key" = 'preserved literal value' + +["model_providers"."azure"] # keep quoted provider table +"name" = "Old Azure name" # keep provider comment +"base_url" = "https://old.invalid/openai/v1" +"env_key" = "OLD_AZURE_KEY" +"wire_api" = "chat" +custom_setting = "preserved-provider-value" + +[features] +web_search_request = true +multiline_basic = """ +model = "not a real setting" +[model_providers.azure] +# this comment belongs to the string +""" +multiline_literal = ''' +wire_api = "also not a real setting" +''' +preserved_array = [ + "first", + "second", # keep array comment +] + +["quoted.table"] +"quoted.key" = "preserved without a trailing newline" +TOML +)" +printf '%s' "$setup_config_text" > "$setup_config" +unset setup_config_text + +PATH="${fake_bin}:$PATH" \ + CODEX_HOME="${setup_agent_root}/.codex" \ + AMIKA_AGENT_CWD="$setup_agent_root" \ + CODEX_MODEL="merged-model" \ + CODEX_REASONING_EFFORT="high" \ + AZURE_OPENAI_BASE_URL="https://merged.invalid/openai/v1" \ + "$AZURE_SETUP_SCRIPT" >/dev/null + +python3 - "$setup_config" <<'PY' +import os +import stat +import sys +import tomllib + +path = sys.argv[1] +with open(path, "rb") as source: + contents = source.read() +config = tomllib.loads(contents.decode("utf-8")) +assert config["model"] == "merged-model" +assert config["model_provider"] == "azure" +assert config["model_reasoning_effort"] == "high" +assert config["sandbox_mode"] == "workspace-write" +assert config["approval_policy"] == "never" +assert config["features"]["web_search_request"] is True +assert config["literal.key"] == "preserved literal value" +assert 'model = "not a real setting"' in config["features"]["multiline_basic"] +assert '[model_providers.azure]' in config["features"]["multiline_basic"] +assert 'wire_api = "also not a real setting"' in config["features"]["multiline_literal"] +assert config["features"]["preserved_array"] == ["first", "second"] +assert config["quoted.table"]["quoted.key"] == "preserved without a trailing newline" +provider = config["model_providers"]["azure"] +assert provider["name"] == "Azure OpenAI" +assert provider["base_url"] == "https://merged.invalid/openai/v1" +assert provider["env_key"] == "AZURE_OPENAI_API_KEY" +assert provider["wire_api"] == "responses" +assert provider["custom_setting"] == "preserved-provider-value" +assert stat.S_IMODE(os.stat(path).st_mode) == 0o600 +assert not contents.endswith((b"\n", b"\r")) +text = contents.decode("utf-8") +assert '# Existing Codex settings must survive Azure setup.' in text +assert '"approval_policy" = "never" # keep quoted root key' in text +assert '"model" = "merged-model" # keep target comment' in text +assert '["model_providers"."azure"] # keep quoted provider table' in text +assert '"name" = "Azure OpenAI" # keep provider comment' in text +assert '''multiline_basic = """ +model = "not a real setting" +[model_providers.azure] +# this comment belongs to the string +"""''' in text +assert '''multiline_literal = \'\'\' +wire_api = "also not a real setting" +\'\'\'''' in text +assert '''preserved_array = [ + "first", + "second", # keep array comment +]''' in text +PY + +inline_setup_root="${tmp_root}/inline-setup" +inline_setup_config="${inline_setup_root}/config.toml" +mkdir -p "$inline_setup_root" +printf '%s' 'model = "old-model" +model_providers = { azure = { name = "inline Azure" } }' > "$inline_setup_config" +cp "$inline_setup_config" "${inline_setup_config}.expected" +set +e +inline_setup_output="$( + env -u AMIKA_AGENT_CWD \ + PATH="${fake_bin}:$PATH" \ + CODEX_HOME="$inline_setup_root" \ + CODEX_MODEL="merged-model" \ + CODEX_REASONING_EFFORT="high" \ + AZURE_OPENAI_BASE_URL="https://merged.invalid/openai/v1" \ + "$AZURE_SETUP_SCRIPT" 2>&1 +)" +inline_setup_status=$? +set -e +[ "$inline_setup_status" -ne 0 ] || fail "inline Azure provider table should fail closed" +cmp -s "$inline_setup_config" "${inline_setup_config}.expected" || \ + fail "failed inline Azure provider merge changed the original config" +grep -F -q -- 'refusing to modify inline Azure provider setting' <<<"$inline_setup_output" || \ + fail "inline Azure provider failure did not explain why the merge was refused" + +nested_setup_root="${tmp_root}/nested-setup" +nested_setup_config="${nested_setup_root}/config.toml" +mkdir -p "$nested_setup_root" +nested_setup_text="$(cat <<'TOML' +["model_providers"."azure"."http_headers"] # nested provider comment +"x-preserved" = "nested-value" +TOML +)" +printf '%s' "$nested_setup_text" > "$nested_setup_config" +unset nested_setup_text +env -u AMIKA_AGENT_CWD \ + PATH="${fake_bin}:$PATH" \ + CODEX_HOME="$nested_setup_root" \ + CODEX_MODEL="nested-model" \ + CODEX_REASONING_EFFORT="medium" \ + AZURE_OPENAI_BASE_URL="https://nested.invalid/openai/v1" \ + "$AZURE_SETUP_SCRIPT" >/dev/null + +python3 - "$nested_setup_config" <<'PY' +import sys +import tomllib + +path = sys.argv[1] +with open(path, "rb") as source: + contents = source.read() +config = tomllib.loads(contents.decode("utf-8")) +provider = config["model_providers"]["azure"] +assert config["model"] == "nested-model" +assert config["model_provider"] == "azure" +assert config["model_reasoning_effort"] == "medium" +assert provider["name"] == "Azure OpenAI" +assert provider["base_url"] == "https://nested.invalid/openai/v1" +assert provider["env_key"] == "AZURE_OPENAI_API_KEY" +assert provider["wire_api"] == "responses" +assert provider["http_headers"] == {"x-preserved": "nested-value"} +text = contents.decode("utf-8") +assert not contents.endswith((b"\n", b"\r")) +assert '''["model_providers"."azure"."http_headers"] # nested provider comment +"x-preserved" = "nested-value"''' in text +assert text.index("[model_providers.azure]") < text.index('["model_providers"."azure"."http_headers"]') +PY + cat > "${fake_bin}/amika" <<'SH' #!/usr/bin/env bash set -euo pipefail +for secret_name in PROFILE_KEY LOCALITY_PROFILE_KEY AMIKA_SECRET_LINE \ + AZURE_OPENAI_API_KEY_VALUE AZURE_OPENAI_API_KEY; do + if [ -n "${!secret_name+x}" ]; then + if [ -n "${FAKE_CHILD_ENV_CHECK_FILE:-}" ]; then + printf 'leaked %s\n' "$secret_name" > "$FAKE_CHILD_ENV_CHECK_FILE" + fi + printf '%s environment leak\n' "$secret_name" >&2 + exit 90 + fi +done printf 'amika' >> "${FAKE_AMIKA_LOG:?}" for arg in "$@"; do printf ' %q' "$arg" >> "$FAKE_AMIKA_LOG" @@ -36,25 +223,67 @@ done printf '\n' >> "$FAKE_AMIKA_LOG" if [ "${1:-}" = "sandbox" ] && [ "${2:-}" = "create" ]; then + if [ -n "${FAKE_CREATE_BLOCK_PID_FILE:-}" ]; then + printf '%s\n' "$$" > "$FAKE_CREATE_BLOCK_PID_FILE" + if [ "${FAKE_CREATE_IGNORE_SIGNALS:-false}" = true ]; then + trap '' HUP INT TERM + fi + exec sleep 300 + fi + if [ -n "${FAKE_CREATE_STATUS:-}" ]; then + exit "$FAKE_CREATE_STATUS" + fi printf 'created\n' exit 0 fi if [ "${1:-}" = "sandbox" ] && [ "${2:-}" = "ssh" ]; then - case " $* " in - *" --profile-key-stdin "*) + last_arg="" + for arg in "$@"; do + last_arg="$arg" + done + encoded="${last_arg##* }" + decoded="$(printf '%s' "$encoded" | base64 -d | tr '\0' '\n')" + decoded_log="$(printf '%s' "$encoded" | base64 -d | tr '\0' ' ')" + printf 'remote %s\n' "$decoded_log" >> "$FAKE_AMIKA_LOG" + if [ -n "${FAKE_SIGNAL_MATCH:-}" ] && grep -F -q -- "$FAKE_SIGNAL_MATCH" <<<"$decoded_log"; then + kill -TERM "$$" + fi + if [ -n "${FAKE_BLOCK_MATCH:-}" ] && grep -F -q -- "$FAKE_BLOCK_MATCH" <<<"$decoded_log"; then + trap '' HUP INT TERM + sleep 300 & + blocking_child=$! + printf '%s %s %s\n' "$PPID" "$$" "$blocking_child" > "${FAKE_BLOCK_PID_FILE:?}" + wait "$blocking_child" + fi + case "$decoded_log" in + *"--profile-key-stdin"*) + failures="${FAKE_PRE_SENTINEL_FAILURES:-0}" + count=0 + if [ -n "${FAKE_PRE_SENTINEL_COUNT:-}" ] && [ -f "$FAKE_PRE_SENTINEL_COUNT" ]; then + count="$(cat "$FAKE_PRE_SENTINEL_COUNT")" + fi + count=$((count + 1)) + if [ -n "${FAKE_PRE_SENTINEL_COUNT:-}" ]; then + printf '%s\n' "$count" > "$FAKE_PRE_SENTINEL_COUNT" + fi + if [ "$count" -le "$failures" ]; then + exit 255 + fi + printf '__LOCALITY_STDIN_READY__\n' IFS= read -r token printf '%s\n' "$token" > "${FAKE_PROFILE_KEY_INPUT:?}" + if [ -n "${FAKE_POST_SENTINEL_STATUS:-}" ]; then + exit "$FAKE_POST_SENTINEL_STATUS" + fi printf '{"ok":true,"command":"sandbox_init","root":"/workspace/scoped"}\n' ;; *"base64 -d > /home/amika/scenario-prompt.md"*) - last_arg="" - for arg in "$@"; do - last_arg="$arg" - done - printf '%s' "$last_arg" | base64 -d > "${FAKE_PROMPT_INPUT:?}" + prompt_base64="$(printf '%s\n' "$decoded" | tail -n 1)" + printf '%s' "$prompt_base64" | base64 -d > "${FAKE_PROMPT_INPUT:?}" ;; *"codex exec"*) + printf '__LOCALITY_STDIN_READY__\n' IFS= read -r azure_key printf '%s\n' "$azure_key" > "${FAKE_AZURE_INPUT:?}" printf '# Fake Launch Gate Memo\n\nVerified report body.\n' > "${FAKE_REPORT:?}" @@ -74,8 +303,187 @@ exit 2 SH chmod +x "${fake_bin}/amika" -profile_key="$(printf 'a%.0s' {1..64})" +cat > "${fake_bin}/date" <<'SH' +#!/usr/bin/env bash +set -euo pipefail + +for secret_name in PROFILE_KEY LOCALITY_PROFILE_KEY AMIKA_SECRET_LINE \ + AZURE_OPENAI_API_KEY_VALUE AZURE_OPENAI_API_KEY; do + if [ -n "${!secret_name+x}" ]; then + printf 'leaked %s\n' "$secret_name" > "${FAKE_CHILD_ENV_CHECK_FILE:?}" + exit 90 + fi +done +printf '20260803-000000\n' +SH +chmod +x "${fake_bin}/date" + +profile_key="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" azure_key="test-azure-key" + +: > "$fake_log" +child_env_check="${tmp_root}/child-env-check" +set +e +allexport_output="$( + printf '%s\n' "$profile_key" | \ + env \ + PROFILE_KEY="inherited-exported-profile-key" \ + LOCALITY_PROFILE_KEY="inherited-documented-profile-key" \ + AMIKA_SECRET_LINE="inherited-legacy-secret" \ + AZURE_OPENAI_API_KEY_VALUE="inherited-exported-azure-key" \ + AZURE_OPENAI_API_KEY="$azure_key" \ + PATH="${fake_bin}:$PATH" \ + FAKE_AMIKA_LOG="$fake_log" \ + FAKE_CHILD_ENV_CHECK_FILE="$child_env_check" \ + FAKE_CREATE_STATUS=17 \ + bash -a "$SCRIPT" \ + --api-url https://api.dev.locality.dev 2>&1 +)" +allexport_status=$? +set -e +[ "$allexport_status" -eq 2 ] || fail "allexport fixture should stop at sandbox creation" +[ ! -e "$child_env_check" ] || \ + fail "secret shell variables leaked to a child under allexport: $(cat "$child_env_check")" +if grep -F -q -- 'environment leak' <<<"$allexport_output"; then + fail "inherited exported secret variables leaked to a child" +fi + +tty_bin="${tmp_root}/tty-bin" +tty_stty_log="${tmp_root}/tty-stty.log" +tty_restored="${tmp_root}/tty-restored" +mkdir -p "$tty_bin" +cat > "${tty_bin}/stty" <<'SH' +#!/usr/bin/env bash +set -u +printf '%s\n' "$*" >> "${FAKE_STTY_LOG:?}" +"${REAL_STTY:?}" "$@" +stty_status=$? +case "${1:-}" in + -g|-echo) ;; + *) + if [ "$stty_status" -eq 0 ]; then + printf 'restored\n' > "${FAKE_STTY_RESTORED:?}" + fi + ;; +esac +exit "$stty_status" +SH +chmod +x "${tty_bin}/stty" + +: > "$fake_log" +set +e +TTY_TEST_PATH="${tty_bin}:${fake_bin}:$PATH" \ + TTY_TEST_SCRIPT="$SCRIPT" \ + TTY_TEST_AZURE_KEY="$azure_key" \ + TTY_TEST_AMIKA_LOG="$fake_log" \ + TTY_TEST_STTY_LOG="$tty_stty_log" \ + TTY_TEST_RESTORED="$tty_restored" \ + TTY_TEST_REAL_STTY="$(command -v stty)" \ + expect -c ' + set timeout 10 + spawn -noecho env \ + PATH=$env(TTY_TEST_PATH) \ + AZURE_OPENAI_API_KEY=$env(TTY_TEST_AZURE_KEY) \ + FAKE_AMIKA_LOG=$env(TTY_TEST_AMIKA_LOG) \ + FAKE_STTY_LOG=$env(TTY_TEST_STTY_LOG) \ + FAKE_STTY_RESTORED=$env(TTY_TEST_RESTORED) \ + REAL_STTY=$env(TTY_TEST_REAL_STTY) \ + $env(TTY_TEST_SCRIPT) \ + --api-url https://api.dev.locality.dev \ + --name interrupted-read + set public_script_pid [exp_pid] + set echo_disabled 0 + for {set attempt 0} {$attempt < 100} {incr attempt} { + after 50 + if {[file exists $env(TTY_TEST_STTY_LOG)]} { + set handle [open $env(TTY_TEST_STTY_LOG) r] + set log [read $handle] + close $handle + if {[string first "-echo\n" $log] >= 0} { + set echo_disabled 1 + break + } + } + } + if {!$echo_disabled} { + catch {exec kill -TERM $public_script_pid} + catch {expect eof} + catch {wait} + exit 124 + } + exec kill -TERM $public_script_pid + expect eof + set result [wait] + if {[lindex $result 2] != 0} { + exit 125 + } + exit [lindex $result 3] + ' >/dev/null 2>&1 +tty_interrupt_status=$? +set -e +[ "$tty_interrupt_status" -eq 143 ] || \ + fail "TERM-interrupted public script credential read should return 143, got ${tty_interrupt_status}" +[ -s "$tty_restored" ] || \ + fail "credential read interruption did not restore the saved terminal state: $(cat "$tty_stty_log")" +grep -F -x -q -- '-g' "$tty_stty_log" || fail "credential read did not capture terminal state" +grep -F -x -q -- '-echo' "$tty_stty_log" || fail "credential read did not disable terminal echo" +[ ! -s "$fake_log" ] || fail "credential read interruption contacted Amika" + +: > "$fake_log" +: > "$tty_stty_log" +TTY_TEST_PATH="${tty_bin}:${fake_bin}:$PATH" \ + TTY_TEST_SCRIPT="$SCRIPT" \ + TTY_TEST_AZURE_KEY="$azure_key" \ + TTY_TEST_AMIKA_LOG="$fake_log" \ + TTY_TEST_STTY_LOG="$tty_stty_log" \ + TTY_TEST_RESTORED="$tty_restored" \ + TTY_TEST_REAL_STTY="$(command -v stty)" \ + TTY_TEST_PROFILE_KEY="$profile_key" \ + expect -c ' + set timeout 10 + spawn -noecho env \ + PATH=$env(TTY_TEST_PATH) \ + AZURE_OPENAI_API_KEY=$env(TTY_TEST_AZURE_KEY) \ + FAKE_AMIKA_LOG=$env(TTY_TEST_AMIKA_LOG) \ + FAKE_STTY_LOG=$env(TTY_TEST_STTY_LOG) \ + FAKE_STTY_RESTORED=$env(TTY_TEST_RESTORED) \ + REAL_STTY=$env(TTY_TEST_REAL_STTY) \ + FAKE_CREATE_STATUS=17 \ + $env(TTY_TEST_SCRIPT) \ + --api-url https://api.dev.locality.dev \ + --name delayed-read + set echo_disabled 0 + for {set attempt 0} {$attempt < 100} {incr attempt} { + after 50 + if {[file exists $env(TTY_TEST_STTY_LOG)]} { + set handle [open $env(TTY_TEST_STTY_LOG) r] + set log [read $handle] + close $handle + if {[string first "-echo\n" $log] >= 0} { + set echo_disabled 1 + break + } + } + } + if {!$echo_disabled} { + catch {exec kill -TERM [exp_pid]} + catch {expect eof} + catch {wait} + exit 124 + } + set key $env(TTY_TEST_PROFILE_KEY) + send -- [string range $key 0 31] + after 1200 + send -- "[string range $key 32 end]\r" + expect eof + set result [wait] + if {[lindex $result 2] != 0 || [lindex $result 3] != 2} { + exit 125 + } + ' >/dev/null 2>&1 || fail "delayed terminal key input was not preserved across polling timeouts" +assert_contains "$fake_log" "sandbox create --remote --name delayed-read --no-git --yes" +grep -F -q -- "$profile_key" "$fake_log" && fail "delayed terminal key leaked to Amika arguments" + output="$( printf '%s\n' "$profile_key" | \ PATH="${fake_bin}:$PATH" \ @@ -92,8 +500,8 @@ output="$( --reasoning medium )" -assert_contains "$fake_log" "sandbox create --remote --name test-snapshot --yes" -assert_contains "$fake_log" "sandbox ssh test-snapshot" +assert_contains "$fake_log" "sandbox create --remote --name test-snapshot --no-git --yes" +assert_contains "$fake_log" "sandbox ssh -t test-snapshot" assert_contains "$fake_log" "Locality_Linux_v" assert_contains "$fake_log" "0.3.7" assert_contains "$fake_log" "692b05460839ba44b85cd1e6b3b6969ad4a3f62f3e81f420c4651159ad7ef195" @@ -107,12 +515,19 @@ assert_contains "$fake_log" "sandbox init" assert_contains "$fake_log" "--api-url https://api.dev.locality.dev" assert_contains "$fake_log" "--root /home/amika/locality-snapshot" assert_contains "$fake_log" "--profile-key-stdin" +assert_contains "$fake_log" "--profile" assert_contains "$fake_log" "/home/amika/scenario-prompt.md" assert_contains "$fake_log" "setup-codex-azure.sh" +assert_contains "$fake_log" 'AMIKA_AGENT_CWD="$HOME"' assert_contains "$fake_log" "codex exec" assert_contains "$fake_log" '< /dev/null' assert_contains "$fake_log" "test-model medium /home/amika/locality-snapshot" assert_contains "$fake_log" "cat /home/amika/final_report.md" +assert_contains "$fake_log" "status --porcelain --untracked-files=all" +profile_exchange_line="$(grep -n -m1 -- '--profile-key-stdin' "$fake_log" | cut -d: -f1)" +repo_prepare_line="$(grep -n -m1 -- 'git clone https://github.com/codeflash-ai/locality.git' "$fake_log" | cut -d: -f1)" +[ "$profile_exchange_line" -lt "$repo_prepare_line" ] || \ + fail "scenario repository must be prepared only after profile authorization succeeds" if grep -F -q -- "$profile_key" "$fake_log"; then fail "Workspace Profile key leaked into Amika arguments" fi @@ -136,6 +551,266 @@ grep -F -q -- '===== /home/amika/final_report.md =====' <<<"$output" || \ grep -F -q -- 'Verified report body.' <<<"$output" || \ fail "terminal output did not include final_report.md" +: > "$fake_log" +set +e +reuse_output="$( + PATH="${fake_bin}:$PATH" \ + AZURE_OPENAI_API_KEY="$azure_key" \ + FAKE_AMIKA_LOG="$fake_log" \ + "$SCRIPT" \ + --api-url https://api.dev.locality.dev \ + --name existing-snapshot \ + --reuse &1 +)" +reuse_status=$? +set -e +[ "$reuse_status" -eq 2 ] || fail "--reuse should be refused with status 2" +[ ! -s "$fake_log" ] || fail "refused --reuse contacted Amika" +grep -F -q -- 'existing sandbox is not a trusted credential boundary' <<<"$reuse_output" || \ + fail "refused --reuse did not explain the trust boundary" + +: > "$fake_log" +set +e +create_failure_output="$( + printf '%s\n' "$profile_key" | \ + PATH="${fake_bin}:$PATH" \ + AZURE_OPENAI_API_KEY="$azure_key" \ + FAKE_AMIKA_LOG="$fake_log" \ + FAKE_CREATE_STATUS=17 \ + "$SCRIPT" \ + --api-url https://api.dev.locality.dev \ + --name colliding-snapshot 2>&1 +)" +create_failure_status=$? +set -e +[ "$create_failure_status" -eq 2 ] || fail "fresh sandbox creation failure should fail closed" +assert_contains "$fake_log" "sandbox create --remote --name colliding-snapshot --no-git --yes" +assert_not_contains "$fake_log" "sandbox ssh" +grep -F -q -- 'no credentials were transferred' <<<"$create_failure_output" || \ + fail "sandbox collision failure did not explain credential safety" + +real_expect="$(command -v expect)" +run_create_interruption() { + local signal="$1" + local expected_status="$2" + local block_pid_file="${tmp_root}/create-block-${signal}.pid" + local input="${tmp_root}/create-interrupt-${signal}.input" + local output="${tmp_root}/create-interrupt-${signal}.output" + local ignore_signals=false + local interrupt_status + local blocked_create_pid + + if [ "$signal" = TERM ]; then + ignore_signals=true + fi + : > "$fake_log" + printf '%s\n' "$profile_key" > "$input" + set +e + CREATE_TEST_PATH="${fake_bin}:$PATH" \ + CREATE_TEST_SCRIPT="$SCRIPT" \ + CREATE_TEST_INPUT="$input" \ + CREATE_TEST_SIGNAL="$signal" \ + CREATE_TEST_AZURE_KEY="$azure_key" \ + CREATE_TEST_AMIKA_LOG="$fake_log" \ + CREATE_TEST_BLOCK_PID_FILE="$block_pid_file" \ + CREATE_TEST_IGNORE_SIGNALS="$ignore_signals" \ + "$real_expect" -c ' + set timeout 10 + spawn -noecho env \ + PATH=$env(CREATE_TEST_PATH) \ + AZURE_OPENAI_API_KEY=$env(CREATE_TEST_AZURE_KEY) \ + FAKE_AMIKA_LOG=$env(CREATE_TEST_AMIKA_LOG) \ + FAKE_CREATE_BLOCK_PID_FILE=$env(CREATE_TEST_BLOCK_PID_FILE) \ + FAKE_CREATE_IGNORE_SIGNALS=$env(CREATE_TEST_IGNORE_SIGNALS) \ + sh -c {exec "$1" --api-url https://api.dev.locality.dev --name "interrupted-create-$3" < "$2"} \ + sh $env(CREATE_TEST_SCRIPT) $env(CREATE_TEST_INPUT) $env(CREATE_TEST_SIGNAL) + set public_script_pid [exp_pid] + set child_started 0 + for {set attempt 0} {$attempt < 100} {incr attempt} { + after 50 + if {[file exists $env(CREATE_TEST_BLOCK_PID_FILE)] && [file size $env(CREATE_TEST_BLOCK_PID_FILE)] > 0} { + set child_started 1 + break + } + } + if {!$child_started} { + catch {exec kill -KILL $public_script_pid} + catch {expect eof} + catch {wait} + exit 124 + } + exec kill -$env(CREATE_TEST_SIGNAL) $public_script_pid + expect { + eof {} + timeout { + catch {exec kill -KILL $public_script_pid} + catch {expect eof} + catch {wait} + exit 124 + } + } + set result [wait] + if {[lindex $result 2] != 0} { + exit 125 + } + exit [lindex $result 3] + ' >"$output" 2>&1 + interrupt_status=$? + set -e + [ "$interrupt_status" -eq "$expected_status" ] || \ + fail "${signal}-interrupted public script sandbox creation should return ${expected_status}, got ${interrupt_status}: $(cat "$output")" + [ -s "$block_pid_file" ] || fail "${signal} sandbox creation interruption fixture did not start" + blocked_create_pid="$(cat "$block_pid_file")" + if kill -0 "$blocked_create_pid" 2>/dev/null; then + fail "${signal}-interrupted public script left sandbox creation child ${blocked_create_pid} running" + fi + assert_contains "$fake_log" "sandbox create --remote --name interrupted-create-${signal} --no-git --yes" + assert_not_contains "$fake_log" "sandbox ssh" +} + +run_create_interruption HUP 129 +run_create_interruption INT 130 +run_create_interruption TERM 143 + +: > "$fake_log" +pre_sentinel_count="${tmp_root}/pre-sentinel.count" +retry_output="$( + printf '%s\n' "$profile_key" | \ + PATH="${fake_bin}:$PATH" \ + AZURE_OPENAI_API_KEY="$azure_key" \ + FAKE_AMIKA_LOG="$fake_log" \ + FAKE_PROFILE_KEY_INPUT="$profile_key_input" \ + FAKE_AZURE_INPUT="$azure_input" \ + FAKE_PROMPT_INPUT="$prompt_input" \ + FAKE_REPORT="$fake_report" \ + FAKE_PRE_SENTINEL_FAILURES=1 \ + FAKE_PRE_SENTINEL_COUNT="$pre_sentinel_count" \ + "$SCRIPT" \ + --api-url https://api.dev.locality.dev \ + --name retry-snapshot \ + --model test-model \ + --reasoning medium 2>&1 +)" +[ "$(cat "$pre_sentinel_count")" -eq 2 ] || fail "pre-sentinel transport failure was not retried exactly once" +grep -F -q -- 'retrying (1/3)' <<<"$retry_output" || fail "pre-sentinel retry was not explained" +[ "$(cat "$profile_key_input")" = "$profile_key" ] || fail "retry did not stream the Workspace Profile key" + +: > "$fake_log" +set +e +signal_output="$( + printf '%s\n' "$profile_key" | \ + PATH="${fake_bin}:$PATH" \ + AZURE_OPENAI_API_KEY="$azure_key" \ + FAKE_AMIKA_LOG="$fake_log" \ + FAKE_PROFILE_KEY_INPUT="$profile_key_input" \ + FAKE_AZURE_INPUT="$azure_input" \ + FAKE_PROMPT_INPUT="$prompt_input" \ + FAKE_REPORT="$fake_report" \ + FAKE_SIGNAL_MATCH='Locality_Linux_v' \ + "$SCRIPT" --api-url https://api.dev.locality.dev --name signaled 2>&1 +)" +signal_status=$? +set -e +[ "$signal_status" -eq 143 ] || fail "signaled Amika child should return 143, got ${signal_status}: ${signal_output}" + +: > "$fake_log" +block_pid_file="${tmp_root}/blocked-child.pids" +interrupt_output="${tmp_root}/interrupt.output" +interrupt_input="${tmp_root}/interrupt.input" +printf '%s\n' "$profile_key" > "$interrupt_input" +PATH="${fake_bin}:$PATH" \ + AZURE_OPENAI_API_KEY="$azure_key" \ + FAKE_AMIKA_LOG="$fake_log" \ + FAKE_PROFILE_KEY_INPUT="$profile_key_input" \ + FAKE_AZURE_INPUT="$azure_input" \ + FAKE_PROMPT_INPUT="$prompt_input" \ + FAKE_REPORT="$fake_report" \ + FAKE_BLOCK_MATCH='Locality_Linux_v' \ + FAKE_BLOCK_PID_FILE="$block_pid_file" \ + "$SCRIPT" --api-url https://api.dev.locality.dev --name interrupted \ + <"$interrupt_input" >"$interrupt_output" 2>&1 & +public_script_pid=$! +for _ in $(seq 1 100); do + [ -s "$block_pid_file" ] && break + sleep 0.05 +done +[ -s "$block_pid_file" ] || fail "interruption fixture did not start the remote child" +read -r blocked_expect_pid blocked_amika_pid blocked_descendant_pid < "$block_pid_file" +kill -TERM "$public_script_pid" +set +e +wait "$public_script_pid" +interrupt_status=$? +set -e +[ "$interrupt_status" -eq 143 ] || \ + fail "TERM-interrupted public script should return 143, got ${interrupt_status}: $(cat "$interrupt_output")" +for _ in $(seq 1 100); do + if ! kill -0 "$blocked_expect_pid" 2>/dev/null && \ + ! kill -0 "$blocked_amika_pid" 2>/dev/null && \ + ! kill -0 "$blocked_descendant_pid" 2>/dev/null; then + break + fi + sleep 0.05 +done +if kill -0 "$blocked_expect_pid" 2>/dev/null; then + fail "interrupted public script left Expect wrapper ${blocked_expect_pid} running" +fi +if kill -0 "$blocked_amika_pid" 2>/dev/null; then + fail "interrupted public script left Amika child ${blocked_amika_pid} running" +fi +if kill -0 "$blocked_descendant_pid" 2>/dev/null; then + fail "interrupted public script left descendant ${blocked_descendant_pid} running" +fi + +: > "$fake_log" +post_sentinel_count="${tmp_root}/post-sentinel.count" +set +e +post_sentinel_output="$( + printf '%s\n' "$profile_key" | \ + PATH="${fake_bin}:$PATH" \ + AZURE_OPENAI_API_KEY="$azure_key" \ + FAKE_AMIKA_LOG="$fake_log" \ + FAKE_PROFILE_KEY_INPUT="$profile_key_input" \ + FAKE_AZURE_INPUT="$azure_input" \ + FAKE_PROMPT_INPUT="$prompt_input" \ + FAKE_REPORT="$fake_report" \ + FAKE_PRE_SENTINEL_COUNT="$post_sentinel_count" \ + FAKE_POST_SENTINEL_STATUS=23 \ + "$SCRIPT" --api-url https://api.dev.locality.dev --name post-sentinel 2>&1 +)" +post_sentinel_status=$? +set -e +[ "$post_sentinel_status" -eq 23 ] || fail "post-sentinel failure should preserve status 23" +[ "$(cat "$post_sentinel_count")" -eq 1 ] || fail "post-sentinel failure must not retry" +if grep -F -q -- 'credential transport closed' <<<"$post_sentinel_output"; then + fail "post-sentinel failure was incorrectly classified as retryable transport" +fi + +: > "$fake_log" +post_sentinel_75_count="${tmp_root}/post-sentinel-75.count" +set +e +post_sentinel_75_output="$( + printf '%s\n' "$profile_key" | \ + PATH="${fake_bin}:$PATH" \ + AZURE_OPENAI_API_KEY="$azure_key" \ + FAKE_AMIKA_LOG="$fake_log" \ + FAKE_PROFILE_KEY_INPUT="$profile_key_input" \ + FAKE_AZURE_INPUT="$azure_input" \ + FAKE_PROMPT_INPUT="$prompt_input" \ + FAKE_REPORT="$fake_report" \ + FAKE_PRE_SENTINEL_COUNT="$post_sentinel_75_count" \ + FAKE_POST_SENTINEL_STATUS=75 \ + "$SCRIPT" --api-url https://api.dev.locality.dev --name post-sentinel-75 2>&1 +)" +post_sentinel_75_status=$? +set -e +[ "$post_sentinel_75_status" -eq 75 ] || \ + fail "post-sentinel child status 75 should be preserved, got ${post_sentinel_75_status}" +[ "$(cat "$post_sentinel_75_count")" -eq 1 ] || \ + fail "post-sentinel child status 75 replayed the credential" +if grep -F -q -- 'credential transport closed' <<<"$post_sentinel_75_output"; then + fail "post-sentinel child status 75 was incorrectly classified as pre-delivery transport failure" +fi + : > "$fake_log" set +e missing_azure_output="$( @@ -173,7 +848,7 @@ invalid_status=$? set -e [ "$invalid_status" -eq 2 ] || fail "empty Workspace Profile key should return usage status 2" [ ! -s "$fake_log" ] || fail "empty Workspace Profile key should fail before creating a sandbox" -grep -F -q -- 'Workspace Profile key must not be empty' <<<"$invalid_output" || \ +grep -F -q -- 'Workspace Profile key must be 64 lowercase hexadecimal characters' <<<"$invalid_output" || \ fail "empty Workspace Profile key error was not actionable" printf 'init Amika Locality snapshot tests passed\n'