From fee3d211eb4632e1ba57cef2529ea57464e3c21e Mon Sep 17 00:00:00 2001 From: Ege Demir Date: Tue, 13 Oct 2026 12:07:08 +0000 Subject: [PATCH 1/4] Add a staging deployment that cannot touch production's state Staging runs on its own host with its own Postgres, Temporal namespace, Clerk instance, GitHub App and Stripe test endpoint. It shares production's code.storage organization, so every repository it touches carries a prefix. - TIN_LITE_CODE_STORAGE_REPO_PREFIX is applied wherever Tin names a repository upstream (lookup, create, delete, sandbox remotes, scoped tokens); stored references keep the unprefixed ID. - TIN_LITE_ENVIRONMENT=staging refuses to boot without a prefix, outside Stripe test mode, or on production's database host or Temporal namespace. - deploy.sh takes TIN_LITE_DEPLOY_TARGET (prod by default, unchanged). Staging keeps its release env on the host and migrates before switching releases. - deploy-staging deploys main and on-demand refs through workload identity scoped to the staging environment and host. - install_staging_postgres.sh sets up stock Postgres like production's; a test stands in for production's pg_strict (no DELETE or UPDATE without WHERE). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deploy-staging.yml | 48 +++++++++ infra/activate_release.sh | 19 +++- infra/deploy.sh | 90 ++++++++++++----- infra/install_staging_postgres.sh | 80 +++++++++++++++ scripts/complete_organic_audit.py | 1 + src/tin_lite/cli.py | 2 + src/tin_lite/code_storage.py | 24 +++-- src/tin_lite/runtime.py | 1 + src/tin_lite/settings.py | 36 +++++++ tests/test_secret_fences.py | 2 +- tests/test_sql_requires_where.py | 78 ++++++++++++++ tests/test_staging_isolation.py | 145 +++++++++++++++++++++++++++ 12 files changed, 491 insertions(+), 35 deletions(-) create mode 100644 .github/workflows/deploy-staging.yml create mode 100644 infra/install_staging_postgres.sh create mode 100644 tests/test_sql_requires_where.py create mode 100644 tests/test_staging_isolation.py diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml new file mode 100644 index 00000000..150628a9 --- /dev/null +++ b/.github/workflows/deploy-staging.yml @@ -0,0 +1,48 @@ +name: deploy-staging + +# Staging runs every merge to main, and any branch on request. Production deploys stay manual. +on: + push: + branches: [main] + workflow_dispatch: + inputs: + ref: + description: Branch, tag or SHA to deploy to staging + required: true + default: main + +permissions: + contents: read + +# One release at a time; a newer request waits instead of interrupting a deploy midway. +concurrency: + group: deploy-staging + cancel-in-progress: false + +jobs: + deploy: + runs-on: ubuntu-latest + timeout-minutes: 30 + # The staging environment is the only identity Google Cloud trusts for this deployer. + environment: + name: staging + url: https://app.staging.tin.computer + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.ref || github.sha }} + persist-credentials: false + - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 + with: + workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.GCP_DEPLOY_SERVICE_ACCOUNT }} + - uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 + - name: Deploy + env: + TIN_LITE_DEPLOY_TARGET: staging + TIN_LITE_GCP_PROJECT: ${{ vars.GCP_PROJECT }} + TIN_LITE_STATIC_IP: ${{ vars.STAGING_STATIC_IP }} + run: bash infra/deploy.sh diff --git a/infra/activate_release.sh b/infra/activate_release.sh index 9408a414..ceb1b190 100755 --- a/infra/activate_release.sh +++ b/infra/activate_release.sh @@ -26,12 +26,29 @@ if [[ -f /tmp/tin-lite-github-app.pem ]]; then 's#^TIN_LITE_GITHUB_APP_PRIVATE_KEY_PATH=.*#TIN_LITE_GITHUB_APP_PRIVATE_KEY_PATH=/etc/tin-lite/github-app.pem#' \ /tmp/tin-lite.env fi -install -m 640 -o tinlite -g tinlite /tmp/tin-lite.env "${release_dir}/.env" +# An uploaded .env wins; otherwise the release env kept on the host (staging) is used. +release_env=/tmp/tin-lite.env +if [[ ! -f "${release_env}" ]]; then + release_env=/etc/tin-lite/release.env +fi +if [[ ! -f "${release_env}" ]]; then + echo "no release env: upload .env or keep /etc/tin-lite/release.env on the host" >&2 + exit 66 +fi +install -m 640 -o tinlite -g tinlite "${release_env}" "${release_dir}/.env" chown -R tinlite:tinlite "${release_dir}" sudo -u tinlite env UV_CACHE_DIR=/var/cache/tin-lite-uv \ /usr/local/bin/uv sync --frozen --no-dev --directory "${release_dir}" +# Migrations run from the new release, under the service env, before it goes live. +if [[ "${TIN_LITE_RUN_MIGRATIONS:-0}" == "1" ]]; then + systemd-run --wait --pipe --collect -p User=tinlite -p WorkingDirectory="${release_dir}" \ + -p EnvironmentFile=/etc/tin-lite/clerk.env -p EnvironmentFile=/etc/tin-lite/runtime.env \ + -E UV_CACHE_DIR=/var/cache/tin-lite-uv \ + /usr/local/bin/uv run --frozen --no-dev tin-lite migrate +fi + ln -sfn "${release_dir}" /opt/tin-lite/current systemctl enable tin-lite-switchboard systemctl restart tin-lite-switchboard diff --git a/infra/deploy.sh b/infra/deploy.sh index bb2fa189..8d5b711a 100755 --- a/infra/deploy.sh +++ b/infra/deploy.sh @@ -3,22 +3,50 @@ set -euo pipefail : "${TIN_LITE_GCP_PROJECT:?Set TIN_LITE_GCP_PROJECT to your deployment project}" +# prod: the release env is uploaded from a local .env. staging: it stays on the host in +# /etc/tin-lite/release.env, and migrations run before the release goes live. +target="${TIN_LITE_DEPLOY_TARGET:-prod}" +case "${target}" in + prod) + instance=tin-lite-switchboard + address=tin-lite-switchboard-ip + default_host=app.tin.computer + upload_env=1 + run_migrations="${TIN_LITE_RUN_MIGRATIONS:-0}" + ;; + staging) + instance=tin-lite-staging + address=tin-lite-staging-ip + default_host=app.staging.tin.computer + upload_env=0 + run_migrations=1 + ;; + *) + echo "TIN_LITE_DEPLOY_TARGET must be prod or staging" >&2 + exit 64 + ;; +esac +zone=us-central1-a +region=us-central1 + root="$(git rev-parse --show-toplevel)" cd "${root}" if [[ -n "$(git status --porcelain --untracked-files=no)" ]]; then echo "tracked worktree changes must be committed before deployment" >&2 exit 65 fi -if [[ ! -f .env ]]; then - echo "local .env is required" >&2 - exit 66 -fi -if rg -q '^OPENAI_API_KEY=' .env; then - echo "OPENAI_API_KEY is forbidden" >&2 - exit 67 +github_private_key_path="" +if [[ "${upload_env}" == "1" ]]; then + if [[ ! -f .env ]]; then + echo "local .env is required" >&2 + exit 66 + fi + if grep -q '^OPENAI_API_KEY=' .env; then + echo "OPENAI_API_KEY is forbidden" >&2 + exit 67 + fi + github_private_key_path="$(sed -n 's/^TIN_LITE_GITHUB_APP_PRIVATE_KEY_PATH=//p' .env | tail -n 1)" fi - -github_private_key_path="$(sed -n 's/^TIN_LITE_GITHUB_APP_PRIVATE_KEY_PATH=//p' .env | tail -n 1)" if [[ -n "${github_private_key_path}" ]]; then if [[ ! -f "${github_private_key_path}" ]]; then echo "TIN_LITE_GITHUB_APP_PRIVATE_KEY_PATH must name a readable local file" >&2 @@ -35,45 +63,57 @@ temporary_dir="$(mktemp -d)" trap 'rm -rf "${temporary_dir}"' EXIT git archive --format=tar.gz --output="${temporary_dir}/tin-lite-release.tar.gz" HEAD +release_files=("${temporary_dir}/tin-lite-release.tar.gz") +if [[ "${upload_env}" == "1" ]]; then + release_files+=(.env) +fi gcloud compute scp \ - "${temporary_dir}/tin-lite-release.tar.gz" \ - .env \ + "${release_files[@]}" \ infra/install_switchboard.sh \ infra/install_forward_proxy.sh \ infra/proxy_policy.py \ src/tin_lite/proxy_grants.py \ infra/activate_release.sh \ - tin-lite-switchboard:/tmp/ \ - --project "${TIN_LITE_GCP_PROJECT}" --zone us-central1-a --quiet + "${instance}:/tmp/" \ + --project "${TIN_LITE_GCP_PROJECT}" --zone "${zone}" --quiet if [[ -n "${github_private_key_path}" ]]; then gcloud compute scp \ "${github_private_key_path}" \ - tin-lite-switchboard:/tmp/tin-lite-github-app.pem \ - --project "${TIN_LITE_GCP_PROJECT}" --zone us-central1-a --quiet - gcloud compute ssh tin-lite-switchboard \ - --project "${TIN_LITE_GCP_PROJECT}" --zone us-central1-a --quiet \ + "${instance}:/tmp/tin-lite-github-app.pem" \ + --project "${TIN_LITE_GCP_PROJECT}" --zone "${zone}" --quiet + gcloud compute ssh "${instance}" \ + --project "${TIN_LITE_GCP_PROJECT}" --zone "${zone}" --quiet \ --command="sudo chown root:root /tmp/tin-lite-github-app.pem && \ sudo chmod 600 /tmp/tin-lite-github-app.pem" fi -static_ip="$(gcloud compute addresses describe tin-lite-switchboard-ip \ - --project "${TIN_LITE_GCP_PROJECT}" --region us-central1 --format='value(address)')" -product_host="${TIN_LITE_PRODUCT_HOST:-app.tin.computer}" +static_ip="${TIN_LITE_STATIC_IP:-}" +if [[ -z "${static_ip}" ]]; then + static_ip="$(gcloud compute addresses describe "${address}" \ + --project "${TIN_LITE_GCP_PROJECT}" --region "${region}" --format='value(address)')" +fi +if [[ ! "${static_ip}" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "the ${target} static IP is not an IPv4 address" >&2 + exit 64 +fi +product_host="${TIN_LITE_PRODUCT_HOST:-${default_host}}" if [[ ! "${product_host}" =~ ^[a-zA-Z0-9.-]+$ ]]; then echo "TIN_LITE_PRODUCT_HOST must be a DNS hostname" >&2 exit 64 fi -gcloud compute ssh tin-lite-switchboard \ - --project "${TIN_LITE_GCP_PROJECT}" --zone us-central1-a --quiet \ +move_env="" +if [[ "${upload_env}" == "1" ]]; then + move_env="sudo mv /tmp/.env /tmp/tin-lite.env && " +fi +gcloud compute ssh "${instance}" \ + --project "${TIN_LITE_GCP_PROJECT}" --zone "${zone}" --quiet \ --command="sudo env TIN_LITE_STATIC_IP=${static_ip} TIN_LITE_PRODUCT_HOST=${product_host} bash /tmp/install_switchboard.sh && \ -sudo mv /tmp/.env /tmp/tin-lite.env && \ -sudo bash /tmp/activate_release.sh ${release_sha}" +${move_env}sudo env TIN_LITE_RUN_MIGRATIONS=${run_migrations} bash /tmp/activate_release.sh ${release_sha}" public_host="tin-lite-switchboard.${static_ip//./-}.sslip.io" curl --http1.1 --fail --silent --show-error --retry 20 --retry-all-errors --retry-delay 3 \ "https://${public_host}/healthz" -product_host="${TIN_LITE_PRODUCT_HOST:-app.tin.computer}" curl --http1.1 --fail --silent --show-error --retry 20 --retry-all-errors --retry-delay 3 \ "https://${product_host}/healthz" diff --git a/infra/install_staging_postgres.sh b/infra/install_staging_postgres.sh new file mode 100644 index 00000000..5c0d5fe0 --- /dev/null +++ b/infra/install_staging_postgres.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Staging's database: stock Postgres on the staging host, set up to behave like production's +# managed Postgres where Tin can tell the difference. Production's pg_strict (no DELETE or +# UPDATE without WHERE) is not available here; tests/test_sql_requires_where.py covers it. +set -euo pipefail + +if [[ "${EUID}" -ne 0 ]]; then + echo "run as root" >&2 + exit 64 +fi + +major="${TIN_LITE_POSTGRES_MAJOR:-18}" +# Production's exact build when the archive still has it; otherwise the latest minor. +version="${TIN_LITE_POSTGRES_VERSION:-18.6-1.pgdg12+2}" +database=tin +role=tin_app +password_file=/etc/tin-lite/postgres-password + +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get install -y ca-certificates curl gnupg postgresql-common +if [[ ! -f /etc/apt/sources.list.d/pgdg.list ]]; then + install -d /usr/share/postgresql-common/pgdg + curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc + echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "${VERSION_CODENAME}")-pgdg main" \ + > /etc/apt/sources.list.d/pgdg.list + apt-get update +fi +if apt-cache madison "postgresql-${major}" | awk -F'|' '{gsub(/ /, "", $2); print $2}' | + grep -qxF "${version}"; then + apt-get install -y "postgresql-${major}=${version}" + apt-mark hold "postgresql-${major}" +else + echo "postgresql-${major} ${version} is not available; installing the latest ${major}.x" >&2 + apt-get install -y "postgresql-${major}" +fi + +install -d -m 755 "/etc/postgresql/${major}/main/conf.d" +cat > "/etc/postgresql/${major}/main/conf.d/tin-staging.conf" <<'EOF' +# Production's values where they shape behavior Tin can observe. +listen_addresses = 'localhost' +max_connections = 25 +ssl = on +timezone = 'UTC' +log_timezone = 'UTC' +work_mem = '2MB' +maintenance_work_mem = '16MB' +shared_buffers = '64MB' +idle_in_transaction_session_timeout = 0 +random_page_cost = 1.1 +effective_io_concurrency = 32 +default_toast_compression = 'lz4' +EOF +systemctl enable "postgresql@${major}-main" +systemctl restart "postgresql@${major}-main" + +install -d -m 750 -o root -g tinlite /etc/tin-lite 2>/dev/null || install -d -m 750 /etc/tin-lite +if [[ ! -s "${password_file}" ]]; then + umask 077 + openssl rand -hex 32 > "${password_file}" +fi +chmod 600 "${password_file}" +password="$(cat "${password_file}")" + +# Like production's application role: no superuser, cannot create databases or roles. +sudo -u postgres psql -v ON_ERROR_STOP=1 -q < None: storage = CodeStorage( organization=settings.code_storage_org, private_key=settings.code_storage_api_key.get_secret_value(), + repo_prefix=settings.code_storage_repo_prefix, ) await storage.ensure_repo(repo_id) database = Database(settings.runtime_dsn) @@ -67,6 +68,7 @@ async def _sync_builtins() -> None: storage = CodeStorage( organization=settings.code_storage_org, private_key=settings.code_storage_api_key.get_secret_value(), + repo_prefix=settings.code_storage_repo_prefix, ) database = Database(settings.runtime_dsn) await database.connect() diff --git a/src/tin_lite/code_storage.py b/src/tin_lite/code_storage.py index a356c2d3..bcc8c7ec 100644 --- a/src/tin_lite/code_storage.py +++ b/src/tin_lite/code_storage.py @@ -181,11 +181,17 @@ class CodeStorage: _http: httpx.AsyncClient | None = None _http_loop: asyncio.AbstractEventLoop | None = None - def __init__(self, *, organization: str, private_key: str) -> None: + def __init__(self, *, organization: str, private_key: str, repo_prefix: str = "") -> None: self._organization = organization self._private_key = private_key + # A deployment sharing an organization (staging) keeps its repositories apart by name. + # Callers and stored references always use the unprefixed ID. + self._repo_prefix = repo_prefix self._client = GitStorage({"name": organization, "key": private_key}) + def _upstream_id(self, repo_id: str) -> str: + return f"{self._repo_prefix}{repo_id}" + # Per-instance and created on first use, so every CodeStorage has its own caches. @functools.cached_property def _pinned(self) -> _PinnedReadCache: @@ -221,7 +227,7 @@ async def ensure_repo( initial_readme: str = "# Project state\n\nManaged by the Tin Lite switchboard.\n", ) -> Repo: try: - repo = await self._client.find_one(id=repo_id) + repo = await self._client.find_one(id=self._upstream_id(repo_id)) except ApiError as exc: # An empty organization currently answers 403 for a repo-scoped lookup before create. if exc.status_code != 403: @@ -229,11 +235,13 @@ async def ensure_repo( repo = None if repo is None: try: - repo = await self._client.create_repo(id=repo_id, default_branch="main", ttl=300) + repo = await self._client.create_repo( + id=self._upstream_id(repo_id), default_branch="main", ttl=300 + ) except ApiError as exc: if exc.status_code != 409: raise - repo = await self._client.find_one(id=repo_id) + repo = await self._client.find_one(id=self._upstream_id(repo_id)) if repo is None: raise if await self.head_sha(repo, "main") is None: @@ -256,7 +264,7 @@ async def delete_repo(self, repo_id: str) -> bool: """True when the repository is gone after this call; upstream cleanup is asynchronous.""" self._forget_repo(repo_id) try: - await self._client.delete_repo(id=repo_id, ttl=300) + await self._client.delete_repo(id=self._upstream_id(repo_id), ttl=300) except ApiError as exc: # 404: never created or already removed; 409: a deletion is already in progress. if exc.status_code not in {404, 409}: @@ -282,7 +290,7 @@ async def get_repo(self, repo_id: str) -> Repo: if cached is not None and cached[1] > now: self._repos.move_to_end(repo_id) return cached[0] - repo = await self._client.find_one(id=repo_id) + repo = await self._client.find_one(id=self._upstream_id(repo_id)) if repo is None: self._repos.pop(repo_id, None) raise LookupError(f"code.storage repository {repo_id!r} does not exist") @@ -315,7 +323,7 @@ def sandbox_remotes(self, *, repo_id: str, branch: str, subject: str) -> Sandbox ["*", ["no-push"]], ], ) - base_url = f"https://{self._organization}.code.storage/{repo_id}" + base_url = f"https://{self._organization}.code.storage/{self._upstream_id(repo_id)}" return SandboxRemotes( canonical_url=f"{base_url}.git", canonical_auth_header=_basic_auth_header(canonical_token), @@ -1702,7 +1710,7 @@ def _generate_token( payload: dict[str, Any] = { "iss": self._organization, "sub": subject, - "repo": repo_id, + "repo": self._upstream_id(repo_id), "scopes": scopes, "iat": now, "exp": now + ttl, diff --git a/src/tin_lite/runtime.py b/src/tin_lite/runtime.py index ec2a375c..b32a8ed6 100644 --- a/src/tin_lite/runtime.py +++ b/src/tin_lite/runtime.py @@ -110,6 +110,7 @@ async def build_runtime(settings: Settings) -> RuntimeServices: storage = CodeStorage( organization=settings.code_storage_org, private_key=settings.code_storage_api_key.get_secret_value(), + repo_prefix=settings.code_storage_repo_prefix, ) system_wiki = await sync_system_wiki( storage=storage, diff --git a/src/tin_lite/settings.py b/src/tin_lite/settings.py index a0f98622..01b6d3c6 100644 --- a/src/tin_lite/settings.py +++ b/src/tin_lite/settings.py @@ -201,6 +201,22 @@ class Settings(BaseSettings): twilio_webhook_url_raw: str | None = Field(default=None, alias="TIN_LITE_TWILIO_WEBHOOK_URL") code_storage_org: str = Field(default="tin", alias="TIN_LITE_CODE_STORAGE_ORG") + # Staging shares the organization with production; every repository it touches carries + # this prefix (for example "staging-"), so it never writes production's repositories. + code_storage_repo_prefix: str = Field( + default="", alias="TIN_LITE_CODE_STORAGE_REPO_PREFIX", pattern=r"^([a-z0-9][a-z0-9-]*-)?$" + ) + # "staging" refuses to boot against production's state; see docs/staging.md. + deployment_environment: Literal["production", "staging", "development"] = Field( + default="production", alias="TIN_LITE_ENVIRONMENT" + ) + # Production's identifiers, set only on staging so the guard can compare against them. + production_database_host: str | None = Field( + default=None, alias="TIN_LITE_PRODUCTION_DATABASE_HOST" + ) + production_temporal_namespace: str | None = Field( + default=None, alias="TIN_LITE_PRODUCTION_TEMPORAL_NAMESPACE" + ) e2b_template: str = Field(default="tin-lite-codex", alias="TIN_LITE_E2B_TEMPLATE") e2b_isolated_template: str | None = Field(default=None, alias="TIN_LITE_E2B_ISOLATED_TEMPLATE") e2b_browser_api_template: str = Field( @@ -353,6 +369,24 @@ def twilio_webhook_url(self) -> str: f"{self.switchboard_public_url.rstrip('/')}/webhooks/twilio/sms" ) + def _reject_production_state(self) -> None: + """Staging must never run against production's database, workflows, files or money.""" + if not self.code_storage_repo_prefix: + raise ValueError("Staging requires TIN_LITE_CODE_STORAGE_REPO_PREFIX") + if self.stripe_mode != "test": + raise ValueError("Staging requires TIN_LITE_STRIPE_MODE=test") + if not self.production_database_host or not self.production_temporal_namespace: + raise ValueError( + "Staging requires TIN_LITE_PRODUCTION_DATABASE_HOST and " + "TIN_LITE_PRODUCTION_TEMPORAL_NAMESPACE to compare against" + ) + if self.database_host == self.production_database_host or ( + self.migration_host == self.production_database_host + ): + raise ValueError("Staging is configured with production's database") + if self.temporal_namespace == self.production_temporal_namespace: + raise ValueError("Staging is configured with production's Temporal namespace") + @model_validator(mode="after") def reject_api_key_auth(self) -> Settings: if (self.codex_api_projects or self.billing_hosted_defaults_enabled) and ( @@ -372,6 +406,8 @@ def reject_api_key_auth(self) -> Settings: f"STRIPE_SECRET_KEY is not a {self.stripe_mode}-mode key; " "TIN_LITE_STRIPE_MODE and the key must match" ) + if self.deployment_environment == "staging": + self._reject_production_state() if "OPENAI_API_KEY" in os.environ or "CODEX_API_KEY" in os.environ: raise ValueError( "OPENAI_API_KEY and CODEX_API_KEY are forbidden; Luna uses its explicit " diff --git a/tests/test_secret_fences.py b/tests/test_secret_fences.py index 1bc12c41..91b1660a 100644 --- a/tests/test_secret_fences.py +++ b/tests/test_secret_fences.py @@ -52,7 +52,7 @@ def test_github_private_key_is_installed_as_host_state_not_release_state() -> No activate = (ROOT / "infra" / "activate_release.sh").read_text() assert 'openssl pkey -in "${github_private_key_path}" -check -noout' in deploy - assert "tin-lite-switchboard:/tmp/tin-lite-github-app.pem" in deploy + assert '"${instance}:/tmp/tin-lite-github-app.pem"' in deploy assert "sudo chmod 600 /tmp/tin-lite-github-app.pem" in deploy assert "install -m 640 -o root -g tinlite" in activate assert "/etc/tin-lite/github-app.pem" in activate diff --git a/tests/test_sql_requires_where.py b/tests/test_sql_requires_where.py new file mode 100644 index 00000000..dce3b2e5 --- /dev/null +++ b/tests/test_sql_requires_where.py @@ -0,0 +1,78 @@ +"""Production Postgres refuses DELETE and UPDATE without WHERE (pg_strict); local ones do not. + +Staging and CI run stock Postgres, so a statement that would fail only in production is caught +here instead: every DELETE and UPDATE in the service's SQL and migrations names a WHERE clause. +""" + +import ast +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +STATEMENT = re.compile(r"\b(DELETE\s+FROM|UPDATE)\b", re.IGNORECASE) +COMMENT = re.compile(r"--[^\n]*|/\*.*?\*/", re.DOTALL) + + +def unguarded(sql: str) -> list[str]: + sql = COMMENT.sub(" ", sql) + found = [] + for match in STATEMENT.finditer(sql): + keyword = match.group(1).upper() + rest = sql[match.end() :] + if keyword == "UPDATE": + before = sql[: match.start()].rstrip().upper() + # FOR UPDATE, FOR NO KEY UPDATE, ON CONFLICT ... DO UPDATE: not UPDATE statements. + if before.endswith(("FOR", "KEY", "DO")): + continue + if not re.match(r'\s+[\w."]+(\s+(AS\s+)?\w+)?\s+SET\b', rest, re.IGNORECASE): + continue + end = rest.find(";") + body = rest if end < 0 else rest[:end] + depth = 0 + for index, char in enumerate(body): + if char == "(": + depth += 1 + elif char == ")": + if depth == 0: # the end of an enclosing CTE + body = body[:index] + break + depth -= 1 + if not re.search(r"\bWHERE\b", body, re.IGNORECASE): + found.append(" ".join(sql[match.start() : match.start() + 100].split())) + return found + + +def service_sql() -> list[tuple[str, str]]: + sources = [] + for path in sorted((ROOT / "src").rglob("*.py")): + for node in ast.walk(ast.parse(path.read_text(), filename=str(path))): + if ( + isinstance(node, ast.Constant) + and isinstance(node.value, str) + and STATEMENT.search(node.value) + and re.search(r"\b(SET|FROM)\b", node.value) + ): + sources.append((f"{path.relative_to(ROOT)}:{node.lineno}", node.value)) + for path in sorted((ROOT / "migrations").glob("*.sql")): + sources.append((str(path.relative_to(ROOT)), path.read_text())) + return sources + + +def test_every_delete_and_update_has_a_where_clause(): + offenders = [ + f"{where}: {statement}" for where, sql in service_sql() for statement in unguarded(sql) + ] + assert not offenders, ( + "add a WHERE clause (WHERE true for a deliberate full-table change):\n" + + ("\n".join(offenders)) + ) + + +def test_the_check_recognizes_unguarded_statements(): + assert unguarded("DELETE FROM runs") + assert unguarded("UPDATE runs SET status = 'x'; SELECT 1 WHERE true") + assert unguarded("WITH gone AS (DELETE FROM runs RETURNING id) SELECT * FROM gone WHERE true") + assert not unguarded("DELETE FROM runs WHERE id = $1") + assert not unguarded("UPDATE runs r SET status = 'x' -- no; WHERE yet\nWHERE r.id = $1") + assert not unguarded("SELECT id FROM runs WHERE id = $1 FOR UPDATE") + assert not unguarded("INSERT INTO t VALUES (1) ON CONFLICT (id) DO UPDATE SET v = 1") diff --git a/tests/test_staging_isolation.py b/tests/test_staging_isolation.py new file mode 100644 index 00000000..7af4323a --- /dev/null +++ b/tests/test_staging_isolation.py @@ -0,0 +1,145 @@ +"""Staging shares the code.storage organization but never production's repositories or state.""" + +import jwt +import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import ec + +from tin_lite.code_storage import CodeStorage +from tin_lite.settings import Settings + + +def _pem() -> str: + return ( + ec.generate_private_key(ec.SECP256R1()) + .private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ) + .decode() + ) + + +class RecordingRepo: + def __init__(self, repo_id: str) -> None: + self.id = repo_id + + async def list_commits(self, **_kwargs): + return {"commits": [{"sha": "a" * 40}]} + + +class RecordingClient: + def __init__(self) -> None: + self.calls: list[tuple[str, str]] = [] + self.existing: set[str] = set() + + async def find_one(self, *, id: str): + self.calls.append(("find", id)) + return RecordingRepo(id) if id in self.existing else None + + async def create_repo(self, *, id: str, **_kwargs): + self.calls.append(("create", id)) + self.existing.add(id) + return RecordingRepo(id) + + async def delete_repo(self, *, id: str, **_kwargs): + self.calls.append(("delete", id)) + self.existing.discard(id) + + +@pytest.mark.asyncio +async def test_prefix_reaches_every_upstream_repository_name() -> None: + pem = _pem() + storage = CodeStorage(organization="tin", private_key=pem, repo_prefix="staging-") + client = RecordingClient() + storage._client = client # type: ignore[assignment] + + repo = await storage.ensure_repo("registry/workflows") + assert repo.id == "staging-registry/workflows" + assert (await storage.get_repo("registry/workflows")) is not None + await storage.delete_repo("registry/workflows") + assert {name for _, name in client.calls} == {"staging-registry/workflows"} + + remotes = storage.sandbox_remotes(repo_id="projects/p1", branch="run", subject="run") + assert remotes.canonical_url == "https://tin.code.storage/staging-projects/p1.git" + assert remotes.ephemeral_url == "https://tin.code.storage/staging-projects/p1+ephemeral.git" + token = storage._generate_token(repo_id="projects/p1", scopes=["git:read"], subject="s", ttl=60) + assert jwt.decode(token, options={"verify_signature": False})["repo"] == "staging-projects/p1" + + +@pytest.mark.asyncio +async def test_no_prefix_keeps_production_names() -> None: + storage = CodeStorage(organization="tin", private_key=_pem()) + client = RecordingClient() + storage._client = client # type: ignore[assignment] + + await storage.ensure_repo("projects/p1") + assert {name for _, name in client.calls} == {"projects/p1"} + remotes = storage.sandbox_remotes(repo_id="projects/p1", branch="run", subject="run") + assert remotes.canonical_url == "https://tin.code.storage/projects/p1.git" + + +def settings_values(monkeypatch, **overrides): + for field in Settings.model_fields.values(): + monkeypatch.delenv(field.alias, raising=False) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("CODEX_API_KEY", raising=False) + values = { + field.alias: 5432 if name.endswith("port") else "synthetic" + for name, field in Settings.model_fields.items() + if field.is_required() + } + values["CLERK_PUBLISHABLE_KEY"] = "pk_test_synthetic" + values.update(overrides) + return values + + +STAGING = { + "TIN_LITE_ENVIRONMENT": "staging", + "TIN_LITE_CODE_STORAGE_REPO_PREFIX": "staging-", + "TIN_LITE_PRODUCTION_DATABASE_HOST": "prod.db.example", + "TIN_LITE_PRODUCTION_TEMPORAL_NAMESPACE": "prod.ns", + "DATABASE_HOST": "127.0.0.1", + "MIGRATION_HOST": "127.0.0.1", + "TEMPORAL_NAMESPACE": "staging.ns", +} + + +def test_staging_settings_load(monkeypatch): + settings = Settings(_env_file=None, **settings_values(monkeypatch, **STAGING)) + assert settings.deployment_environment == "staging" + assert settings.code_storage_repo_prefix == "staging-" + + +def test_production_is_the_default_and_needs_no_prefix(monkeypatch): + settings = Settings(_env_file=None, **settings_values(monkeypatch)) + assert settings.deployment_environment == "production" + assert settings.code_storage_repo_prefix == "" + + +@pytest.mark.parametrize( + ("override", "message"), + [ + ({"TIN_LITE_CODE_STORAGE_REPO_PREFIX": ""}, "REPO_PREFIX"), + ({"TIN_LITE_STRIPE_MODE": "live"}, "STRIPE_MODE=test"), + ({"TIN_LITE_PRODUCTION_DATABASE_HOST": None}, "compare against"), + ({"DATABASE_HOST": "prod.db.example"}, "production's database"), + ({"MIGRATION_HOST": "prod.db.example"}, "production's database"), + ({"TEMPORAL_NAMESPACE": "prod.ns"}, "Temporal namespace"), + ], +) +def test_staging_refuses_production_state(monkeypatch, override, message): + values = settings_values(monkeypatch, **{**STAGING, **override}) + values = {key: value for key, value in values.items() if value is not None} + with pytest.raises(ValueError, match=message): + Settings(_env_file=None, **values) + + +@pytest.mark.parametrize("prefix", ["staging", "Staging-", "staging/", "-", "../x-"]) +def test_repo_prefix_shape(monkeypatch, prefix): + with pytest.raises(ValueError): + Settings( + _env_file=None, + **settings_values(monkeypatch, TIN_LITE_CODE_STORAGE_REPO_PREFIX=prefix), + ) From 71248565638e94efd9e6b11cbfeeaab16d5a323e Mon Sep 17 00:00:00 2001 From: Ege Demir Date: Tue, 13 Oct 2026 12:07:50 +0000 Subject: [PATCH 2/4] Document the staging deployment Co-Authored-By: Claude Opus 5.5 --- docs/README.md | 1 + docs/staging.md | 82 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 docs/staging.md diff --git a/docs/README.md b/docs/README.md index 9b3c2d4f..764c4107 100644 --- a/docs/README.md +++ b/docs/README.md @@ -18,6 +18,7 @@ remain authoritative; older contract versions are retained for compatibility. ## Self-hosting and operations - [Engineering reference](engineering.md): configuration, local commands and service APIs. +- [Staging](staging.md): deploying a change end to end before production, and what it shares. - [Optional billing](self-hosted-billing.md), [billing coverage](workflow-billing-coverage.md) and [model accounting](model-service-accounting.md). - [Authentication and integrations](auth-and-integrations.md), diff --git a/docs/staging.md b/docs/staging.md new file mode 100644 index 00000000..fba43ba8 --- /dev/null +++ b/docs/staging.md @@ -0,0 +1,82 @@ +# Staging + +Hosted Tin has a staging deployment at `https://app.staging.tin.computer` for running a +change end to end, as production would, before it reaches `app.tin.computer`. Local +development does not reproduce production's providers, callbacks or release process; +staging does, with its own state. + +A deploy to staging is not a production deployment. Production stays a separate, +explicitly authorized step. + +## Deploying + +- Every push to `main` deploys to staging (`.github/workflows/deploy-staging.yml`). +- Any branch, tag or SHA deploys on request: **Actions → deploy-staging → Run workflow**, + or `gh workflow run deploy-staging -f ref=`. One deploy runs at a time; a newer + request waits. A branch deploy stays until the next one, so say in the team channel + when you take staging for a branch. +- From a workstation: `TIN_LITE_DEPLOY_TARGET=staging TIN_LITE_GCP_PROJECT= + bash infra/deploy.sh`. + +A staging deploy installs the host scripts, unpacks the release, runs `tin-lite migrate` +from the new release and only then switches `current` and restarts. Its database is +long-lived and never reset, so it applies every migration and catalog change in the same +order production will. A release that would crash production's boot (a changed executor, +a reused built-in ID) crashes staging's first. + +The workflow authenticates to Google Cloud with workload identity. Google trusts only +jobs from this repository in the `staging` GitHub environment, and the deployer can log +in to the staging host only. The environment's variables name the provider, service +account, project and static IP. + +## What is separate and what is shared + +| | Staging | +|---|---| +| Host | Its own VM, installed by the same `install_switchboard.sh` | +| Postgres | Local to the staging host (`install_staging_postgres.sh`) | +| Temporal | Its own namespace | +| code.storage | Production's organization, its own key, every repository prefixed `staging-` | +| Clerk | A development instance | +| GitHub | Its own GitHub App and founder-account OAuth App | +| Stripe | Test mode, with its own webhook endpoint | +| Keyword Planner (gak) | Its own service on the staging host | +| Models, E2B, data providers | Production's accounts and keys | +| Google and X OAuth | Production's clients, once the staging callbacks are registered | + +`TIN_LITE_ENVIRONMENT=staging` refuses to boot without `TIN_LITE_CODE_STORAGE_REPO_PREFIX`, +outside Stripe test mode, or when the database host or Temporal namespace equals +production's (`TIN_LITE_PRODUCTION_DATABASE_HOST`, `TIN_LITE_PRODUCTION_TEMPORAL_NAMESPACE`). + +The prefix applies wherever Tin names a repository upstream: lookups, creation, deletion, +sandbox git remotes and scoped tokens. Stored references keep the unprefixed ID, so +staging writes `staging-registry/workflows` and `staging-projects/` and never +production's repositories. The key is still organization-wide; rotate it alone if the +staging host is compromised. + +## Where its configuration lives + +On the staging host, readable by root and the service group only: + +- `/etc/tin-lite/release.env`: secrets and settings, copied into each release as `.env`. + Nothing is uploaded from a workstation. +- `/etc/tin-lite/clerk.env` and `/etc/tin-lite/runtime.env`: the service environment. + `install_switchboard.sh` regenerates the hosts and proxy keys in `runtime.env` and keeps + every other line. +- `/etc/tin-lite/github-app.pem` and `/etc/tin-lite/postgres-password`. + +## How it differs from production + +- Production's Postgres refuses `DELETE` and `UPDATE` without `WHERE` (`pg_strict`); + staging's does not. `tests/test_sql_requires_where.py` catches such statements in CI. +- The database is on the same host: no network latency, failover or connection drops. + Its settings otherwise follow production's (version, `max_connections`, time zone, + memory, a role that is not a superuser). +- Clerk's development instance keeps sessions differently and is shared with the Tin + website's staging site, so its sign-in paths point there. Tin's own sign-in works; an + MCP connection's sign-in and consent use the instance's hosted pages. +- Real model, sandbox and data-provider calls cost real money. Keep test projects few and + their spending limits low; pause schedules you are not testing. + +Fixture tests prove the contract and staging proves the release; production still needs +its one real run after a production deploy. From 7f73aef0f313951ea484522c3eb88c7fe6c46d99 Mon Sep 17 00:00:00 2001 From: Ege Demir Date: Tue, 13 Oct 2026 12:10:35 +0000 Subject: [PATCH 3/4] Compare staging's hosts case-insensitively and give staging deploys time to boot Co-Authored-By: Claude Opus 5.5 --- infra/deploy.sh | 7 +++++-- src/tin_lite/settings.py | 11 +++++++---- tests/test_staging_isolation.py | 1 + 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/infra/deploy.sh b/infra/deploy.sh index 8d5b711a..6738f27f 100755 --- a/infra/deploy.sh +++ b/infra/deploy.sh @@ -13,6 +13,7 @@ case "${target}" in default_host=app.tin.computer upload_env=1 run_migrations="${TIN_LITE_RUN_MIGRATIONS:-0}" + health_retries=20 ;; staging) instance=tin-lite-staging @@ -20,6 +21,8 @@ case "${target}" in default_host=app.staging.tin.computer upload_env=0 run_migrations=1 + # A first boot publishes the whole catalog and a restart drains running workflows. + health_retries=100 ;; *) echo "TIN_LITE_DEPLOY_TARGET must be prod or staging" >&2 @@ -112,8 +115,8 @@ gcloud compute ssh "${instance}" \ ${move_env}sudo env TIN_LITE_RUN_MIGRATIONS=${run_migrations} bash /tmp/activate_release.sh ${release_sha}" public_host="tin-lite-switchboard.${static_ip//./-}.sslip.io" -curl --http1.1 --fail --silent --show-error --retry 20 --retry-all-errors --retry-delay 3 \ +curl --http1.1 --fail --silent --show-error --retry "${health_retries}" --retry-all-errors --retry-delay 3 \ "https://${public_host}/healthz" -curl --http1.1 --fail --silent --show-error --retry 20 --retry-all-errors --retry-delay 3 \ +curl --http1.1 --fail --silent --show-error --retry "${health_retries}" --retry-all-errors --retry-delay 3 \ "https://${product_host}/healthz" diff --git a/src/tin_lite/settings.py b/src/tin_lite/settings.py index 01b6d3c6..e7a49c1b 100644 --- a/src/tin_lite/settings.py +++ b/src/tin_lite/settings.py @@ -380,11 +380,14 @@ def _reject_production_state(self) -> None: "Staging requires TIN_LITE_PRODUCTION_DATABASE_HOST and " "TIN_LITE_PRODUCTION_TEMPORAL_NAMESPACE to compare against" ) - if self.database_host == self.production_database_host or ( - self.migration_host == self.production_database_host - ): + + def same(left: str, right: str) -> bool: + return left.strip().rstrip(".").lower() == right.strip().rstrip(".").lower() + + production_host = self.production_database_host + if same(self.database_host, production_host) or same(self.migration_host, production_host): raise ValueError("Staging is configured with production's database") - if self.temporal_namespace == self.production_temporal_namespace: + if same(self.temporal_namespace, self.production_temporal_namespace): raise ValueError("Staging is configured with production's Temporal namespace") @model_validator(mode="after") diff --git a/tests/test_staging_isolation.py b/tests/test_staging_isolation.py index 7af4323a..b7572ec3 100644 --- a/tests/test_staging_isolation.py +++ b/tests/test_staging_isolation.py @@ -126,6 +126,7 @@ def test_production_is_the_default_and_needs_no_prefix(monkeypatch): ({"TIN_LITE_PRODUCTION_DATABASE_HOST": None}, "compare against"), ({"DATABASE_HOST": "prod.db.example"}, "production's database"), ({"MIGRATION_HOST": "prod.db.example"}, "production's database"), + ({"DATABASE_HOST": "PROD.db.example."}, "production's database"), ({"TEMPORAL_NAMESPACE": "prod.ns"}, "Temporal namespace"), ], ) From 98789ad9c55e480c82ec6d0caf5beef1128e55cb Mon Sep 17 00:00:00 2001 From: Ege Demir Date: Tue, 13 Oct 2026 12:22:05 +0000 Subject: [PATCH 4/4] Give the runtime test's settings the code.storage prefix Co-Authored-By: Claude Opus 5.5 --- tests/test_runtime.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_runtime.py b/tests/test_runtime.py index 31b6cf03..55d9deb3 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -52,6 +52,7 @@ def register(*args, **kwargs): billing_enabled=billing_enabled, runtime_dsn="postgresql://runtime", code_storage_org="tin", + code_storage_repo_prefix="", code_storage_api_key=secret, e2b_api_key=secret, e2b_template="tin-lite-codex",