diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml new file mode 100644 index 00000000..150628a9 --- /dev/null +++ b/.github/workflows/deploy-staging.yml @@ -0,0 +1,48 @@ +name: deploy-staging + +# Staging runs every merge to main, and any branch on request. Production deploys stay manual. +on: + push: + branches: [main] + workflow_dispatch: + inputs: + ref: + description: Branch, tag or SHA to deploy to staging + required: true + default: main + +permissions: + contents: read + +# One release at a time; a newer request waits instead of interrupting a deploy midway. +concurrency: + group: deploy-staging + cancel-in-progress: false + +jobs: + deploy: + runs-on: ubuntu-latest + timeout-minutes: 30 + # The staging environment is the only identity Google Cloud trusts for this deployer. + environment: + name: staging + url: https://app.staging.tin.computer + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.ref || github.sha }} + persist-credentials: false + - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 + with: + workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.GCP_DEPLOY_SERVICE_ACCOUNT }} + - uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 + - name: Deploy + env: + TIN_LITE_DEPLOY_TARGET: staging + TIN_LITE_GCP_PROJECT: ${{ vars.GCP_PROJECT }} + TIN_LITE_STATIC_IP: ${{ vars.STAGING_STATIC_IP }} + run: bash infra/deploy.sh diff --git a/docs/README.md b/docs/README.md index 9b3c2d4f..764c4107 100644 --- a/docs/README.md +++ b/docs/README.md @@ -18,6 +18,7 @@ remain authoritative; older contract versions are retained for compatibility. ## Self-hosting and operations - [Engineering reference](engineering.md): configuration, local commands and service APIs. +- [Staging](staging.md): deploying a change end to end before production, and what it shares. - [Optional billing](self-hosted-billing.md), [billing coverage](workflow-billing-coverage.md) and [model accounting](model-service-accounting.md). - [Authentication and integrations](auth-and-integrations.md), diff --git a/docs/staging.md b/docs/staging.md new file mode 100644 index 00000000..fba43ba8 --- /dev/null +++ b/docs/staging.md @@ -0,0 +1,82 @@ +# Staging + +Hosted Tin has a staging deployment at `https://app.staging.tin.computer` for running a +change end to end, as production would, before it reaches `app.tin.computer`. Local +development does not reproduce production's providers, callbacks or release process; +staging does, with its own state. + +A deploy to staging is not a production deployment. Production stays a separate, +explicitly authorized step. + +## Deploying + +- Every push to `main` deploys to staging (`.github/workflows/deploy-staging.yml`). +- Any branch, tag or SHA deploys on request: **Actions → deploy-staging → Run workflow**, + or `gh workflow run deploy-staging -f ref=`. One deploy runs at a time; a newer + request waits. A branch deploy stays until the next one, so say in the team channel + when you take staging for a branch. +- From a workstation: `TIN_LITE_DEPLOY_TARGET=staging TIN_LITE_GCP_PROJECT= + bash infra/deploy.sh`. + +A staging deploy installs the host scripts, unpacks the release, runs `tin-lite migrate` +from the new release and only then switches `current` and restarts. Its database is +long-lived and never reset, so it applies every migration and catalog change in the same +order production will. A release that would crash production's boot (a changed executor, +a reused built-in ID) crashes staging's first. + +The workflow authenticates to Google Cloud with workload identity. Google trusts only +jobs from this repository in the `staging` GitHub environment, and the deployer can log +in to the staging host only. The environment's variables name the provider, service +account, project and static IP. + +## What is separate and what is shared + +| | Staging | +|---|---| +| Host | Its own VM, installed by the same `install_switchboard.sh` | +| Postgres | Local to the staging host (`install_staging_postgres.sh`) | +| Temporal | Its own namespace | +| code.storage | Production's organization, its own key, every repository prefixed `staging-` | +| Clerk | A development instance | +| GitHub | Its own GitHub App and founder-account OAuth App | +| Stripe | Test mode, with its own webhook endpoint | +| Keyword Planner (gak) | Its own service on the staging host | +| Models, E2B, data providers | Production's accounts and keys | +| Google and X OAuth | Production's clients, once the staging callbacks are registered | + +`TIN_LITE_ENVIRONMENT=staging` refuses to boot without `TIN_LITE_CODE_STORAGE_REPO_PREFIX`, +outside Stripe test mode, or when the database host or Temporal namespace equals +production's (`TIN_LITE_PRODUCTION_DATABASE_HOST`, `TIN_LITE_PRODUCTION_TEMPORAL_NAMESPACE`). + +The prefix applies wherever Tin names a repository upstream: lookups, creation, deletion, +sandbox git remotes and scoped tokens. Stored references keep the unprefixed ID, so +staging writes `staging-registry/workflows` and `staging-projects/` and never +production's repositories. The key is still organization-wide; rotate it alone if the +staging host is compromised. + +## Where its configuration lives + +On the staging host, readable by root and the service group only: + +- `/etc/tin-lite/release.env`: secrets and settings, copied into each release as `.env`. + Nothing is uploaded from a workstation. +- `/etc/tin-lite/clerk.env` and `/etc/tin-lite/runtime.env`: the service environment. + `install_switchboard.sh` regenerates the hosts and proxy keys in `runtime.env` and keeps + every other line. +- `/etc/tin-lite/github-app.pem` and `/etc/tin-lite/postgres-password`. + +## How it differs from production + +- Production's Postgres refuses `DELETE` and `UPDATE` without `WHERE` (`pg_strict`); + staging's does not. `tests/test_sql_requires_where.py` catches such statements in CI. +- The database is on the same host: no network latency, failover or connection drops. + Its settings otherwise follow production's (version, `max_connections`, time zone, + memory, a role that is not a superuser). +- Clerk's development instance keeps sessions differently and is shared with the Tin + website's staging site, so its sign-in paths point there. Tin's own sign-in works; an + MCP connection's sign-in and consent use the instance's hosted pages. +- Real model, sandbox and data-provider calls cost real money. Keep test projects few and + their spending limits low; pause schedules you are not testing. + +Fixture tests prove the contract and staging proves the release; production still needs +its one real run after a production deploy. diff --git a/infra/activate_release.sh b/infra/activate_release.sh index 9408a414..ceb1b190 100755 --- a/infra/activate_release.sh +++ b/infra/activate_release.sh @@ -26,12 +26,29 @@ if [[ -f /tmp/tin-lite-github-app.pem ]]; then 's#^TIN_LITE_GITHUB_APP_PRIVATE_KEY_PATH=.*#TIN_LITE_GITHUB_APP_PRIVATE_KEY_PATH=/etc/tin-lite/github-app.pem#' \ /tmp/tin-lite.env fi -install -m 640 -o tinlite -g tinlite /tmp/tin-lite.env "${release_dir}/.env" +# An uploaded .env wins; otherwise the release env kept on the host (staging) is used. +release_env=/tmp/tin-lite.env +if [[ ! -f "${release_env}" ]]; then + release_env=/etc/tin-lite/release.env +fi +if [[ ! -f "${release_env}" ]]; then + echo "no release env: upload .env or keep /etc/tin-lite/release.env on the host" >&2 + exit 66 +fi +install -m 640 -o tinlite -g tinlite "${release_env}" "${release_dir}/.env" chown -R tinlite:tinlite "${release_dir}" sudo -u tinlite env UV_CACHE_DIR=/var/cache/tin-lite-uv \ /usr/local/bin/uv sync --frozen --no-dev --directory "${release_dir}" +# Migrations run from the new release, under the service env, before it goes live. +if [[ "${TIN_LITE_RUN_MIGRATIONS:-0}" == "1" ]]; then + systemd-run --wait --pipe --collect -p User=tinlite -p WorkingDirectory="${release_dir}" \ + -p EnvironmentFile=/etc/tin-lite/clerk.env -p EnvironmentFile=/etc/tin-lite/runtime.env \ + -E UV_CACHE_DIR=/var/cache/tin-lite-uv \ + /usr/local/bin/uv run --frozen --no-dev tin-lite migrate +fi + ln -sfn "${release_dir}" /opt/tin-lite/current systemctl enable tin-lite-switchboard systemctl restart tin-lite-switchboard diff --git a/infra/deploy.sh b/infra/deploy.sh index bb2fa189..6738f27f 100755 --- a/infra/deploy.sh +++ b/infra/deploy.sh @@ -3,22 +3,53 @@ set -euo pipefail : "${TIN_LITE_GCP_PROJECT:?Set TIN_LITE_GCP_PROJECT to your deployment project}" +# prod: the release env is uploaded from a local .env. staging: it stays on the host in +# /etc/tin-lite/release.env, and migrations run before the release goes live. +target="${TIN_LITE_DEPLOY_TARGET:-prod}" +case "${target}" in + prod) + instance=tin-lite-switchboard + address=tin-lite-switchboard-ip + default_host=app.tin.computer + upload_env=1 + run_migrations="${TIN_LITE_RUN_MIGRATIONS:-0}" + health_retries=20 + ;; + staging) + instance=tin-lite-staging + address=tin-lite-staging-ip + default_host=app.staging.tin.computer + upload_env=0 + run_migrations=1 + # A first boot publishes the whole catalog and a restart drains running workflows. + health_retries=100 + ;; + *) + echo "TIN_LITE_DEPLOY_TARGET must be prod or staging" >&2 + exit 64 + ;; +esac +zone=us-central1-a +region=us-central1 + root="$(git rev-parse --show-toplevel)" cd "${root}" if [[ -n "$(git status --porcelain --untracked-files=no)" ]]; then echo "tracked worktree changes must be committed before deployment" >&2 exit 65 fi -if [[ ! -f .env ]]; then - echo "local .env is required" >&2 - exit 66 -fi -if rg -q '^OPENAI_API_KEY=' .env; then - echo "OPENAI_API_KEY is forbidden" >&2 - exit 67 +github_private_key_path="" +if [[ "${upload_env}" == "1" ]]; then + if [[ ! -f .env ]]; then + echo "local .env is required" >&2 + exit 66 + fi + if grep -q '^OPENAI_API_KEY=' .env; then + echo "OPENAI_API_KEY is forbidden" >&2 + exit 67 + fi + github_private_key_path="$(sed -n 's/^TIN_LITE_GITHUB_APP_PRIVATE_KEY_PATH=//p' .env | tail -n 1)" fi - -github_private_key_path="$(sed -n 's/^TIN_LITE_GITHUB_APP_PRIVATE_KEY_PATH=//p' .env | tail -n 1)" if [[ -n "${github_private_key_path}" ]]; then if [[ ! -f "${github_private_key_path}" ]]; then echo "TIN_LITE_GITHUB_APP_PRIVATE_KEY_PATH must name a readable local file" >&2 @@ -35,45 +66,57 @@ temporary_dir="$(mktemp -d)" trap 'rm -rf "${temporary_dir}"' EXIT git archive --format=tar.gz --output="${temporary_dir}/tin-lite-release.tar.gz" HEAD +release_files=("${temporary_dir}/tin-lite-release.tar.gz") +if [[ "${upload_env}" == "1" ]]; then + release_files+=(.env) +fi gcloud compute scp \ - "${temporary_dir}/tin-lite-release.tar.gz" \ - .env \ + "${release_files[@]}" \ infra/install_switchboard.sh \ infra/install_forward_proxy.sh \ infra/proxy_policy.py \ src/tin_lite/proxy_grants.py \ infra/activate_release.sh \ - tin-lite-switchboard:/tmp/ \ - --project "${TIN_LITE_GCP_PROJECT}" --zone us-central1-a --quiet + "${instance}:/tmp/" \ + --project "${TIN_LITE_GCP_PROJECT}" --zone "${zone}" --quiet if [[ -n "${github_private_key_path}" ]]; then gcloud compute scp \ "${github_private_key_path}" \ - tin-lite-switchboard:/tmp/tin-lite-github-app.pem \ - --project "${TIN_LITE_GCP_PROJECT}" --zone us-central1-a --quiet - gcloud compute ssh tin-lite-switchboard \ - --project "${TIN_LITE_GCP_PROJECT}" --zone us-central1-a --quiet \ + "${instance}:/tmp/tin-lite-github-app.pem" \ + --project "${TIN_LITE_GCP_PROJECT}" --zone "${zone}" --quiet + gcloud compute ssh "${instance}" \ + --project "${TIN_LITE_GCP_PROJECT}" --zone "${zone}" --quiet \ --command="sudo chown root:root /tmp/tin-lite-github-app.pem && \ sudo chmod 600 /tmp/tin-lite-github-app.pem" fi -static_ip="$(gcloud compute addresses describe tin-lite-switchboard-ip \ - --project "${TIN_LITE_GCP_PROJECT}" --region us-central1 --format='value(address)')" -product_host="${TIN_LITE_PRODUCT_HOST:-app.tin.computer}" +static_ip="${TIN_LITE_STATIC_IP:-}" +if [[ -z "${static_ip}" ]]; then + static_ip="$(gcloud compute addresses describe "${address}" \ + --project "${TIN_LITE_GCP_PROJECT}" --region "${region}" --format='value(address)')" +fi +if [[ ! "${static_ip}" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "the ${target} static IP is not an IPv4 address" >&2 + exit 64 +fi +product_host="${TIN_LITE_PRODUCT_HOST:-${default_host}}" if [[ ! "${product_host}" =~ ^[a-zA-Z0-9.-]+$ ]]; then echo "TIN_LITE_PRODUCT_HOST must be a DNS hostname" >&2 exit 64 fi -gcloud compute ssh tin-lite-switchboard \ - --project "${TIN_LITE_GCP_PROJECT}" --zone us-central1-a --quiet \ +move_env="" +if [[ "${upload_env}" == "1" ]]; then + move_env="sudo mv /tmp/.env /tmp/tin-lite.env && " +fi +gcloud compute ssh "${instance}" \ + --project "${TIN_LITE_GCP_PROJECT}" --zone "${zone}" --quiet \ --command="sudo env TIN_LITE_STATIC_IP=${static_ip} TIN_LITE_PRODUCT_HOST=${product_host} bash /tmp/install_switchboard.sh && \ -sudo mv /tmp/.env /tmp/tin-lite.env && \ -sudo bash /tmp/activate_release.sh ${release_sha}" +${move_env}sudo env TIN_LITE_RUN_MIGRATIONS=${run_migrations} bash /tmp/activate_release.sh ${release_sha}" public_host="tin-lite-switchboard.${static_ip//./-}.sslip.io" -curl --http1.1 --fail --silent --show-error --retry 20 --retry-all-errors --retry-delay 3 \ +curl --http1.1 --fail --silent --show-error --retry "${health_retries}" --retry-all-errors --retry-delay 3 \ "https://${public_host}/healthz" -product_host="${TIN_LITE_PRODUCT_HOST:-app.tin.computer}" -curl --http1.1 --fail --silent --show-error --retry 20 --retry-all-errors --retry-delay 3 \ +curl --http1.1 --fail --silent --show-error --retry "${health_retries}" --retry-all-errors --retry-delay 3 \ "https://${product_host}/healthz" diff --git a/infra/install_staging_postgres.sh b/infra/install_staging_postgres.sh new file mode 100644 index 00000000..5c0d5fe0 --- /dev/null +++ b/infra/install_staging_postgres.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Staging's database: stock Postgres on the staging host, set up to behave like production's +# managed Postgres where Tin can tell the difference. Production's pg_strict (no DELETE or +# UPDATE without WHERE) is not available here; tests/test_sql_requires_where.py covers it. +set -euo pipefail + +if [[ "${EUID}" -ne 0 ]]; then + echo "run as root" >&2 + exit 64 +fi + +major="${TIN_LITE_POSTGRES_MAJOR:-18}" +# Production's exact build when the archive still has it; otherwise the latest minor. +version="${TIN_LITE_POSTGRES_VERSION:-18.6-1.pgdg12+2}" +database=tin +role=tin_app +password_file=/etc/tin-lite/postgres-password + +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get install -y ca-certificates curl gnupg postgresql-common +if [[ ! -f /etc/apt/sources.list.d/pgdg.list ]]; then + install -d /usr/share/postgresql-common/pgdg + curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc + echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "${VERSION_CODENAME}")-pgdg main" \ + > /etc/apt/sources.list.d/pgdg.list + apt-get update +fi +if apt-cache madison "postgresql-${major}" | awk -F'|' '{gsub(/ /, "", $2); print $2}' | + grep -qxF "${version}"; then + apt-get install -y "postgresql-${major}=${version}" + apt-mark hold "postgresql-${major}" +else + echo "postgresql-${major} ${version} is not available; installing the latest ${major}.x" >&2 + apt-get install -y "postgresql-${major}" +fi + +install -d -m 755 "/etc/postgresql/${major}/main/conf.d" +cat > "/etc/postgresql/${major}/main/conf.d/tin-staging.conf" <<'EOF' +# Production's values where they shape behavior Tin can observe. +listen_addresses = 'localhost' +max_connections = 25 +ssl = on +timezone = 'UTC' +log_timezone = 'UTC' +work_mem = '2MB' +maintenance_work_mem = '16MB' +shared_buffers = '64MB' +idle_in_transaction_session_timeout = 0 +random_page_cost = 1.1 +effective_io_concurrency = 32 +default_toast_compression = 'lz4' +EOF +systemctl enable "postgresql@${major}-main" +systemctl restart "postgresql@${major}-main" + +install -d -m 750 -o root -g tinlite /etc/tin-lite 2>/dev/null || install -d -m 750 /etc/tin-lite +if [[ ! -s "${password_file}" ]]; then + umask 077 + openssl rand -hex 32 > "${password_file}" +fi +chmod 600 "${password_file}" +password="$(cat "${password_file}")" + +# Like production's application role: no superuser, cannot create databases or roles. +sudo -u postgres psql -v ON_ERROR_STOP=1 -q < None: storage = CodeStorage( organization=settings.code_storage_org, private_key=settings.code_storage_api_key.get_secret_value(), + repo_prefix=settings.code_storage_repo_prefix, ) await storage.ensure_repo(repo_id) database = Database(settings.runtime_dsn) @@ -67,6 +68,7 @@ async def _sync_builtins() -> None: storage = CodeStorage( organization=settings.code_storage_org, private_key=settings.code_storage_api_key.get_secret_value(), + repo_prefix=settings.code_storage_repo_prefix, ) database = Database(settings.runtime_dsn) await database.connect() diff --git a/src/tin_lite/code_storage.py b/src/tin_lite/code_storage.py index a356c2d3..bcc8c7ec 100644 --- a/src/tin_lite/code_storage.py +++ b/src/tin_lite/code_storage.py @@ -181,11 +181,17 @@ class CodeStorage: _http: httpx.AsyncClient | None = None _http_loop: asyncio.AbstractEventLoop | None = None - def __init__(self, *, organization: str, private_key: str) -> None: + def __init__(self, *, organization: str, private_key: str, repo_prefix: str = "") -> None: self._organization = organization self._private_key = private_key + # A deployment sharing an organization (staging) keeps its repositories apart by name. + # Callers and stored references always use the unprefixed ID. + self._repo_prefix = repo_prefix self._client = GitStorage({"name": organization, "key": private_key}) + def _upstream_id(self, repo_id: str) -> str: + return f"{self._repo_prefix}{repo_id}" + # Per-instance and created on first use, so every CodeStorage has its own caches. @functools.cached_property def _pinned(self) -> _PinnedReadCache: @@ -221,7 +227,7 @@ async def ensure_repo( initial_readme: str = "# Project state\n\nManaged by the Tin Lite switchboard.\n", ) -> Repo: try: - repo = await self._client.find_one(id=repo_id) + repo = await self._client.find_one(id=self._upstream_id(repo_id)) except ApiError as exc: # An empty organization currently answers 403 for a repo-scoped lookup before create. if exc.status_code != 403: @@ -229,11 +235,13 @@ async def ensure_repo( repo = None if repo is None: try: - repo = await self._client.create_repo(id=repo_id, default_branch="main", ttl=300) + repo = await self._client.create_repo( + id=self._upstream_id(repo_id), default_branch="main", ttl=300 + ) except ApiError as exc: if exc.status_code != 409: raise - repo = await self._client.find_one(id=repo_id) + repo = await self._client.find_one(id=self._upstream_id(repo_id)) if repo is None: raise if await self.head_sha(repo, "main") is None: @@ -256,7 +264,7 @@ async def delete_repo(self, repo_id: str) -> bool: """True when the repository is gone after this call; upstream cleanup is asynchronous.""" self._forget_repo(repo_id) try: - await self._client.delete_repo(id=repo_id, ttl=300) + await self._client.delete_repo(id=self._upstream_id(repo_id), ttl=300) except ApiError as exc: # 404: never created or already removed; 409: a deletion is already in progress. if exc.status_code not in {404, 409}: @@ -282,7 +290,7 @@ async def get_repo(self, repo_id: str) -> Repo: if cached is not None and cached[1] > now: self._repos.move_to_end(repo_id) return cached[0] - repo = await self._client.find_one(id=repo_id) + repo = await self._client.find_one(id=self._upstream_id(repo_id)) if repo is None: self._repos.pop(repo_id, None) raise LookupError(f"code.storage repository {repo_id!r} does not exist") @@ -315,7 +323,7 @@ def sandbox_remotes(self, *, repo_id: str, branch: str, subject: str) -> Sandbox ["*", ["no-push"]], ], ) - base_url = f"https://{self._organization}.code.storage/{repo_id}" + base_url = f"https://{self._organization}.code.storage/{self._upstream_id(repo_id)}" return SandboxRemotes( canonical_url=f"{base_url}.git", canonical_auth_header=_basic_auth_header(canonical_token), @@ -1702,7 +1710,7 @@ def _generate_token( payload: dict[str, Any] = { "iss": self._organization, "sub": subject, - "repo": repo_id, + "repo": self._upstream_id(repo_id), "scopes": scopes, "iat": now, "exp": now + ttl, diff --git a/src/tin_lite/runtime.py b/src/tin_lite/runtime.py index ec2a375c..b32a8ed6 100644 --- a/src/tin_lite/runtime.py +++ b/src/tin_lite/runtime.py @@ -110,6 +110,7 @@ async def build_runtime(settings: Settings) -> RuntimeServices: storage = CodeStorage( organization=settings.code_storage_org, private_key=settings.code_storage_api_key.get_secret_value(), + repo_prefix=settings.code_storage_repo_prefix, ) system_wiki = await sync_system_wiki( storage=storage, diff --git a/src/tin_lite/settings.py b/src/tin_lite/settings.py index a0f98622..e7a49c1b 100644 --- a/src/tin_lite/settings.py +++ b/src/tin_lite/settings.py @@ -201,6 +201,22 @@ class Settings(BaseSettings): twilio_webhook_url_raw: str | None = Field(default=None, alias="TIN_LITE_TWILIO_WEBHOOK_URL") code_storage_org: str = Field(default="tin", alias="TIN_LITE_CODE_STORAGE_ORG") + # Staging shares the organization with production; every repository it touches carries + # this prefix (for example "staging-"), so it never writes production's repositories. + code_storage_repo_prefix: str = Field( + default="", alias="TIN_LITE_CODE_STORAGE_REPO_PREFIX", pattern=r"^([a-z0-9][a-z0-9-]*-)?$" + ) + # "staging" refuses to boot against production's state; see docs/staging.md. + deployment_environment: Literal["production", "staging", "development"] = Field( + default="production", alias="TIN_LITE_ENVIRONMENT" + ) + # Production's identifiers, set only on staging so the guard can compare against them. + production_database_host: str | None = Field( + default=None, alias="TIN_LITE_PRODUCTION_DATABASE_HOST" + ) + production_temporal_namespace: str | None = Field( + default=None, alias="TIN_LITE_PRODUCTION_TEMPORAL_NAMESPACE" + ) e2b_template: str = Field(default="tin-lite-codex", alias="TIN_LITE_E2B_TEMPLATE") e2b_isolated_template: str | None = Field(default=None, alias="TIN_LITE_E2B_ISOLATED_TEMPLATE") e2b_browser_api_template: str = Field( @@ -353,6 +369,27 @@ def twilio_webhook_url(self) -> str: f"{self.switchboard_public_url.rstrip('/')}/webhooks/twilio/sms" ) + def _reject_production_state(self) -> None: + """Staging must never run against production's database, workflows, files or money.""" + if not self.code_storage_repo_prefix: + raise ValueError("Staging requires TIN_LITE_CODE_STORAGE_REPO_PREFIX") + if self.stripe_mode != "test": + raise ValueError("Staging requires TIN_LITE_STRIPE_MODE=test") + if not self.production_database_host or not self.production_temporal_namespace: + raise ValueError( + "Staging requires TIN_LITE_PRODUCTION_DATABASE_HOST and " + "TIN_LITE_PRODUCTION_TEMPORAL_NAMESPACE to compare against" + ) + + def same(left: str, right: str) -> bool: + return left.strip().rstrip(".").lower() == right.strip().rstrip(".").lower() + + production_host = self.production_database_host + if same(self.database_host, production_host) or same(self.migration_host, production_host): + raise ValueError("Staging is configured with production's database") + if same(self.temporal_namespace, self.production_temporal_namespace): + raise ValueError("Staging is configured with production's Temporal namespace") + @model_validator(mode="after") def reject_api_key_auth(self) -> Settings: if (self.codex_api_projects or self.billing_hosted_defaults_enabled) and ( @@ -372,6 +409,8 @@ def reject_api_key_auth(self) -> Settings: f"STRIPE_SECRET_KEY is not a {self.stripe_mode}-mode key; " "TIN_LITE_STRIPE_MODE and the key must match" ) + if self.deployment_environment == "staging": + self._reject_production_state() if "OPENAI_API_KEY" in os.environ or "CODEX_API_KEY" in os.environ: raise ValueError( "OPENAI_API_KEY and CODEX_API_KEY are forbidden; Luna uses its explicit " diff --git a/tests/test_runtime.py b/tests/test_runtime.py index 31b6cf03..55d9deb3 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -52,6 +52,7 @@ def register(*args, **kwargs): billing_enabled=billing_enabled, runtime_dsn="postgresql://runtime", code_storage_org="tin", + code_storage_repo_prefix="", code_storage_api_key=secret, e2b_api_key=secret, e2b_template="tin-lite-codex", diff --git a/tests/test_secret_fences.py b/tests/test_secret_fences.py index 1bc12c41..91b1660a 100644 --- a/tests/test_secret_fences.py +++ b/tests/test_secret_fences.py @@ -52,7 +52,7 @@ def test_github_private_key_is_installed_as_host_state_not_release_state() -> No activate = (ROOT / "infra" / "activate_release.sh").read_text() assert 'openssl pkey -in "${github_private_key_path}" -check -noout' in deploy - assert "tin-lite-switchboard:/tmp/tin-lite-github-app.pem" in deploy + assert '"${instance}:/tmp/tin-lite-github-app.pem"' in deploy assert "sudo chmod 600 /tmp/tin-lite-github-app.pem" in deploy assert "install -m 640 -o root -g tinlite" in activate assert "/etc/tin-lite/github-app.pem" in activate diff --git a/tests/test_sql_requires_where.py b/tests/test_sql_requires_where.py new file mode 100644 index 00000000..dce3b2e5 --- /dev/null +++ b/tests/test_sql_requires_where.py @@ -0,0 +1,78 @@ +"""Production Postgres refuses DELETE and UPDATE without WHERE (pg_strict); local ones do not. + +Staging and CI run stock Postgres, so a statement that would fail only in production is caught +here instead: every DELETE and UPDATE in the service's SQL and migrations names a WHERE clause. +""" + +import ast +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +STATEMENT = re.compile(r"\b(DELETE\s+FROM|UPDATE)\b", re.IGNORECASE) +COMMENT = re.compile(r"--[^\n]*|/\*.*?\*/", re.DOTALL) + + +def unguarded(sql: str) -> list[str]: + sql = COMMENT.sub(" ", sql) + found = [] + for match in STATEMENT.finditer(sql): + keyword = match.group(1).upper() + rest = sql[match.end() :] + if keyword == "UPDATE": + before = sql[: match.start()].rstrip().upper() + # FOR UPDATE, FOR NO KEY UPDATE, ON CONFLICT ... DO UPDATE: not UPDATE statements. + if before.endswith(("FOR", "KEY", "DO")): + continue + if not re.match(r'\s+[\w."]+(\s+(AS\s+)?\w+)?\s+SET\b', rest, re.IGNORECASE): + continue + end = rest.find(";") + body = rest if end < 0 else rest[:end] + depth = 0 + for index, char in enumerate(body): + if char == "(": + depth += 1 + elif char == ")": + if depth == 0: # the end of an enclosing CTE + body = body[:index] + break + depth -= 1 + if not re.search(r"\bWHERE\b", body, re.IGNORECASE): + found.append(" ".join(sql[match.start() : match.start() + 100].split())) + return found + + +def service_sql() -> list[tuple[str, str]]: + sources = [] + for path in sorted((ROOT / "src").rglob("*.py")): + for node in ast.walk(ast.parse(path.read_text(), filename=str(path))): + if ( + isinstance(node, ast.Constant) + and isinstance(node.value, str) + and STATEMENT.search(node.value) + and re.search(r"\b(SET|FROM)\b", node.value) + ): + sources.append((f"{path.relative_to(ROOT)}:{node.lineno}", node.value)) + for path in sorted((ROOT / "migrations").glob("*.sql")): + sources.append((str(path.relative_to(ROOT)), path.read_text())) + return sources + + +def test_every_delete_and_update_has_a_where_clause(): + offenders = [ + f"{where}: {statement}" for where, sql in service_sql() for statement in unguarded(sql) + ] + assert not offenders, ( + "add a WHERE clause (WHERE true for a deliberate full-table change):\n" + + ("\n".join(offenders)) + ) + + +def test_the_check_recognizes_unguarded_statements(): + assert unguarded("DELETE FROM runs") + assert unguarded("UPDATE runs SET status = 'x'; SELECT 1 WHERE true") + assert unguarded("WITH gone AS (DELETE FROM runs RETURNING id) SELECT * FROM gone WHERE true") + assert not unguarded("DELETE FROM runs WHERE id = $1") + assert not unguarded("UPDATE runs r SET status = 'x' -- no; WHERE yet\nWHERE r.id = $1") + assert not unguarded("SELECT id FROM runs WHERE id = $1 FOR UPDATE") + assert not unguarded("INSERT INTO t VALUES (1) ON CONFLICT (id) DO UPDATE SET v = 1") diff --git a/tests/test_staging_isolation.py b/tests/test_staging_isolation.py new file mode 100644 index 00000000..b7572ec3 --- /dev/null +++ b/tests/test_staging_isolation.py @@ -0,0 +1,146 @@ +"""Staging shares the code.storage organization but never production's repositories or state.""" + +import jwt +import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import ec + +from tin_lite.code_storage import CodeStorage +from tin_lite.settings import Settings + + +def _pem() -> str: + return ( + ec.generate_private_key(ec.SECP256R1()) + .private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ) + .decode() + ) + + +class RecordingRepo: + def __init__(self, repo_id: str) -> None: + self.id = repo_id + + async def list_commits(self, **_kwargs): + return {"commits": [{"sha": "a" * 40}]} + + +class RecordingClient: + def __init__(self) -> None: + self.calls: list[tuple[str, str]] = [] + self.existing: set[str] = set() + + async def find_one(self, *, id: str): + self.calls.append(("find", id)) + return RecordingRepo(id) if id in self.existing else None + + async def create_repo(self, *, id: str, **_kwargs): + self.calls.append(("create", id)) + self.existing.add(id) + return RecordingRepo(id) + + async def delete_repo(self, *, id: str, **_kwargs): + self.calls.append(("delete", id)) + self.existing.discard(id) + + +@pytest.mark.asyncio +async def test_prefix_reaches_every_upstream_repository_name() -> None: + pem = _pem() + storage = CodeStorage(organization="tin", private_key=pem, repo_prefix="staging-") + client = RecordingClient() + storage._client = client # type: ignore[assignment] + + repo = await storage.ensure_repo("registry/workflows") + assert repo.id == "staging-registry/workflows" + assert (await storage.get_repo("registry/workflows")) is not None + await storage.delete_repo("registry/workflows") + assert {name for _, name in client.calls} == {"staging-registry/workflows"} + + remotes = storage.sandbox_remotes(repo_id="projects/p1", branch="run", subject="run") + assert remotes.canonical_url == "https://tin.code.storage/staging-projects/p1.git" + assert remotes.ephemeral_url == "https://tin.code.storage/staging-projects/p1+ephemeral.git" + token = storage._generate_token(repo_id="projects/p1", scopes=["git:read"], subject="s", ttl=60) + assert jwt.decode(token, options={"verify_signature": False})["repo"] == "staging-projects/p1" + + +@pytest.mark.asyncio +async def test_no_prefix_keeps_production_names() -> None: + storage = CodeStorage(organization="tin", private_key=_pem()) + client = RecordingClient() + storage._client = client # type: ignore[assignment] + + await storage.ensure_repo("projects/p1") + assert {name for _, name in client.calls} == {"projects/p1"} + remotes = storage.sandbox_remotes(repo_id="projects/p1", branch="run", subject="run") + assert remotes.canonical_url == "https://tin.code.storage/projects/p1.git" + + +def settings_values(monkeypatch, **overrides): + for field in Settings.model_fields.values(): + monkeypatch.delenv(field.alias, raising=False) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("CODEX_API_KEY", raising=False) + values = { + field.alias: 5432 if name.endswith("port") else "synthetic" + for name, field in Settings.model_fields.items() + if field.is_required() + } + values["CLERK_PUBLISHABLE_KEY"] = "pk_test_synthetic" + values.update(overrides) + return values + + +STAGING = { + "TIN_LITE_ENVIRONMENT": "staging", + "TIN_LITE_CODE_STORAGE_REPO_PREFIX": "staging-", + "TIN_LITE_PRODUCTION_DATABASE_HOST": "prod.db.example", + "TIN_LITE_PRODUCTION_TEMPORAL_NAMESPACE": "prod.ns", + "DATABASE_HOST": "127.0.0.1", + "MIGRATION_HOST": "127.0.0.1", + "TEMPORAL_NAMESPACE": "staging.ns", +} + + +def test_staging_settings_load(monkeypatch): + settings = Settings(_env_file=None, **settings_values(monkeypatch, **STAGING)) + assert settings.deployment_environment == "staging" + assert settings.code_storage_repo_prefix == "staging-" + + +def test_production_is_the_default_and_needs_no_prefix(monkeypatch): + settings = Settings(_env_file=None, **settings_values(monkeypatch)) + assert settings.deployment_environment == "production" + assert settings.code_storage_repo_prefix == "" + + +@pytest.mark.parametrize( + ("override", "message"), + [ + ({"TIN_LITE_CODE_STORAGE_REPO_PREFIX": ""}, "REPO_PREFIX"), + ({"TIN_LITE_STRIPE_MODE": "live"}, "STRIPE_MODE=test"), + ({"TIN_LITE_PRODUCTION_DATABASE_HOST": None}, "compare against"), + ({"DATABASE_HOST": "prod.db.example"}, "production's database"), + ({"MIGRATION_HOST": "prod.db.example"}, "production's database"), + ({"DATABASE_HOST": "PROD.db.example."}, "production's database"), + ({"TEMPORAL_NAMESPACE": "prod.ns"}, "Temporal namespace"), + ], +) +def test_staging_refuses_production_state(monkeypatch, override, message): + values = settings_values(monkeypatch, **{**STAGING, **override}) + values = {key: value for key, value in values.items() if value is not None} + with pytest.raises(ValueError, match=message): + Settings(_env_file=None, **values) + + +@pytest.mark.parametrize("prefix", ["staging", "Staging-", "staging/", "-", "../x-"]) +def test_repo_prefix_shape(monkeypatch, prefix): + with pytest.raises(ValueError): + Settings( + _env_file=None, + **settings_values(monkeypatch, TIN_LITE_CODE_STORAGE_REPO_PREFIX=prefix), + )