PostHog analytics for the CLI and install scripts

.github/workflows/publish-install-script.yml

@@ -20,6 +20,42 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Inject PostHog analytics configuration
+        # Substituted into the checkout only - never committed back. A source
+        # checkout keeps the placeholders, so a clone reports no analytics.
+        env:
+          POSTHOG_PROJECT_TOKEN: ${{ secrets.POSTHOG_PROJECT_TOKEN }}
+          POSTHOG_CAPTURE_URL: ${{ vars.POSTHOG_CAPTURE_URL }}
+        run: |
+          set -euo pipefail
+
+          # Fail loudly rather than ship an artifact that silently reports nothing.
+          case "${POSTHOG_PROJECT_TOKEN:-}" in
+            phc_*) ;;
+            *) echo "::error::POSTHOG_PROJECT_TOKEN secret is missing or does not start with phc_"; exit 1 ;;
+          esac
+          case "${POSTHOG_CAPTURE_URL:-}" in
+            https://*) ;;
+            *) echo "::error::POSTHOG_CAPTURE_URL variable is missing or is not an https URL"; exit 1 ;;
+          esac
+
+          for f in install/bash/install.sh install/powershell/install.ps1; do
+            # '|' as the sed delimiter: the URL contains '/', the token cannot contain '|'.
+            # Written through a temp file rather than with 'sed -i', which is not
+            # portable, and copied back with 'cat' so the file keeps its mode.
+            tmp="$(mktemp)"
+            sed -e "s|__POSTHOG_PROJECT_TOKEN__|${POSTHOG_PROJECT_TOKEN}|g" \
+                -e "s|__POSTHOG_CAPTURE_URL__|${POSTHOG_CAPTURE_URL}|g" \
+                "$f" > "$tmp"
+            cat "$tmp" > "$f"
+            rm -f "$tmp"
+            if grep -q "__POSTHOG_" "$f"; then
+              echo "::error::$f still holds an unsubstituted PostHog placeholder"
+              exit 1
+            fi
+            echo "Injected analytics configuration into $f"
+          done
+
       - name: Upload install scripts to R2
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}

.github/workflows/publish-to-pypi.yml

@@ -38,6 +38,42 @@ jobs:
           curl -LsSf https://astral.sh/uv/install.sh | sh
           echo "$HOME/.local/bin" >> $GITHUB_PATH
 
+      - name: Inject PostHog analytics configuration
+        # Substituted into the checkout only - never committed back. A source
+        # checkout keeps the placeholders, so a clone reports no analytics.
+        env:
+          POSTHOG_PROJECT_TOKEN: ${{ secrets.POSTHOG_PROJECT_TOKEN }}
+          POSTHOG_CAPTURE_URL: ${{ vars.POSTHOG_CAPTURE_URL }}
+        run: |
+          set -euo pipefail
+
+          # Fail loudly rather than ship an artifact that silently reports nothing.
+          case "${POSTHOG_PROJECT_TOKEN:-}" in
+            phc_*) ;;
+            *) echo "::error::POSTHOG_PROJECT_TOKEN secret is missing or does not start with phc_"; exit 1 ;;
+          esac
+          case "${POSTHOG_CAPTURE_URL:-}" in
+            https://*) ;;
+            *) echo "::error::POSTHOG_CAPTURE_URL variable is missing or is not an https URL"; exit 1 ;;
+          esac
+
+          for f in analytics.py; do
+            # '|' as the sed delimiter: the URL contains '/', the token cannot contain '|'.
+            # Written through a temp file rather than with 'sed -i', which is not
+            # portable, and copied back with 'cat' so the file keeps its mode.
+            tmp="$(mktemp)"
+            sed -e "s|__POSTHOG_PROJECT_TOKEN__|${POSTHOG_PROJECT_TOKEN}|g" \
+                -e "s|__POSTHOG_CAPTURE_URL__|${POSTHOG_CAPTURE_URL}|g" \
+                "$f" > "$tmp"
+            cat "$tmp" > "$f"
+            rm -f "$tmp"
+            if grep -q "__POSTHOG_" "$f"; then
+              echo "::error::$f still holds an unsubstituted PostHog placeholder"
+              exit 1
+            fi
+            echo "Injected analytics configuration into $f"
+          done
+
       - name: Build package
         # hatch-vcs reads the version straight from the release tag.
         run: |

analytics.py

@@ -0,0 +1,146 @@
+"""Product analytics via PostHog.
+
+Events go to the same PostHog project the web app uses, keyed on the user's
+email address. The web app identifies people by email too (posthog.identify in
+codeplain-webapp), so a render lands on the same PostHog person as that user's
+signup, plan selection and payment.
+
+Consent is shared with crash reporting: CODEPLAIN_TELEMETRY governs both (see
+plain2code_telemetry.telemetry_enabled). The email is the only identity the CLI
+has; it never mints an anonymous id, so a run that cannot resolve the email
+sends nothing.
+
+The install scripts send their own "cli_installed" event with curl. This module
+covers only the events the CLI itself emits.
+"""
+
+import os
+import platform
+from typing import Any, Optional
+
+import requests
+
+from plain2code_state import RunState
+from plain2code_telemetry import telemetry_enabled
+from system_config import system_config
+
+# Both values are substituted at publish time by the publish-to-pypi workflow,
+# from the repository's POSTHOG_PROJECT_TOKEN secret and POSTHOG_CAPTURE_URL
+# variable. A source checkout keeps the placeholders below, so a run from a
+# clone reports nothing unless the environment supplies both values.
+POSTHOG_PROJECT_TOKEN = "__POSTHOG_PROJECT_TOKEN__"
+POSTHOG_CAPTURE_URL = "__POSTHOG_CAPTURE_URL__"
+
+PROJECT_TOKEN_ENV_VAR = "CODEPLAIN_POSTHOG_PROJECT_TOKEN"
+CAPTURE_URL_ENV_VAR = "CODEPLAIN_POSTHOG_CAPTURE_URL"
+
+CAPTURE_TIMEOUT_SECONDS = 2
+
+RENDER_FINISHED_EVENT = "cli_render_finished"
+
+OUTCOME_SUCCEEDED = "succeeded"
+OUTCOME_CANCELLED = "cancelled"
+OUTCOME_CRASHED = "crashed"
+OUTCOME_FAILED = "failed"
+
+
+def _posthog_config() -> Optional[tuple[str, str]]:
+    """Return (capture_url, project_token), or None if analytics are unconfigured.
+
+    A PostHog project token always starts with "phc_", so that prefix is also
+    what tells a published build apart from a source checkout still carrying a
+    placeholder. The environment wins over the built-in values, which is how a
+    source checkout can report events at all.
+    """
+    token = os.environ.get(PROJECT_TOKEN_ENV_VAR, "").strip() or POSTHOG_PROJECT_TOKEN
+    url = os.environ.get(CAPTURE_URL_ENV_VAR, "").strip() or POSTHOG_CAPTURE_URL
+
+    if not token.startswith("phc_") or not url.startswith("https://"):
+        return None
+
+    return url, token
+
+
+def _render_outcome(run_state: RunState, crashed: bool) -> str:
+    """Classify the render the way the exit summary does, with crashes split out
+    of the generic failure case."""
+    if run_state.render_succeeded:
+        return OUTCOME_SUCCEEDED
+    if run_state.render_cancelled:
+        return OUTCOME_CANCELLED
+    if crashed:
+        return OUTCOME_CRASHED
+    return OUTCOME_FAILED
+
+
+def _capture(config: tuple[str, str], event: str, distinct_id: str, properties: dict[str, Any]) -> bool:
+    """Send a single event to PostHog. Returns True if PostHog accepted it."""
+    capture_url, project_token = config
+    payload: dict[str, Any] = {
+        "api_key": project_token,
+        "event": event,
+        "distinct_id": distinct_id,
+        "properties": {
+            "$lib": "codeplain-cli",
+            "$lib_version": system_config.client_version,
+            **properties,
+        },
+    }
+    response = requests.post(capture_url, json=payload, timeout=CAPTURE_TIMEOUT_SECONDS)
+    return response.ok
+
+
+def capture_render_finished(
+    run_state: RunState,
+    args,
+    module_count: int,
+    error_type: Optional[str] = None,
+    crashed: bool = False,
+) -> bool:
+    """Report how a render ended. Returns True if an event was sent.
+
+    Nothing is sent when analytics are unconfigured (a source checkout), when the
+    user opted out, or when the run never learned who the user is (an invalid or
+    missing API key fails before the connection check returns the email).
+
+    No spec content, file paths or error messages are sent - only the exception
+    class name, which cannot carry proprietary content.
+    """
+    if not telemetry_enabled():
+        return False
+
+    if not run_state.user_email:
+        return False
+
+    config = _posthog_config()
+    if config is None:
+        return False
+
+    outcome = _render_outcome(run_state, crashed)
+
+    try:
+        return _capture(
+            config,
+            RENDER_FINISHED_EVENT,
+            run_state.user_email,
+            {
+                "render_id": run_state.render_id,
+                "outcome": outcome,
+                "rendered_functionalities": run_state.rendered_functionalities,
+                # Matches the duration the exit summary prints.
+                "render_time_seconds": run_state.render_time_accumulated,
+                "module_count": module_count,
+                "client_version": system_config.client_version,
+                "os": platform.system(),
+                "headless": bool(getattr(args, "headless", False)),
+                "unittests_script_provided": bool(getattr(args, "unittests_script", None)),
+                "conformance_tests_script_provided": bool(getattr(args, "conformance_tests_script", None)),
+                "prepare_environment_script_provided": bool(getattr(args, "prepare_environment_script", None)),
+                # A cancel also unwinds through an exception; only report the
+                # exception type when the render actually went wrong.
+                "error_type": error_type if outcome in (OUTCOME_FAILED, OUTCOME_CRASHED) else None,
+            },
+        )
+    except Exception:
+        # Analytics must never break the CLI or mask the original error.
+        return False

install/bash/install.sh

@@ -27,6 +27,22 @@ export YELLOW GREEN GREEN_LIGHT GREEN_DARK BLUE BLACK WHITE RED GRAY GRAY_LIGHT
 
 NONINTERACTIVE="${CODEPLAIN_INSTALL_NONINTERACTIVE:-0}"
 
+# PostHog analytics. Both values are substituted at publish time by the
+# publish-install-script workflow, from the repository's POSTHOG_PROJECT_TOKEN
+# secret and POSTHOG_CAPTURE_URL variable. A copy run straight from a clone
+# keeps the placeholders and reports nothing, unless the environment supplies
+# both values.
+POSTHOG_CAPTURE_URL="${CODEPLAIN_POSTHOG_CAPTURE_URL:-__POSTHOG_CAPTURE_URL__}"
+POSTHOG_PROJECT_TOKEN="${CODEPLAIN_POSTHOG_PROJECT_TOKEN:-__POSTHOG_PROJECT_TOKEN__}"
+
+# Email of the user the API key belongs to; set by validate_api_key. Analytics
+# identify a user by email, so an install that never verified a key sends no
+# event.
+VALIDATED_USER_EMAIL=""
+
+# "fresh" or "upgrade"; set when codeplain is installed below.
+INSTALL_TYPE=""
+
 if [ "$NONINTERACTIVE" = "1" ]; then
     echo "Running in non-interactive mode (CODEPLAIN_INSTALL_NONINTERACTIVE=1)"
 else
@@ -84,20 +100,101 @@ trim_whitespace() {
 }
 
 # Verify an API key against the Codeplain API's /status endpoint.
-# Sets the global VALIDATION_HTTP_CODE and returns 0 only when the key is valid
-# (HTTP 200). This checks only the API key, nothing else about the install.
+# Sets the globals VALIDATION_HTTP_CODE and VALIDATED_USER_EMAIL, and returns 0
+# only when the key is valid (HTTP 200). This checks only the API key, nothing
+# else about the install.
 validate_api_key() {
     local key="$1"
-    local http_code
-    http_code=$(curl -s -o /dev/null -w "%{http_code}" \
+    local response
+    response=$(curl -s -w $'\n%{http_code}' \
         --max-time 30 \
         -X POST "${CODEPLAIN_API_URL}/status" \
         -H "Content-Type: application/json" \
         --data "{\"api_key\":\"${key}\"}" 2>/dev/null || true)
-    VALIDATION_HTTP_CODE="${http_code:-000}"
+
+    # The status code is appended on its own last line; everything above it is
+    # the JSON body.
+    VALIDATION_HTTP_CODE="$(printf '%s' "$response" | tail -n 1)"
+    VALIDATION_HTTP_CODE="${VALIDATION_HTTP_CODE:-000}"
+
+    # /status reports the user this key belongs to. The email identifies the
+    # user in analytics (see capture_install_event). Matching on "email" with
+    # the leading quote skips "organization_owner_email", which is a different
+    # user in an organization.
+    local body
+    body="$(printf '%s' "$response" | sed '$d')"
+    VALIDATED_USER_EMAIL="$(printf '%s' "$body" \
+        | grep -o '"email"[[:space:]]*:[[:space:]]*"[^"]*"' \
+        | head -n 1 \
+        | sed 's/.*"\([^"]*\)"$/\1/' || true)"
+
     [ "$VALIDATION_HTTP_CODE" = "200" ]
 }
 
+# Analytics honor the same CODEPLAIN_TELEMETRY opt-out as the CLI's crash
+# reporting. The CLI defaults to off outside a real user install; running this
+# installer *is* a real user install, so here the default is on.
+telemetry_enabled() {
+    local setting
+    setting="$(printf '%s' "${CODEPLAIN_TELEMETRY:-}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')"
+    case "$setting" in
+        0 | false | off) return 1 ;;
+    esac
+    return 0
+}
+
+# True only once the publish workflow (or the environment) has supplied a real
+# PostHog token and host. Every PostHog project token starts with "phc_", so
+# that prefix also tells a published installer apart from an unsubstituted one.
+analytics_configured() {
+    case "$POSTHOG_PROJECT_TOKEN" in
+        phc_*) ;;
+        *) return 1 ;;
+    esac
+    case "$POSTHOG_CAPTURE_URL" in
+        https://*) ;;
+        *) return 1 ;;
+    esac
+    return 0
+}
+
+# Report the finished install to PostHog, keyed on the user's email so the
+# install lands on the same person as their signup on the web app. Takes "true"
+# or "false" for whether 'codeplain --status' ran successfully.
+# Never fails the install: no email, an opt-out, or an unreachable PostHog are
+# all silent no-ops.
+capture_install_event() {
+    local install_verified="$1"
+
+    if [ -z "$VALIDATED_USER_EMAIL" ] || ! telemetry_enabled || ! analytics_configured; then
+        return 0
+    fi
+
+    local installed_version noninteractive payload
+    installed_version="$(uv tool list 2>/dev/null | grep "^codeplain" | sed 's/codeplain v//' || true)"
+    if [ "$NONINTERACTIVE" = "1" ]; then
+        noninteractive=true
+    else
+        noninteractive=false
+    fi
+
+    payload="$(printf '{"api_key":"%s","event":"cli_installed","distinct_id":"%s","properties":{"$lib":"codeplain-install-script","client_version":"%s","os":"%s","install_type":"%s","install_verified":%s,"plain_forge_installed":%s,"plyn_installed":%s,"noninteractive":%s}}' \
+        "$POSTHOG_PROJECT_TOKEN" \
+        "$VALIDATED_USER_EMAIL" \
+        "$installed_version" \
+        "$(uname -s)" \
+        "$INSTALL_TYPE" \
+        "$install_verified" \
+        "${PLAIN_FORGE_INSTALLED:-false}" \
+        "${PLYN_INSTALLED:-false}" \
+        "$noninteractive")"
+
+    curl -s -o /dev/null --max-time 2 \
+        -X POST "$POSTHOG_CAPTURE_URL" \
+        -H "Content-Type: application/json" \
+        --data "$payload" > /dev/null 2>&1 || true
+}
+
 # Check if uv is installed
 if ! command -v uv &> /dev/null; then
     echo -e "${GRAY}uv is not installed.${NC}"
@@ -121,6 +218,7 @@ echo -e ""
 # Install or upgrade codeplain using uv tool
 if uv tool list 2>/dev/null | grep -q "^codeplain"; then
     CURRENT_VERSION=$(uv tool list 2>/dev/null | grep "^codeplain" | sed 's/codeplain v//')
+    INSTALL_TYPE="upgrade"
     echo -e "${GRAY}codeplain ${CURRENT_VERSION} is already installed.${NC}"
     echo -e "Upgrading to latest version..."
     echo -e ""
@@ -132,6 +230,7 @@ if uv tool list 2>/dev/null | grep -q "^codeplain"; then
         echo -e "${GREEN}✓${NC} codeplain upgraded from ${CURRENT_VERSION} to ${NEW_VERSION}!"
     fi
 else
+    INSTALL_TYPE="fresh"
     echo -e "Installing codeplain...${NC}"
     echo -e ""
     uv tool install codeplain
@@ -445,6 +544,7 @@ if [ -n "${CODEPLAIN_API_KEY:-}" ]; then
         echo "$verify_output"
         echo -e "${GRAY}Please restart your terminal and try again, or reinstall with:${NC}"
         echo -e "  uv tool install --force codeplain"
+        capture_install_event false
         exit 1
     fi
 fi
@@ -481,6 +581,9 @@ echo ""
 echo -e "  ${GRAY}Happy development!${NC} 🚀"
 echo ""
 
+# Reported last: exec below replaces this process, so nothing after it runs.
+capture_install_event true
+
 if [ "$NONINTERACTIVE" != "1" ]; then
     # Replace this subshell with a fresh shell that has the new environment
     # Reconnect stdin to terminal (needed when running via curl | bash)