From 3c557b6f377d8e4ce768d79b66eab49649dd4a44 Mon Sep 17 00:00:00 2001 From: Nejc Stebe Date: Fri, 21 Aug 2026 13:21:23 +0200 Subject: [PATCH 1/3] report render outcomes to PostHog Send a "cli_render_finished" event when a render ends, keyed on the user's email. The web app already identifies people by email, so a render lands on the same PostHog person as that user's signup, plan and payment. The project token and capture host are not committed. They are placeholders that the publish-to-pypi workflow substitutes from the repository's POSTHOG_PROJECT_TOKEN secret and POSTHOG_CAPTURE_URL variable, and the workflow fails rather than ship a build that would silently report nothing. A source checkout therefore reports nothing at all, unless CODEPLAIN_POSTHOG_PROJECT_TOKEN and CODEPLAIN_POSTHOG_CAPTURE_URL are set -- which is how a dev run can exercise analytics without a release. Consent is shared with crash reporting: CODEPLAIN_TELEMETRY governs both. The email is the only identity the CLI has - it mints no anonymous ids, so a run that never resolved the email (invalid or missing API key) sends nothing. No spec content, paths or error messages are sent; a failure is described by its exception class name only. --- .github/workflows/publish-to-pypi.yml | 36 +++++ analytics.py | 146 ++++++++++++++++++ plain2code.py | 10 ++ tests/test_analytics.py | 212 ++++++++++++++++++++++++++ tests/test_plain2code.py | 80 ++++++++++ 5 files changed, 484 insertions(+) create mode 100644 analytics.py create mode 100644 tests/test_analytics.py diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index 01d06903..54c35e7e 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -38,6 +38,42 @@ jobs: curl -LsSf https://astral.sh/uv/install.sh | sh echo "$HOME/.local/bin" >> $GITHUB_PATH + - name: Inject PostHog analytics configuration + # Substituted into the checkout only - never committed back. A source + # checkout keeps the placeholders, so a clone reports no analytics. + env: + POSTHOG_PROJECT_TOKEN: ${{ secrets.POSTHOG_PROJECT_TOKEN }} + POSTHOG_CAPTURE_URL: ${{ vars.POSTHOG_CAPTURE_URL }} + run: | + set -euo pipefail + + # Fail loudly rather than ship an artifact that silently reports nothing. + case "${POSTHOG_PROJECT_TOKEN:-}" in + phc_*) ;; + *) echo "::error::POSTHOG_PROJECT_TOKEN secret is missing or does not start with phc_"; exit 1 ;; + esac + case "${POSTHOG_CAPTURE_URL:-}" in + https://*) ;; + *) echo "::error::POSTHOG_CAPTURE_URL variable is missing or is not an https URL"; exit 1 ;; + esac + + for f in analytics.py; do + # '|' as the sed delimiter: the URL contains '/', the token cannot contain '|'. + # Written through a temp file rather than with 'sed -i', which is not + # portable, and copied back with 'cat' so the file keeps its mode. + tmp="$(mktemp)" + sed -e "s|__POSTHOG_PROJECT_TOKEN__|${POSTHOG_PROJECT_TOKEN}|g" \ + -e "s|__POSTHOG_CAPTURE_URL__|${POSTHOG_CAPTURE_URL}|g" \ + "$f" > "$tmp" + cat "$tmp" > "$f" + rm -f "$tmp" + if grep -q "__POSTHOG_" "$f"; then + echo "::error::$f still holds an unsubstituted PostHog placeholder" + exit 1 + fi + echo "Injected analytics configuration into $f" + done + - name: Build package # hatch-vcs reads the version straight from the release tag. run: | diff --git a/analytics.py b/analytics.py new file mode 100644 index 00000000..74a1ef6c --- /dev/null +++ b/analytics.py @@ -0,0 +1,146 @@ +"""Product analytics via PostHog. + +Events go to the same PostHog project the web app uses, keyed on the user's +email address. The web app identifies people by email too (posthog.identify in +codeplain-webapp), so a render lands on the same PostHog person as that user's +signup, plan selection and payment. + +Consent is shared with crash reporting: CODEPLAIN_TELEMETRY governs both (see +plain2code_telemetry.telemetry_enabled). The email is the only identity the CLI +has; it never mints an anonymous id, so a run that cannot resolve the email +sends nothing. + +The install scripts send their own "cli_installed" event with curl. This module +covers only the events the CLI itself emits. +""" + +import os +import platform +from typing import Any, Optional + +import requests + +from plain2code_state import RunState +from plain2code_telemetry import telemetry_enabled +from system_config import system_config + +# Both values are substituted at publish time by the publish-to-pypi workflow, +# from the repository's POSTHOG_PROJECT_TOKEN secret and POSTHOG_CAPTURE_URL +# variable. A source checkout keeps the placeholders below, so a run from a +# clone reports nothing unless the environment supplies both values. +POSTHOG_PROJECT_TOKEN = "__POSTHOG_PROJECT_TOKEN__" +POSTHOG_CAPTURE_URL = "__POSTHOG_CAPTURE_URL__" + +PROJECT_TOKEN_ENV_VAR = "CODEPLAIN_POSTHOG_PROJECT_TOKEN" +CAPTURE_URL_ENV_VAR = "CODEPLAIN_POSTHOG_CAPTURE_URL" + +CAPTURE_TIMEOUT_SECONDS = 2 + +RENDER_FINISHED_EVENT = "cli_render_finished" + +OUTCOME_SUCCEEDED = "succeeded" +OUTCOME_CANCELLED = "cancelled" +OUTCOME_CRASHED = "crashed" +OUTCOME_FAILED = "failed" + + +def _posthog_config() -> Optional[tuple[str, str]]: + """Return (capture_url, project_token), or None if analytics are unconfigured. + + A PostHog project token always starts with "phc_", so that prefix is also + what tells a published build apart from a source checkout still carrying a + placeholder. The environment wins over the built-in values, which is how a + source checkout can report events at all. + """ + token = os.environ.get(PROJECT_TOKEN_ENV_VAR, "").strip() or POSTHOG_PROJECT_TOKEN + url = os.environ.get(CAPTURE_URL_ENV_VAR, "").strip() or POSTHOG_CAPTURE_URL + + if not token.startswith("phc_") or not url.startswith("https://"): + return None + + return url, token + + +def _render_outcome(run_state: RunState, crashed: bool) -> str: + """Classify the render the way the exit summary does, with crashes split out + of the generic failure case.""" + if run_state.render_succeeded: + return OUTCOME_SUCCEEDED + if run_state.render_cancelled: + return OUTCOME_CANCELLED + if crashed: + return OUTCOME_CRASHED + return OUTCOME_FAILED + + +def _capture(config: tuple[str, str], event: str, distinct_id: str, properties: dict[str, Any]) -> bool: + """Send a single event to PostHog. Returns True if PostHog accepted it.""" + capture_url, project_token = config + payload: dict[str, Any] = { + "api_key": project_token, + "event": event, + "distinct_id": distinct_id, + "properties": { + "$lib": "codeplain-cli", + "$lib_version": system_config.client_version, + **properties, + }, + } + response = requests.post(capture_url, json=payload, timeout=CAPTURE_TIMEOUT_SECONDS) + return response.ok + + +def capture_render_finished( + run_state: RunState, + args, + module_count: int, + error_type: Optional[str] = None, + crashed: bool = False, +) -> bool: + """Report how a render ended. Returns True if an event was sent. + + Nothing is sent when analytics are unconfigured (a source checkout), when the + user opted out, or when the run never learned who the user is (an invalid or + missing API key fails before the connection check returns the email). + + No spec content, file paths or error messages are sent - only the exception + class name, which cannot carry proprietary content. + """ + if not telemetry_enabled(): + return False + + if not run_state.user_email: + return False + + config = _posthog_config() + if config is None: + return False + + outcome = _render_outcome(run_state, crashed) + + try: + return _capture( + config, + RENDER_FINISHED_EVENT, + run_state.user_email, + { + "render_id": run_state.render_id, + "outcome": outcome, + "rendered_functionalities": run_state.rendered_functionalities, + # Matches the duration the exit summary prints. + "render_time_seconds": run_state.render_time_accumulated, + "module_count": module_count, + "client_version": system_config.client_version, + "os": platform.system(), + "headless": bool(getattr(args, "headless", False)), + "unittests_script_provided": bool(getattr(args, "unittests_script", None)), + "conformance_tests_script_provided": bool(getattr(args, "conformance_tests_script", None)), + "prepare_environment_script_provided": bool(getattr(args, "prepare_environment_script", None)), + # A cancel also unwinds through an exception; only report the + # exception type when the render actually went wrong. + "error_type": error_type if outcome in (OUTCOME_FAILED, OUTCOME_CRASHED) else None, + }, + ) + except Exception: + # Analytics must never break the CLI or mask the original error. + return False diff --git a/plain2code.py b/plain2code.py index 3c30deda..95926032 100644 --- a/plain2code.py +++ b/plain2code.py @@ -15,6 +15,7 @@ import plain_file import plain_modules import plain_spec +from analytics import capture_render_finished from cli_output import print_dry_run_output, print_exit_summary, print_status from event_bus import EventBus from module_renderer import ModuleRenderer @@ -403,6 +404,7 @@ def main(): # noqa: C901 exc_info = None error_message = None + error_type = None try: # Validate API key is present @@ -412,6 +414,7 @@ def main(): # noqa: C901 ) render(plain_module, args, run_state, event_bus, default_log_level) except BaseException as e: + error_type = type(e).__name__ if isinstance(e, KeyboardInterrupt): error_message = "Keyboard interrupt" else: @@ -428,6 +431,13 @@ def main(): # noqa: C901 args.filename, error_message=error_message, ) + capture_render_finished( + run_state, + args, + module_count=len(plain_module.all_required_modules) + 1, + error_type=error_type, + crashed=exc_info is not None, + ) # Remove any scratch extractions created for archive-only (".module") modules. for module in plain_module.all_required_modules + [plain_module]: module.cleanup_scratch() diff --git a/tests/test_analytics.py b/tests/test_analytics.py new file mode 100644 index 00000000..fde8d513 --- /dev/null +++ b/tests/test_analytics.py @@ -0,0 +1,212 @@ +from argparse import Namespace + +import pytest + +import analytics +import plain2code_telemetry +from analytics import ( + CAPTURE_URL_ENV_VAR, + OUTCOME_CANCELLED, + OUTCOME_CRASHED, + OUTCOME_FAILED, + OUTCOME_SUCCEEDED, + PROJECT_TOKEN_ENV_VAR, + RENDER_FINISHED_EVENT, + capture_render_finished, +) +from plain2code_state import RunState +from plain2code_telemetry import TELEMETRY_ENV_VAR + +# The token and host are placeholders in a source checkout; the publish workflow +# substitutes them. Tests supply them the way a developer would - via the +# environment. +TEST_TOKEN = "phc_testtoken" +TEST_URL = "https://eu.i.posthog.com/i/v0/e/" + + +class FakeResponse: + def __init__(self, ok=True): + self.ok = ok + + +@pytest.fixture(autouse=True) +def analytics_env(monkeypatch): + """Put the module in the state a published install would be in. + + Tests run from a source checkout, where telemetry is off by default and the + PostHog placeholders are unsubstituted; pretend to be production and supply + the config through the environment so the default path is covered. + """ + monkeypatch.delenv(TELEMETRY_ENV_VAR, raising=False) + monkeypatch.setattr(plain2code_telemetry.system_config, "environment", "production") + monkeypatch.setenv(PROJECT_TOKEN_ENV_VAR, TEST_TOKEN) + monkeypatch.setenv(CAPTURE_URL_ENV_VAR, TEST_URL) + + +@pytest.fixture +def sent(monkeypatch): + """Record capture requests instead of sending them over the network.""" + requests = [] + + def fake_post(url, json=None, timeout=None): + requests.append({"url": url, "payload": json, "timeout": timeout}) + return FakeResponse() + + monkeypatch.setattr(analytics.requests, "post", fake_post) + return requests + + +def make_args(**overrides): + args = Namespace( + headless=False, + unittests_script="run_unittests.sh", + conformance_tests_script=None, + prepare_environment_script=None, + ) + for key, value in overrides.items(): + setattr(args, key, value) + return args + + +def make_run_state(**overrides): + run_state = RunState(spec_filename="test.plain") + run_state.user_email = "user@codeplain.ai" + for key, value in overrides.items(): + setattr(run_state, key, value) + return run_state + + +def test_render_finished_event_is_sent_with_properties(sent): + run_state = make_run_state(render_succeeded=True, rendered_functionalities=4, render_time_accumulated=125) + + assert capture_render_finished(run_state, make_args(headless=True), module_count=3) + + assert len(sent) == 1 + payload = sent[0]["payload"] + assert sent[0]["url"] == TEST_URL + assert payload["api_key"] == TEST_TOKEN + assert payload["event"] == RENDER_FINISHED_EVENT + # The web app identifies people by email too, so CLI and web events land on + # the same PostHog person. + assert payload["distinct_id"] == "user@codeplain.ai" + + properties = payload["properties"] + assert properties["render_id"] == run_state.render_id + assert properties["outcome"] == OUTCOME_SUCCEEDED + assert properties["rendered_functionalities"] == 4 + assert properties["render_time_seconds"] == 125 + assert properties["module_count"] == 3 + assert properties["headless"] is True + assert properties["unittests_script_provided"] is True + assert properties["conformance_tests_script_provided"] is False + assert properties["prepare_environment_script_provided"] is False + assert properties["error_type"] is None + assert properties["$lib"] == "codeplain-cli" + + +def test_no_spec_content_is_sent(sent): + """Only the exception class name may describe a failure - never the message, + the spec filename or a path.""" + run_state = make_run_state(spec_filename="/home/user/secret_project.plain") + + assert capture_render_finished(run_state, make_args(), module_count=1, error_type="PlainSyntaxError") + + properties = sent[0]["payload"]["properties"] + assert "secret_project" not in str(properties) + assert properties["error_type"] == "PlainSyntaxError" + + +@pytest.mark.parametrize( + "state, crashed, expected", + [ + ({"render_succeeded": True}, False, OUTCOME_SUCCEEDED), + ({"render_cancelled": True}, False, OUTCOME_CANCELLED), + # A cancel unwinds through SystemExit, which is not a crash. + ({"render_cancelled": True}, True, OUTCOME_CANCELLED), + ({}, True, OUTCOME_CRASHED), + ({}, False, OUTCOME_FAILED), + ], +) +def test_outcome_classification(sent, state, crashed, expected): + assert capture_render_finished(make_run_state(**state), make_args(), module_count=1, crashed=crashed) + + assert sent[0]["payload"]["properties"]["outcome"] == expected + + +def test_error_type_is_dropped_for_a_cancelled_render(sent): + run_state = make_run_state(render_cancelled=True) + + assert capture_render_finished(run_state, make_args(), module_count=1, error_type="SystemExit") + + assert sent[0]["payload"]["properties"]["error_type"] is None + + +@pytest.mark.parametrize("value", ["0", "false", "off", "OFF", " False "]) +def test_opt_out_sends_nothing(monkeypatch, sent, value): + monkeypatch.setenv(TELEMETRY_ENV_VAR, value) + + assert not capture_render_finished(make_run_state(), make_args(), module_count=1) + assert sent == [] + + +def test_disabled_outside_production(monkeypatch, sent): + monkeypatch.setattr(plain2code_telemetry.system_config, "environment", "development") + + assert not capture_render_finished(make_run_state(), make_args(), module_count=1) + assert sent == [] + + +def test_nothing_is_sent_without_a_user_email(sent): + """The email is the only identity the CLI has; a run that never resolved it + (invalid or missing API key) is not reported.""" + run_state = make_run_state(user_email=None) + + assert not capture_render_finished(run_state, make_args(), module_count=1) + assert sent == [] + + +def test_nothing_is_sent_from_an_unconfigured_checkout(monkeypatch, sent): + """A source checkout carries unsubstituted placeholders. It must report + nothing rather than post to a nonexistent endpoint.""" + monkeypatch.delenv(PROJECT_TOKEN_ENV_VAR) + monkeypatch.delenv(CAPTURE_URL_ENV_VAR) + + assert not analytics.POSTHOG_PROJECT_TOKEN.startswith("phc_"), "a real token must never be committed" + assert not capture_render_finished(make_run_state(), make_args(), module_count=1) + assert sent == [] + + +@pytest.mark.parametrize( + "token, url", + [ + ("__POSTHOG_PROJECT_TOKEN__", TEST_URL), + ("not-a-posthog-token", TEST_URL), + ("", TEST_URL), + (TEST_TOKEN, "__POSTHOG_CAPTURE_URL__"), + (TEST_TOKEN, "http://insecure.example.com/"), + (TEST_TOKEN, ""), + ], +) +def test_a_half_substituted_config_sends_nothing(monkeypatch, sent, token, url): + monkeypatch.setenv(PROJECT_TOKEN_ENV_VAR, token) + monkeypatch.setenv(CAPTURE_URL_ENV_VAR, url) + + assert not capture_render_finished(make_run_state(), make_args(), module_count=1) + assert sent == [] + + +def test_capture_never_raises(monkeypatch): + def broken_post(*args, **kwargs): + raise RuntimeError("network on fire") + + monkeypatch.setattr(analytics.requests, "post", broken_post) + + assert capture_render_finished(make_run_state(), make_args(), module_count=1) is False + + +def test_capture_uses_a_short_timeout(sent): + """A render can end on Ctrl-C; analytics must not hold up the exit.""" + assert capture_render_finished(make_run_state(), make_args(), module_count=1) + + assert sent[0]["timeout"] == analytics.CAPTURE_TIMEOUT_SECONDS + assert analytics.CAPTURE_TIMEOUT_SECONDS <= 2 diff --git a/tests/test_plain2code.py b/tests/test_plain2code.py index 84c93975..f15cca52 100644 --- a/tests/test_plain2code.py +++ b/tests/test_plain2code.py @@ -91,3 +91,83 @@ def test_warning_covers_required_modules_for_real_plain_module(get_test_data_pat mock_console.warning.assert_called_once() warning_message = mock_console.warning.call_args.args[0] assert "required_with_acceptance_tests" in warning_message + + +def _run_main_with(render_side_effect, required_module_count): + """Drive plain2code.main() far enough to reach its exit path. + + Everything outside the exit path is stubbed: the point is to observe what + main() reports once the render is over. + """ + args = Namespace( + version=False, + status=False, + full_plain=False, + dry_run=False, + headless=True, + filename="test.plain", + template_dir=None, + build_folder="plain_modules", + api="https://api.codeplain.ai", + api_key="test-key", + replay_with=None, + render_range=None, + render_from=None, + log_to_file=False, + log_file_name=None, + unittests_script=None, + conformance_tests_script=None, + prepare_environment_script=None, + ) + required_modules = [SimpleNamespace(cleanup_scratch=lambda: None) for _ in range(required_module_count)] + plain_module = SimpleNamespace( + plain_source={}, + all_required_modules=required_modules, + cleanup_scratch=lambda: None, + ) + + with ( + patch.object(plain2code, "parse_arguments", return_value=args), + patch.object(plain2code.file_utils, "get_template_directories", return_value=["templates"]), + patch.object(plain2code.plain_modules, "PlainModule", return_value=plain_module), + patch.object(plain2code, "setup_logging", return_value="INFO"), + patch.object(plain2code, "initialize_telemetry"), + patch.object(plain2code, "print_exit_summary"), + patch.object(plain2code, "dump_crash_logs"), + patch.object(plain2code, "capture_crash"), + patch.object(plain2code, "render", side_effect=render_side_effect) as render_mock, + patch.object(plain2code, "capture_render_finished") as capture_mock, + patch.object(plain2code.sys, "exit"), + ): + plain2code.main() + + assert render_mock.called + assert capture_mock.call_count == 1 + return capture_mock.call_args + + +def test_render_outcome_is_reported_for_a_successful_render(): + call_args = _run_main_with(render_side_effect=None, required_module_count=1) + + assert call_args.kwargs["module_count"] == 2 + assert call_args.kwargs["error_type"] is None + assert call_args.kwargs["crashed"] is False + + +def test_render_outcome_is_reported_for_an_expected_failure(): + call_args = _run_main_with( + render_side_effect=plain2code.PlainSyntaxError("bad spec"), + required_module_count=0, + ) + + assert call_args.kwargs["module_count"] == 1 + assert call_args.kwargs["error_type"] == "PlainSyntaxError" + # An expected error is a failed render, not a crash. + assert call_args.kwargs["crashed"] is False + + +def test_render_outcome_is_reported_for_an_unexpected_crash(): + call_args = _run_main_with(render_side_effect=RuntimeError("boom"), required_module_count=0) + + assert call_args.kwargs["error_type"] == "RuntimeError" + assert call_args.kwargs["crashed"] is True From 8b3f2d195fc333c8bfdcf35c2de8cdbf6d51bd94 Mon Sep 17 00:00:00 2001 From: Nejc Stebe Date: Fri, 21 Aug 2026 13:21:23 +0200 Subject: [PATCH 2/3] report finished installs to PostHog Send a "cli_installed" event at the end of both install scripts, keyed on the user's email so an install joins the same PostHog person as their signup. Both installers already call /status to verify the API key; they now read the email out of that response. As with the CLI, the token and host are placeholders. The publish-install-script workflow substitutes them on their way to R2, so the copies users curl are configured while the copies in this repository are not. A clone reports nothing unless CODEPLAIN_POSTHOG_* is set in the environment. The failed-verification path reports too, with install_verified false. An install that never verified a key sends nothing, since there is no identity for it. Analytics honor the same CODEPLAIN_TELEMETRY opt-out as the CLI. The CLI defaults to off outside a production install; the installers default to on, because running an installer is a real user install. --- .github/workflows/publish-install-script.yml | 36 ++++++ install/bash/install.sh | 113 ++++++++++++++++++- install/powershell/install.ps1 | 90 ++++++++++++++- 3 files changed, 233 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish-install-script.yml b/.github/workflows/publish-install-script.yml index 94f2b1b0..8061d0b6 100644 --- a/.github/workflows/publish-install-script.yml +++ b/.github/workflows/publish-install-script.yml @@ -20,6 +20,42 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Inject PostHog analytics configuration + # Substituted into the checkout only - never committed back. A source + # checkout keeps the placeholders, so a clone reports no analytics. + env: + POSTHOG_PROJECT_TOKEN: ${{ secrets.POSTHOG_PROJECT_TOKEN }} + POSTHOG_CAPTURE_URL: ${{ vars.POSTHOG_CAPTURE_URL }} + run: | + set -euo pipefail + + # Fail loudly rather than ship an artifact that silently reports nothing. + case "${POSTHOG_PROJECT_TOKEN:-}" in + phc_*) ;; + *) echo "::error::POSTHOG_PROJECT_TOKEN secret is missing or does not start with phc_"; exit 1 ;; + esac + case "${POSTHOG_CAPTURE_URL:-}" in + https://*) ;; + *) echo "::error::POSTHOG_CAPTURE_URL variable is missing or is not an https URL"; exit 1 ;; + esac + + for f in install/bash/install.sh install/powershell/install.ps1; do + # '|' as the sed delimiter: the URL contains '/', the token cannot contain '|'. + # Written through a temp file rather than with 'sed -i', which is not + # portable, and copied back with 'cat' so the file keeps its mode. + tmp="$(mktemp)" + sed -e "s|__POSTHOG_PROJECT_TOKEN__|${POSTHOG_PROJECT_TOKEN}|g" \ + -e "s|__POSTHOG_CAPTURE_URL__|${POSTHOG_CAPTURE_URL}|g" \ + "$f" > "$tmp" + cat "$tmp" > "$f" + rm -f "$tmp" + if grep -q "__POSTHOG_" "$f"; then + echo "::error::$f still holds an unsubstituted PostHog placeholder" + exit 1 + fi + echo "Injected analytics configuration into $f" + done + - name: Upload install scripts to R2 env: AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} diff --git a/install/bash/install.sh b/install/bash/install.sh index dfd371e2..905ed534 100755 --- a/install/bash/install.sh +++ b/install/bash/install.sh @@ -27,6 +27,22 @@ export YELLOW GREEN GREEN_LIGHT GREEN_DARK BLUE BLACK WHITE RED GRAY GRAY_LIGHT NONINTERACTIVE="${CODEPLAIN_INSTALL_NONINTERACTIVE:-0}" +# PostHog analytics. Both values are substituted at publish time by the +# publish-install-script workflow, from the repository's POSTHOG_PROJECT_TOKEN +# secret and POSTHOG_CAPTURE_URL variable. A copy run straight from a clone +# keeps the placeholders and reports nothing, unless the environment supplies +# both values. +POSTHOG_CAPTURE_URL="${CODEPLAIN_POSTHOG_CAPTURE_URL:-__POSTHOG_CAPTURE_URL__}" +POSTHOG_PROJECT_TOKEN="${CODEPLAIN_POSTHOG_PROJECT_TOKEN:-__POSTHOG_PROJECT_TOKEN__}" + +# Email of the user the API key belongs to; set by validate_api_key. Analytics +# identify a user by email, so an install that never verified a key sends no +# event. +VALIDATED_USER_EMAIL="" + +# "fresh" or "upgrade"; set when codeplain is installed below. +INSTALL_TYPE="" + if [ "$NONINTERACTIVE" = "1" ]; then echo "Running in non-interactive mode (CODEPLAIN_INSTALL_NONINTERACTIVE=1)" else @@ -84,20 +100,101 @@ trim_whitespace() { } # Verify an API key against the Codeplain API's /status endpoint. -# Sets the global VALIDATION_HTTP_CODE and returns 0 only when the key is valid -# (HTTP 200). This checks only the API key, nothing else about the install. +# Sets the globals VALIDATION_HTTP_CODE and VALIDATED_USER_EMAIL, and returns 0 +# only when the key is valid (HTTP 200). This checks only the API key, nothing +# else about the install. validate_api_key() { local key="$1" - local http_code - http_code=$(curl -s -o /dev/null -w "%{http_code}" \ + local response + response=$(curl -s -w $'\n%{http_code}' \ --max-time 30 \ -X POST "${CODEPLAIN_API_URL}/status" \ -H "Content-Type: application/json" \ --data "{\"api_key\":\"${key}\"}" 2>/dev/null || true) - VALIDATION_HTTP_CODE="${http_code:-000}" + + # The status code is appended on its own last line; everything above it is + # the JSON body. + VALIDATION_HTTP_CODE="$(printf '%s' "$response" | tail -n 1)" + VALIDATION_HTTP_CODE="${VALIDATION_HTTP_CODE:-000}" + + # /status reports the user this key belongs to. The email identifies the + # user in analytics (see capture_install_event). Matching on "email" with + # the leading quote skips "organization_owner_email", which is a different + # user in an organization. + local body + body="$(printf '%s' "$response" | sed '$d')" + VALIDATED_USER_EMAIL="$(printf '%s' "$body" \ + | grep -o '"email"[[:space:]]*:[[:space:]]*"[^"]*"' \ + | head -n 1 \ + | sed 's/.*"\([^"]*\)"$/\1/' || true)" + [ "$VALIDATION_HTTP_CODE" = "200" ] } +# Analytics honor the same CODEPLAIN_TELEMETRY opt-out as the CLI's crash +# reporting. The CLI defaults to off outside a real user install; running this +# installer *is* a real user install, so here the default is on. +telemetry_enabled() { + local setting + setting="$(printf '%s' "${CODEPLAIN_TELEMETRY:-}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')" + case "$setting" in + 0 | false | off) return 1 ;; + esac + return 0 +} + +# True only once the publish workflow (or the environment) has supplied a real +# PostHog token and host. Every PostHog project token starts with "phc_", so +# that prefix also tells a published installer apart from an unsubstituted one. +analytics_configured() { + case "$POSTHOG_PROJECT_TOKEN" in + phc_*) ;; + *) return 1 ;; + esac + case "$POSTHOG_CAPTURE_URL" in + https://*) ;; + *) return 1 ;; + esac + return 0 +} + +# Report the finished install to PostHog, keyed on the user's email so the +# install lands on the same person as their signup on the web app. Takes "true" +# or "false" for whether 'codeplain --status' ran successfully. +# Never fails the install: no email, an opt-out, or an unreachable PostHog are +# all silent no-ops. +capture_install_event() { + local install_verified="$1" + + if [ -z "$VALIDATED_USER_EMAIL" ] || ! telemetry_enabled || ! analytics_configured; then + return 0 + fi + + local installed_version noninteractive payload + installed_version="$(uv tool list 2>/dev/null | grep "^codeplain" | sed 's/codeplain v//' || true)" + if [ "$NONINTERACTIVE" = "1" ]; then + noninteractive=true + else + noninteractive=false + fi + + payload="$(printf '{"api_key":"%s","event":"cli_installed","distinct_id":"%s","properties":{"$lib":"codeplain-install-script","client_version":"%s","os":"%s","install_type":"%s","install_verified":%s,"plain_forge_installed":%s,"plyn_installed":%s,"noninteractive":%s}}' \ + "$POSTHOG_PROJECT_TOKEN" \ + "$VALIDATED_USER_EMAIL" \ + "$installed_version" \ + "$(uname -s)" \ + "$INSTALL_TYPE" \ + "$install_verified" \ + "${PLAIN_FORGE_INSTALLED:-false}" \ + "${PLYN_INSTALLED:-false}" \ + "$noninteractive")" + + curl -s -o /dev/null --max-time 2 \ + -X POST "$POSTHOG_CAPTURE_URL" \ + -H "Content-Type: application/json" \ + --data "$payload" > /dev/null 2>&1 || true +} + # Check if uv is installed if ! command -v uv &> /dev/null; then echo -e "${GRAY}uv is not installed.${NC}" @@ -121,6 +218,7 @@ echo -e "" # Install or upgrade codeplain using uv tool if uv tool list 2>/dev/null | grep -q "^codeplain"; then CURRENT_VERSION=$(uv tool list 2>/dev/null | grep "^codeplain" | sed 's/codeplain v//') + INSTALL_TYPE="upgrade" echo -e "${GRAY}codeplain ${CURRENT_VERSION} is already installed.${NC}" echo -e "Upgrading to latest version..." echo -e "" @@ -132,6 +230,7 @@ if uv tool list 2>/dev/null | grep -q "^codeplain"; then echo -e "${GREEN}✓${NC} codeplain upgraded from ${CURRENT_VERSION} to ${NEW_VERSION}!" fi else + INSTALL_TYPE="fresh" echo -e "Installing codeplain...${NC}" echo -e "" uv tool install codeplain @@ -445,6 +544,7 @@ if [ -n "${CODEPLAIN_API_KEY:-}" ]; then echo "$verify_output" echo -e "${GRAY}Please restart your terminal and try again, or reinstall with:${NC}" echo -e " uv tool install --force codeplain" + capture_install_event false exit 1 fi fi @@ -481,6 +581,9 @@ echo "" echo -e " ${GRAY}Happy development!${NC} 🚀" echo "" +# Reported last: exec below replaces this process, so nothing after it runs. +capture_install_event true + if [ "$NONINTERACTIVE" != "1" ]; then # Replace this subshell with a fresh shell that has the new environment # Reconnect stdin to terminal (needed when running via curl | bash) diff --git a/install/powershell/install.ps1 b/install/powershell/install.ps1 index 7f9f2e33..59841b1a 100644 --- a/install/powershell/install.ps1 +++ b/install/powershell/install.ps1 @@ -17,6 +17,22 @@ if (-not $env:CODEPLAIN_API_URL) { $env:CODEPLAIN_API_URL = "https://api.codeplain.ai" } +# PostHog analytics. Both values are substituted at publish time by the +# publish-install-script workflow, from the repository's POSTHOG_PROJECT_TOKEN +# secret and POSTHOG_CAPTURE_URL variable. A copy run straight from a clone +# keeps the placeholders and reports nothing, unless the environment supplies +# both values. +$POSTHOG_CAPTURE_URL = if ($env:CODEPLAIN_POSTHOG_CAPTURE_URL) { $env:CODEPLAIN_POSTHOG_CAPTURE_URL } else { "__POSTHOG_CAPTURE_URL__" } +$POSTHOG_PROJECT_TOKEN = if ($env:CODEPLAIN_POSTHOG_PROJECT_TOKEN) { $env:CODEPLAIN_POSTHOG_PROJECT_TOKEN } else { "__POSTHOG_PROJECT_TOKEN__" } + +# Email of the user the API key belongs to; set by Test-ApiKey. Analytics +# identify a user by email, so an install that never verified a key sends no +# event. +$validatedUserEmail = "" + +# "fresh" or "upgrade"; set when codeplain is installed below. +$installType = "" + # Brand Colors (True Color / 24-bit) $ESC = [char]27 $YELLOW = "$ESC[38;2;224;255;110m" # #E0FF6E @@ -161,6 +177,8 @@ function Assert-Git { # Verify an API key against the Codeplain API's /status endpoint. # Returns a status string: "valid" (HTTP 200), "invalid" (HTTP 401), or # "error" (could not reach the API). This checks only the API key. +# On success it also records the email of the user the key belongs to in +# $validatedUserEmail, which identifies the user in analytics. function Test-ApiKey { param([string]$Key) @@ -173,7 +191,15 @@ function Test-ApiKey { -TimeoutSec 30 ` -UseBasicParsing ` -ErrorAction Stop - if ($response.StatusCode -eq 200) { return "valid" } + if ($response.StatusCode -eq 200) { + # Analytics are optional; never fail key validation over them. + try { + $script:validatedUserEmail = ($response.Content | ConvertFrom-Json).user.email + } catch { + $script:validatedUserEmail = "" + } + return "valid" + } return "error" } catch { $statusCode = $null @@ -185,6 +211,63 @@ function Test-ApiKey { } } +# Analytics honor the same CODEPLAIN_TELEMETRY opt-out as the CLI's crash +# reporting. The CLI defaults to off outside a real user install; running this +# installer *is* a real user install, so here the default is on. +function Test-TelemetryEnabled { + $setting = "$($env:CODEPLAIN_TELEMETRY)".Trim().ToLower() + return @("0", "false", "off") -notcontains $setting +} + +# Report the finished install to PostHog, keyed on the user's email so the +# install lands on the same person as their signup on the web app. +# Never fails the install: no email, an opt-out, or an unreachable PostHog are +# all silent no-ops. +# True only once the publish workflow (or the environment) has supplied a real +# PostHog token and host. Every PostHog project token starts with "phc_", so +# that prefix also tells a published installer apart from an unsubstituted one. +function Test-AnalyticsConfigured { + return $POSTHOG_PROJECT_TOKEN.StartsWith("phc_") -and $POSTHOG_CAPTURE_URL.StartsWith("https://") +} + +function Send-InstallEvent { + param([bool]$InstallVerified) + + if (-not $validatedUserEmail -or -not (Test-TelemetryEnabled) -or -not (Test-AnalyticsConfigured)) { return } + + try { + $versionLine = @(uv tool list 2>$null) | Where-Object { $_ -match '^codeplain' } | Select-Object -First 1 + $installedVersion = ($versionLine -replace 'codeplain v', '').Trim() + + $payload = @{ + api_key = $POSTHOG_PROJECT_TOKEN + event = "cli_installed" + distinct_id = $validatedUserEmail + properties = @{ + '$lib' = "codeplain-install-script" + client_version = $installedVersion + # This installer is the Windows counterpart of install.sh, + # which reports the output of 'uname -s'. + os = "Windows" + install_type = $installType + install_verified = $InstallVerified + plain_forge_installed = $plainForgeInstalled + plyn_installed = $plynInstalled + noninteractive = $nonInteractive + } + } | ConvertTo-Json -Compress -Depth 4 + + Invoke-RestMethod -Uri $POSTHOG_CAPTURE_URL ` + -Method Post ` + -ContentType "application/json" ` + -Body $payload ` + -TimeoutSec 2 ` + -ErrorAction Stop | Out-Null + } catch { + # Analytics must never break the install. + } +} + # Check if uv is installed if (-not (Get-Command uv -ErrorAction SilentlyContinue)) { Write-Host "${GRAY}uv is not installed.${NC}" @@ -215,6 +298,7 @@ try { # Install or upgrade codeplain using uv tool $codeplainLine = $uvOutput | Where-Object { $_ -match '^codeplain' } | Select-Object -First 1 if ($codeplainLine) { + $installType = "upgrade" $currentVersion = ($codeplainLine -replace 'codeplain v', '').Trim() Write-Host "${GRAY}codeplain ${currentVersion} is already installed.${NC}" Write-Host "Upgrading to latest version..." @@ -237,6 +321,7 @@ if ($codeplainLine) { Write-Host "${GREEN}${CHECK}${NC} codeplain upgraded from ${currentVersion} to ${newVersion}!" } } else { + $installType = "fresh" Write-Host "Installing codeplain...${NC}" Write-Host "" uv tool install codeplain @@ -553,6 +638,7 @@ if ($env:CODEPLAIN_API_KEY) { Write-Host $verifyOutput Write-Host "${GRAY}Please restart your terminal and try again, or reinstall with:${NC}" Write-Host " uv tool install --force codeplain" + Send-InstallEvent $false Stop-Install return } @@ -588,6 +674,8 @@ Write-Host "" Write-Host " ${GRAY}Happy development!${NC} ${ROCKET}" Write-Host "" +Send-InstallEvent $true + # Refresh environment for this session # Unlike bash's exec "$SHELL", PowerShell doesn't need to restart the shell. # The API key is already set in $env:CODEPLAIN_API_KEY for this session, From 6ea971fe0c9f55d81d7877c527c713eeea9dfa42 Mon Sep 17 00:00:00 2001 From: Nejc Stebe Date: Fri, 21 Aug 2026 13:21:23 +0200 Subject: [PATCH 3/3] keep e2e runs out of PostHog and Sentry The e2e suite installs and renders for real with a real API key. Set CODEPLAIN_TELEMETRY=0 for the container and for the Windows runs so test installs and test renders do not show up as user activity. --- tests/e2e/conftest.py | 6 ++++++ tests/e2e/test_hello_world_python_windows.py | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index b3a2336e..50824e9c 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -49,6 +49,10 @@ def e2e_container(docker_image: str, api_key: str): f"CODEPLAIN_API_KEY={api_key}", "-e", "CODEPLAIN_INSTALL_NONINTERACTIVE=1", + # A test run is not a real install or a real render; keep its events + # out of PostHog and its crashes out of Sentry. + "-e", + "CODEPLAIN_TELEMETRY=0", docker_image, ], capture_output=True, @@ -120,6 +124,8 @@ def codeplain_exe(api_key: str) -> Path: **os.environ, "CODEPLAIN_API_KEY": api_key, "CODEPLAIN_INSTALL_NONINTERACTIVE": "1", + # A test run is not a real install; keep its events out of PostHog. + "CODEPLAIN_TELEMETRY": "0", } result = subprocess.run( ["pwsh", "-NoProfile", "-File", str(INSTALL_PS1)], diff --git a/tests/e2e/test_hello_world_python_windows.py b/tests/e2e/test_hello_world_python_windows.py index 49688aa2..2ec5eb76 100644 --- a/tests/e2e/test_hello_world_python_windows.py +++ b/tests/e2e/test_hello_world_python_windows.py @@ -18,7 +18,8 @@ def test_render_and_run_hello_world_python_windows(codeplain_exe: Path, api_key: str, tmp_path: Path): shutil.copy(EXAMPLE_PLAIN, tmp_path / "hello_world_python.plain") - env = {**os.environ, "CODEPLAIN_API_KEY": api_key} + # A test render is not a real render; keep its events out of PostHog. + env = {**os.environ, "CODEPLAIN_API_KEY": api_key, "CODEPLAIN_TELEMETRY": "0"} result = subprocess.run( [str(codeplain_exe), "--help"],