feat(embed): point the stack at an OpenTelemetry collector with one setting

GitOrigin-RevId: 59edca760c98daf8c8042b5c8bd4cf62f690139e

Author: svalleru

embed/Makefile

@@ -1,5 +1,5 @@
 SHELL := /bin/bash
-.PHONY: lint config test images stores-check vector-validate sync-configs kubernetes-render terraform-validate
+.PHONY: lint config test images stores-check vector-validate otel-validate sync-configs kubernetes-render terraform-validate
 
 # Every compose invocation runs against compose/, where compose.yaml and .env
 # live; the other two install shapes have their own directories beside it.
@@ -14,7 +14,7 @@ BATS_FILES := $(strip $(wildcard tests/*.bats))
 config:
 	$(COMPOSE) config -q
 
-lint: config vector-validate kubernetes-render terraform-validate
+lint: config vector-validate otel-validate kubernetes-render terraform-validate
 	@if [ -n "$(strip $(SHELL_FILES))" ]; then shellcheck -x $(SHELL_FILES); fi
 	@if [ -n "$(strip $(BATS_FILES))" ]; then shellcheck -x -s bash $(BATS_FILES); fi
 
@@ -38,9 +38,20 @@ vector-validate:
 	docker run --rm -v "$(CURDIR)/compose/config/vector/vector.toml:/etc/vector/vector.toml:ro" \
 	  timberio/vector:0.51.1-alpine validate --no-environment /etc/vector/vector.toml
 
-# After editing compose/config/clickhouse/config.xml or
-# compose/config/vector/vector.toml: rewrite the inline copies in
-# compose/compose.yaml and their SHA-256 stamps.
+# The collector parses the config and builds every pipeline without starting
+# them, so a pipeline naming a component the config does not define fails
+# here rather than on a host. The image is the one compose.yaml pins, and the
+# Kubernetes copy of the config is this file as is.
+OTEL_IMAGE := $(shell sed -n 's/^    image: \(otel\/opentelemetry-collector-contrib:[^ ]*\)$$/\1/p' compose/compose.yaml)
+otel-validate:
+	@test -n "$(OTEL_IMAGE)" || { echo "otel-validate: compose/compose.yaml pins no otel/opentelemetry-collector-contrib image" >&2; exit 1; }
+	docker run --rm -v "$(CURDIR)/compose/config/otel/otel-collector.yaml:/etc/otelcol-contrib/config.yaml:ro" \
+	  $(OTEL_IMAGE) validate --config=/etc/otelcol-contrib/config.yaml
+
+# After editing compose/config/clickhouse/config.xml,
+# compose/config/vector/vector.toml or compose/config/otel/otel-collector.yaml:
+# rewrite the inline copies in compose/compose.yaml, their SHA-256 stamps and
+# the Kubernetes copies under kubernetes/config/.
 sync-configs:
 	python3 compose/scripts/dev/sync-configs.py
 

embed/README.md

@@ -56,6 +56,11 @@ deployment see [e2b.dev/enterprise](https://e2b.dev/enterprise).
   each one. Those reach the sandbox through the header routing above, at
   the address in `E2B_DASHBOARD_HOST` (default `localhost`); set it when a
   browser on another machine opens the dashboard without a tunnel.
+- **OpenTelemetry out.** One setting sends the E2B services' metrics, traces
+  and logs to your collector. Point it at the built-in one, a second line
+  away, to keep the metrics the dashboard's charts draw. The
+  [reference](docs/REFERENCE.md#observability) says what each choice gives
+  you.
 - **One version everywhere.** Embed is released at the platform version once
   that release is tagged, and that release moves every platform pin in
   [`compose/.env`](compose/.env) and the kustomization. To pin an install, pin
@@ -90,9 +95,9 @@ network edge.
 | 5109 | api | the machine only | edge gRPC |
 
 Port 5008 is the one to guard most: nothing authenticates it, and anyone who
-reaches it has the whole orchestrator. The stores, Vector and the pprof
-endpoints stay on loopback; [What runs where](docs/REFERENCE.md#what-runs-where)
-lists them.
+reaches it has the whole orchestrator. The stores, Vector, the built-in
+collector and the pprof endpoints stay on loopback;
+[What runs where](docs/REFERENCE.md#what-runs-where) lists them.
 
 ## Not supported
 
@@ -116,13 +121,14 @@ lists them.
   tests.
 - `make test` runs the bats suite in `tests/`.
 - `make stores-check` runs the store-level integration check.
-- `make sync-configs` re-inlines the 2 configs into the compose file.
+- `make sync-configs` re-inlines the 3 configs into the compose file.
 
 What each target needs, and when the stack images have to be rebuilt, is in
 [Developing](docs/REFERENCE.md#developing).
 
 ## Reference
 
 [`docs/REFERENCE.md`](docs/REFERENCE.md) has the rest: what runs where and in
-which order, how logs reach ClickHouse, the secrets each shape holds, how
-images and pins are released, building templates, and the developer tooling.
+which order, how logs and metrics reach ClickHouse, the secrets each shape
+holds, how images and pins are released, building templates, and the
+developer tooling.

embed/compose/.env

@@ -29,6 +29,10 @@ RUNTIME_COMMIT=7278c2a380767c9989da73cf4c04b1af1b32da18
 #SANDBOX_ACCESS_TOKEN_HASH_SEED=
 # Optional: the address a browser uses to reach sandbox traffic through client-proxy (port 3002). Unset, the dashboard tells the browser http://localhost:3002, which is right when you open the dashboard on the machine itself or through an SSH tunnel. Opening it from another machine without a tunnel needs that machine's address here, then `up` again.
 #E2B_DASHBOARD_HOST=
+# Optional: the host:port of an OpenTelemetry collector that api, the orchestrator, client-proxy and dashboard-api send their metrics, traces and logs to over OTLP/gRPC (plaintext). Unset or empty, nothing is exported. 127.0.0.1:4317 is the built-in collector the next line starts; for a collector of your own, put its address here and leave the next line commented. Change either, then `up` again; to stop the built-in collector, comment both lines out and run `docker compose rm -sf otel-collector` before `up`.
+#E2B_OTEL_COLLECTOR_GRPC_ENDPOINT=127.0.0.1:4317
+# Optional: run the built-in collector (the otel-collector service), which keeps the e2b.* metrics in this stack's ClickHouse for the api's metrics endpoints and the dashboard's charts. Compose reads this variable itself.
+#COMPOSE_PROFILES=otel
 
 # The three stack images this package builds, published by the same release;
 # tests/pins.bats keeps them in step with kubernetes/kustomization.yaml and

embed/compose/README.md

@@ -179,7 +179,8 @@ refill loop spins and logs `no free slots` every few seconds on an idle host.
 `FORCE_REBUILD=1`, settable in the
 same two places, forces `base-template` to rebuild even when a `base` row
 already shows `ready`, for a stale or broken template. `TEAM_API_KEY`,
-`ADMIN_TOKEN` and `SANDBOX_ACCESS_TOKEN_HASH_SEED` are under Secrets below.
+`ADMIN_TOKEN` and `SANDBOX_ACCESS_TOKEN_HASH_SEED` are under Secrets below,
+and `E2B_OTEL_COLLECTOR_GRPC_ENDPOINT` under Telemetry.
 
 An optional, git-ignored `env/api.local.env` can add api variables. Create
 the `env/` directory next to `compose.yaml` yourself, since the two-file
@@ -329,14 +330,44 @@ for the api process only. Read them from the volume instead:
 `docker compose run --rm --no-deps --entrypoint cat
 api-secrets /run/e2b/api.env`.
 
+### Telemetry
+
+Nothing is exported by default, so the dashboard's monitoring charts and the
+SDK's `getMetrics` (`get_metrics` in Python) stay empty. To run the built-in
+OpenTelemetry collector, which keeps those metrics in this stack's
+ClickHouse, uncomment the two lines that ship commented at the end of the
+optional block in `.env`:
+
+```bash
+E2B_OTEL_COLLECTOR_GRPC_ENDPOINT=127.0.0.1:4317
+COMPOSE_PROFILES=otel
+```
+
+Then run `docker compose up -d --wait`, which starts `otel-collector` and
+recreates the four services that export to it; recreating the orchestrator
+ends the sandboxes running on it. While the collector runs,
+`curl -s 127.0.0.1:13133` answers `Server available`, and once a sandbox has
+been running for a few seconds its rows show up:
+`docker compose exec clickhouse clickhouse-client --user clickhouse --password clickhouse --query "SELECT count() FROM sandbox_metrics_gauge"`.
+
+For a collector of your own, uncomment only the first line and put that
+collector's address in it. To turn the built-in one off again, comment both
+lines out, run `docker compose rm -sf otel-collector` and then
+`docker compose up -d --wait`. The reference's
+[Observability](../docs/REFERENCE.md#observability) says what each choice
+gives you and how to forward traces and logs as well.
+
 ### Upgrading
 
 Download `compose.yaml` and `.env` again, with the same command as
-[Install](#install), and run `docker compose up -d --wait --pull always`. The
-`.env` you get pins the stack images that go with those files, so the two
-always move together. To take a particular commit rather than the newest, put
-the commit in place of `main` in the two URLs, as in
-`raw.githubusercontent.com/e2b-dev/runtime/<commit>/embed/compose/…`; the raw
+[Install](#install), and run `docker compose up -d --wait --pull always`.
+Before that `up`, carry the optional lines you had set, such as
+`E2B_DASHBOARD_HOST` or the two under [Telemetry](#telemetry), over into the
+new `.env`: it ships them commented again, and without the Telemetry pair the
+services stop exporting. The `.env` you get pins the stack images that go with
+those files, so the two always move together. To take a particular commit
+rather than the newest, put the commit in place of `main` in the two URLs, as
+in `raw.githubusercontent.com/e2b-dev/runtime/<commit>/embed/compose/…`; the raw
 host ignores `?ref=`. Binary checksums live inside the tools
 image, so bumping a Firecracker artifact means taking newer files rather than
 editing anything on the host. Running sandboxes end when the orchestrator
@@ -356,7 +387,9 @@ restarts; a single machine has nowhere to drain them to.
 - Expected log noise, all harmless: an OIDC warning from api at startup,
   because no identity provider is configured, and, every ten seconds from both
   api and the orchestrator, `failed to upload metrics: exporter export
-  timeout`, because no OTEL collector is configured and the endpoint is empty.
+  timeout`, because no OpenTelemetry collector is configured and the endpoint
+  is empty. That line stops once `E2B_OTEL_COLLECTOR_GRPC_ENDPOINT` names a
+  collector that answers ([Telemetry](#telemetry)).
   dashboard-api also logs `ADMIN_AUTH_PROVIDER_CONFIG is not configured` once
   at startup; the management endpoints it guards are not used here.
 - To run without the dashboard, remove the `dashboard` and `dashboard-api`

embed/compose/compose.yaml

@@ -5,10 +5,12 @@
 # variable. Without it Compose renders a blank image name or a blank kernel
 # version and only warns, and the operator meets either a confusing "neither
 # an image nor a build context" error or a fetch-artifacts failure after the
-# host has already been mutated. The eight knobs that are genuinely optional and
-# are absent from .env (TEAM_API_KEY, ADMIN_TOKEN, SANDBOX_ACCESS_TOKEN_HASH_SEED
-# and E2B_DASHBOARD_HOST only as commented lines) keep the ${VAR:-...} form: HUGEPAGES, NBDS_MAX, PF_MIN_FREE_GIB,
-# FORCE_REBUILD and E2B_DASHBOARD_HOST carry a working default, while
+# host has already been mutated. The nine knobs that are genuinely optional and
+# are absent from .env (TEAM_API_KEY, ADMIN_TOKEN, SANDBOX_ACCESS_TOKEN_HASH_SEED,
+# E2B_DASHBOARD_HOST and E2B_OTEL_COLLECTOR_GRPC_ENDPOINT only as commented
+# lines) keep the ${VAR:-...} form: HUGEPAGES, NBDS_MAX, PF_MIN_FREE_GIB,
+# FORCE_REBUILD, E2B_DASHBOARD_HOST and E2B_OTEL_COLLECTOR_GRPC_ENDPOINT carry a
+# working default (the last an empty one, which exports no telemetry), while
 # TEAM_API_KEY, ADMIN_TOKEN and SANDBOX_ACCESS_TOKEN_HASH_SEED carry none — the
 # stack generates all three per install (seed for the key, api-secrets for the
 # api's two), and a value set in .env or the shell overrides the generated one.
@@ -102,6 +104,30 @@ services:
       timeout: 2s
       retries: 12
 
+  # The built-in OpenTelemetry collector, run only with the otel profile:
+  # COMPOSE_PROFILES=otel in .env, which Compose reads itself. It receives
+  # OTLP/gRPC on 127.0.0.1:4317, the address to give
+  # E2B_OTEL_COLLECTOR_GRPC_ENDPOINT, and writes the e2b.* metrics into
+  # ClickHouse (config/otel/otel-collector.yaml says what it keeps). The image
+  # has no shell, so there is no healthcheck; `curl -s 127.0.0.1:13133` on the
+  # host is the check. No service depends on it, `ready` included.
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:0.146.0
+    profiles: [otel]
+    restart: unless-stopped
+    network_mode: host
+    depends_on:
+      clickhouse-migrator:
+        condition: service_completed_successfully
+    environment:
+      # sha256 of config/otel/otel-collector.yaml, same purpose as
+      # VECTOR_CONFIG_SHA256 on the vector service.
+      OTEL_CONFIG_SHA256: 96621f22e4e8ab2b420a6bc659c0b47239535f90627951aba449a309f614ce5f
+    command: ["--config=/etc/otelcol-contrib/config.yaml"]
+    configs:
+      - source: otel-collector-config
+        target: /etc/otelcol-contrib/config.yaml
+
   db-migrator:
     image: ${E2B_DB_MIGRATOR_IMAGE:?fetch the .env that ships beside compose.yaml (README, Install)}
     restart: "no"
@@ -240,6 +266,10 @@ services:
       REDIS_URL: 127.0.0.1:6379
       CLICKHOUSE_CONNECTION_STRING: clickhouse://clickhouse:clickhouse@127.0.0.1:9000/default
       LOGS_COLLECTOR_ADDRESS: http://127.0.0.1:30006
+      # Where this service and api, client-proxy and dashboard-api export their
+      # metrics, traces and logs over OTLP/gRPC. Empty unless .env sets
+      # E2B_OTEL_COLLECTOR_GRPC_ENDPOINT, and empty exports nothing.
+      OTEL_COLLECTOR_GRPC_ENDPOINT: ${E2B_OTEL_COLLECTOR_GRPC_ENDPOINT:-}
       TEMPLATE_STORAGE_URL: file:///var/lib/e2b/storage/templates
       BUILD_CACHE_STORAGE_URL: file:///var/lib/e2b/storage/build-cache
       ARTIFACTS_REGISTRY_PROVIDER: Local
@@ -304,6 +334,7 @@ services:
       # LOKI_URL, and tests/pins.bats refuses that pin.
       LOGS_READ_CONFIG: "true"
       LOGS_COLLECTOR_ADDRESS: http://127.0.0.1:30006
+      OTEL_COLLECTOR_GRPC_ENDPOINT: ${E2B_OTEL_COLLECTOR_GRPC_ENDPOINT:-}
       API_INTERNAL_GRPC_PORT: "5009"
       API_EDGE_GRPC_PORT: "5109"
       PPROF_PORT: "6060"
@@ -388,6 +419,7 @@ services:
       POSTGRES_CONNECTION_STRING: postgres://postgres:postgres@127.0.0.1:5432/postgres?sslmode=disable
       REDIS_URL: 127.0.0.1:6379
       CLICKHOUSE_CONNECTION_STRING: clickhouse://clickhouse:clickhouse@127.0.0.1:9000/default
+      OTEL_COLLECTOR_GRPC_ENDPOINT: ${E2B_OTEL_COLLECTOR_GRPC_ENDPOINT:-}
       AUTH_PROVIDER_CONFIG: '{"jwt":[]}'
     volumes:
       - seed-state:/run/e2b:ro
@@ -472,6 +504,7 @@ services:
       API_INTERNAL_GRPC_ADDRESS: 127.0.0.1:5009
       PROXY_PORT: "3002"
       HEALTH_PORT: "3003"
+      OTEL_COLLECTOR_GRPC_ENDPOINT: ${E2B_OTEL_COLLECTOR_GRPC_ENDPOINT:-}
     depends_on:
       api:
         condition: service_healthy
@@ -853,3 +886,93 @@ configs:
       skip_unknown_fields = false
       compression = "gzip"
       batch.timeout_secs = 1
+  otel-collector-config:
+    content: |
+      # The built-in OpenTelemetry collector. The services export to it once their
+      # collector endpoint is 127.0.0.1:4317. It keeps the e2b.* metrics and writes
+      # them into the stack's own ClickHouse, into the metrics_gauge and metrics_sum
+      # tables the clickhouse-migrator creates; the api's metrics endpoints read
+      # what those two feed. Traces, logs and the other metrics are dropped; the
+      # commented otlp/upstream exporter below forwards traces and logs instead.
+      # Every listener binds loopback: the collector shares the machine's network
+      # with the services, and nothing off the machine has a reason to reach it.
+      receivers:
+        otlp:
+          protocols:
+            grpc:
+              endpoint: 127.0.0.1:4317
+              max_recv_msg_size_mib: 100
+
+      processors:
+        # Batches traces and logs on their way to nop, or to otlp/upstream once
+        # that forwards them.
+        batch:
+          timeout: 5s
+
+        batch/clickhouse:
+          timeout: 5s
+          send_batch_size: 50000
+
+        filter/external_metrics:
+          metrics:
+            include:
+              match_type: regexp
+              metric_names:
+                - "e2b.*"
+
+      extensions:
+        # `curl -s 127.0.0.1:13133` on the machine answers while the collector runs.
+        # The image has no shell, so there is nothing to check from inside it.
+        health_check:
+          endpoint: 127.0.0.1:13133
+
+      exporters:
+        clickhouse:
+          endpoint: tcp://127.0.0.1:9000
+          database: default
+          username: clickhouse
+          password: clickhouse
+          async_insert: true
+          create_schema: false
+          metrics_tables:
+            gauge:
+              name: metrics_gauge
+            sum:
+              name: metrics_sum
+
+        # Accepts traces and logs and discards them. Without a pipeline for them the
+        # receiver would answer Unimplemented, and every service would log each
+        # batch it refused.
+        nop:
+
+        # To forward traces and logs to a collector of your own, uncomment this
+        # exporter, put that collector's OTLP/gRPC address in its endpoint, and
+        # name it in place of nop in the traces and logs pipelines at the end.
+        # otlp/upstream:
+        #   endpoint: collector.example.com:4317
+        #   tls:
+        #     insecure: true
+
+      service:
+        telemetry:
+          logs:
+            level: warn
+          # No internal metrics: by default the collector serves its own on
+          # 127.0.0.1:8888, one more listener on the machine, and refuses to start
+          # when something else holds that port.
+          metrics:
+            level: none
+        extensions: [health_check]
+        pipelines:
+          metrics/external:
+            receivers: [otlp]
+            processors: [filter/external_metrics, batch/clickhouse]
+            exporters: [clickhouse]
+          traces:
+            receivers: [otlp]
+            processors: [batch]
+            exporters: [nop]
+          logs:
+            receivers: [otlp]
+            processors: [batch]
+            exporters: [nop]