From 565eddd55f10e484b52ce1648448d1565b3d3aec Mon Sep 17 00:00:00 2001 From: ch4r10t33r Date: Fri, 6 Feb 2026 14:56:05 +0000 Subject: [PATCH 01/20] feat: add isAggregator flag to validator configuration Add support for configuring nodes as aggregators through validator-config.yaml. This allows selective designation of nodes to perform aggregation duties by setting isAggregator: true in the validator configuration. Changes: - Add isAggregator field (default: false) to all validators in both local and ansible configs - Update parse-vc.sh to extract and export isAggregator flag - Modify all client command scripts to pass --is-aggregator flag when enabled - Add isAggregator status to node information output --- ansible-devnet/genesis/validator-config.yaml | 7 +++++++ client-cmds/ethlambda-cmd.sh | 12 ++++++++++-- client-cmds/grandine-cmd.sh | 12 ++++++++++-- client-cmds/lantern-cmd.sh | 12 ++++++++++-- client-cmds/lighthouse-cmd.sh | 12 ++++++++++-- client-cmds/qlean-cmd.sh | 9 +++++++++ client-cmds/ream-cmd.sh | 12 ++++++++++-- client-cmds/zeam-cmd.sh | 12 ++++++++++-- local-devnet/genesis/validator-config.yaml | 7 +++++++ parse-vc.sh | 8 ++++++++ 10 files changed, 91 insertions(+), 12 deletions(-) diff --git a/ansible-devnet/genesis/validator-config.yaml b/ansible-devnet/genesis/validator-config.yaml index 81e80f5..091275a 100644 --- a/ansible-devnet/genesis/validator-config.yaml +++ b/ansible-devnet/genesis/validator-config.yaml @@ -14,6 +14,7 @@ validators: ip: "46.224.123.223" quic: 9001 metricsPort: 9095 + isAggregator: false count: 1 # number of indices for this node - name: "ream_0" @@ -25,6 +26,7 @@ validators: ip: "77.42.27.219" quic: 9001 metricsPort: 9095 + isAggregator: false devnet: 1 count: 1 @@ -36,6 +38,7 @@ validators: ip: "46.224.123.220" quic: 9001 metricsPort: 9095 + isAggregator: false count: 1 - name: "lantern_0" @@ -47,6 +50,7 @@ validators: ip: "46.224.135.177" quic: 9001 metricsPort: 9095 + isAggregator: false count: 1 - name: "lighthouse_0" @@ -58,6 +62,7 @@ validators: ip: "46.224.135.169" quic: 9001 metricsPort: 9095 + isAggregator: false count: 1 - name: "grandine_0" @@ -66,6 +71,7 @@ validators: ip: "37.27.250.20" quic: 9001 metricsPort: 9095 + isAggregator: false count: 1 - name: "ethlambda_0" @@ -74,4 +80,5 @@ validators: ip: "78.47.44.215" quic: 9001 metricsPort: 9095 + isAggregator: false count: 1 \ No newline at end of file diff --git a/client-cmds/ethlambda-cmd.sh b/client-cmds/ethlambda-cmd.sh index 50bd63c..a7967f2 100644 --- a/client-cmds/ethlambda-cmd.sh +++ b/client-cmds/ethlambda-cmd.sh @@ -4,6 +4,12 @@ binary_path="$scriptDir/../ethlambda/target/release/ethlambda" +# Set aggregator flag based on isAggregator value +aggregator_flag="" +if [ "$isAggregator" == "true" ]; then + aggregator_flag="--is-aggregator" +fi + # Command when running as binary node_binary="$binary_path \ --custom-network-config-dir $configDir \ @@ -11,7 +17,8 @@ node_binary="$binary_path \ --node-id $item \ --node-key $configDir/$item.key \ --metrics-address 0.0.0.0 \ - --metrics-port $metricsPort" + --metrics-port $metricsPort \ + $aggregator_flag" # Command when running as docker container node_docker="ghcr.io/lambdaclass/ethlambda:devnet2 \ @@ -20,6 +27,7 @@ node_docker="ghcr.io/lambdaclass/ethlambda:devnet2 \ --node-id $item \ --node-key /config/$item.key \ --metrics-address 0.0.0.0 \ - --metrics-port $metricsPort" + --metrics-port $metricsPort \ + $aggregator_flag" node_setup="docker" diff --git a/client-cmds/grandine-cmd.sh b/client-cmds/grandine-cmd.sh index cd92472..63c9de3 100644 --- a/client-cmds/grandine-cmd.sh +++ b/client-cmds/grandine-cmd.sh @@ -1,5 +1,11 @@ #!/bin/bash +# Set aggregator flag based on isAggregator value +aggregator_flag="" +if [ "$isAggregator" == "true" ]; then + aggregator_flag="--is-aggregator" +fi + node_binary="$grandine_bin \ --genesis $configDir/config.yaml \ --validator-registry-path $configDir/validators.yaml \ @@ -11,7 +17,8 @@ node_binary="$grandine_bin \ --metrics \ --http-address 0.0.0.0 \ --http-port $metricsPort \ - --hash-sig-key-dir $configDir/hash-sig-keys" + --hash-sig-key-dir $configDir/hash-sig-keys \ + $aggregator_flag" node_docker="sifrai/lean:devnet-2 \ --genesis /config/config.yaml \ @@ -24,7 +31,8 @@ node_docker="sifrai/lean:devnet-2 \ --metrics \ --http-address 0.0.0.0 \ --http-port $metricsPort \ - --hash-sig-key-dir /config/hash-sig-keys" + --hash-sig-key-dir /config/hash-sig-keys \ + $aggregator_flag" # choose either binary or docker node_setup="docker" diff --git a/client-cmds/lantern-cmd.sh b/client-cmds/lantern-cmd.sh index b918355..2a2940f 100755 --- a/client-cmds/lantern-cmd.sh +++ b/client-cmds/lantern-cmd.sh @@ -8,6 +8,12 @@ if [ -n "$devnet" ]; then devnet_flag="--devnet $devnet" fi +# Set aggregator flag based on isAggregator value +aggregator_flag="" +if [ "$isAggregator" == "true" ]; then + aggregator_flag="--is-aggregator" +fi + # Lantern's repo: https://github.com/Pier-Two/lantern node_binary="$scriptDir/lantern/build/lantern_cli \ --data-dir $dataDir/$item \ @@ -22,7 +28,8 @@ node_binary="$scriptDir/lantern/build/lantern_cli \ --metrics-port $metricsPort \ --http-port 5055 \ --log-level debug \ - --hash-sig-key-dir $configDir/hash-sig-keys" + --hash-sig-key-dir $configDir/hash-sig-keys \ + $aggregator_flag" node_docker="$LANTERN_IMAGE --data-dir /data \ --genesis-config /config/config.yaml \ @@ -36,7 +43,8 @@ node_docker="$LANTERN_IMAGE --data-dir /data \ --metrics-port $metricsPort \ --http-port 5055 \ --log-level debug \ - --hash-sig-key-dir /config/hash-sig-keys" + --hash-sig-key-dir /config/hash-sig-keys \ + $aggregator_flag" # choose either binary or docker node_setup="docker" diff --git a/client-cmds/lighthouse-cmd.sh b/client-cmds/lighthouse-cmd.sh index 1e129c2..219b0e1 100644 --- a/client-cmds/lighthouse-cmd.sh +++ b/client-cmds/lighthouse-cmd.sh @@ -3,6 +3,12 @@ # Metrics enabled by default metrics_flag="--metrics" +# Set aggregator flag based on isAggregator value +aggregator_flag="" +if [ "$isAggregator" == "true" ]; then + aggregator_flag="--is-aggregator" +fi + node_binary="$lighthouse_bin lean_node \ --datadir \"$dataDir/$item\" \ --config \"$configDir/config.yaml\" \ @@ -14,7 +20,8 @@ node_binary="$lighthouse_bin lean_node \ --socket-port $quicPort\ $metrics_flag \ --metrics-address 0.0.0.0 \ - --metrics-port $metricsPort" + --metrics-port $metricsPort \ + $aggregator_flag" node_docker="hopinheimer/lighthouse:latest lighthouse lean_node \ --datadir /data \ @@ -27,6 +34,7 @@ node_docker="hopinheimer/lighthouse:latest lighthouse lean_node \ --socket-port $quicPort\ $metrics_flag \ --metrics-address 0.0.0.0 \ - --metrics-port $metricsPort" + --metrics-port $metricsPort \ + $aggregator_flag" node_setup="docker" diff --git a/client-cmds/qlean-cmd.sh b/client-cmds/qlean-cmd.sh index 28de40b..3dd34c3 100644 --- a/client-cmds/qlean-cmd.sh +++ b/client-cmds/qlean-cmd.sh @@ -3,6 +3,13 @@ #-----------------------qlean setup---------------------- # expects "qlean" submodule or symlink inside "lean-quickstart" root directory # https://github.com/qdrvm/qlean-mini + +# Set aggregator flag based on isAggregator value +aggregator_flag="" +if [ "$isAggregator" == "true" ]; then + aggregator_flag="--is-aggregator" +fi + node_binary="$scriptDir/qlean/build/src/executable/qlean \ --modules-dir $scriptDir/qlean/build/src/modules \ --genesis $configDir/config.yaml \ @@ -15,6 +22,7 @@ node_binary="$scriptDir/qlean/build/src/executable/qlean \ --node-id $item --node-key $configDir/$privKeyPath \ --listen-addr /ip4/0.0.0.0/udp/$quicPort/quic-v1 \ --prometheus-port $metricsPort \ + $aggregator_flag \ -ldebug \ -ltrace" @@ -29,6 +37,7 @@ node_docker="qdrvm/qlean-mini:devnet-2 \ --node-id $item --node-key /config/$privKeyPath \ --listen-addr /ip4/0.0.0.0/udp/$quicPort/quic-v1 \ --prometheus-port $metricsPort \ + $aggregator_flag \ -ldebug \ -ltrace" diff --git a/client-cmds/ream-cmd.sh b/client-cmds/ream-cmd.sh index 9985c92..04bd8ec 100755 --- a/client-cmds/ream-cmd.sh +++ b/client-cmds/ream-cmd.sh @@ -4,6 +4,12 @@ # Metrics enabled by default metrics_flag="--metrics" +# Set aggregator flag based on isAggregator value +aggregator_flag="" +if [ "$isAggregator" == "true" ]; then + aggregator_flag="--is-aggregator" +fi + # modify the path to the ream binary as per your system node_binary="$scriptDir/../ream/target/release/ream --data-dir $dataDir/$item \ lean_node \ @@ -15,7 +21,8 @@ node_binary="$scriptDir/../ream/target/release/ream --data-dir $dataDir/$item \ $metrics_flag \ --metrics-address 0.0.0.0 \ --metrics-port $metricsPort \ - --http-address 0.0.0.0" + --http-address 0.0.0.0 \ + $aggregator_flag" node_docker="ghcr.io/reamlabs/ream:latest-devnet2 --data-dir /data \ lean_node \ @@ -27,7 +34,8 @@ node_docker="ghcr.io/reamlabs/ream:latest-devnet2 --data-dir /data \ $metrics_flag \ --metrics-address 0.0.0.0 \ --metrics-port $metricsPort \ - --http-address 0.0.0.0" + --http-address 0.0.0.0 \ + $aggregator_flag" # choose either binary or docker node_setup="docker" diff --git a/client-cmds/zeam-cmd.sh b/client-cmds/zeam-cmd.sh index 4b56d7a..e9fd36d 100644 --- a/client-cmds/zeam-cmd.sh +++ b/client-cmds/zeam-cmd.sh @@ -6,13 +6,20 @@ # Metrics enabled by default metrics_flag="--metrics_enable" +# Set aggregator flag based on isAggregator value +aggregator_flag="" +if [ "$isAggregator" == "true" ]; then + aggregator_flag="--is-aggregator" +fi + node_binary="$scriptDir/../zig-out/bin/zeam node \ --custom_genesis $configDir \ --validator_config $validatorConfig \ --data-dir $dataDir/$item \ --node-id $item --node-key $configDir/$item.key \ $metrics_flag \ - --api-port $metricsPort" + --api-port $metricsPort \ + $aggregator_flag" node_docker="--security-opt seccomp=unconfined blockblaz/zeam:devnet2 node \ --custom_genesis /config \ @@ -20,7 +27,8 @@ node_docker="--security-opt seccomp=unconfined blockblaz/zeam:devnet2 node \ --data-dir /data \ --node-id $item --node-key /config/$item.key \ $metrics_flag \ - --api-port $metricsPort" + --api-port $metricsPort \ + $aggregator_flag" # choose either binary or docker node_setup="docker" \ No newline at end of file diff --git a/local-devnet/genesis/validator-config.yaml b/local-devnet/genesis/validator-config.yaml index 7f99d48..72d3ad3 100644 --- a/local-devnet/genesis/validator-config.yaml +++ b/local-devnet/genesis/validator-config.yaml @@ -14,6 +14,7 @@ validators: ip: "127.0.0.1" quic: 9001 metricsPort: 8081 + isAggregator: false count: 1 # number of indices for this node - name: "ream_0" @@ -25,6 +26,7 @@ validators: ip: "127.0.0.1" quic: 9002 metricsPort: 8082 + isAggregator: false devnet: 1 count: 1 @@ -36,6 +38,7 @@ validators: ip: "127.0.0.1" quic: 9003 metricsPort: 8083 + isAggregator: false count: 1 - name: "lantern_0" @@ -47,6 +50,7 @@ validators: ip: "127.0.0.1" quic: 9004 metricsPort: 8084 + isAggregator: false count: 1 - name: "lighthouse_0" @@ -58,6 +62,7 @@ validators: ip: "127.0.0.1" quic: 9005 metricsPort: 8085 + isAggregator: false count: 1 - name: "grandine_0" @@ -66,6 +71,7 @@ validators: ip: "127.0.0.1" quic: 9006 metricsPort: 8086 + isAggregator: false count: 1 - name: "ethlambda_0" @@ -77,4 +83,5 @@ validators: ip: "127.0.0.1" quic: 9007 metricsPort: 8087 + isAggregator: false count: 1 diff --git a/parse-vc.sh b/parse-vc.sh index 8b50e82..629f4e1 100644 --- a/parse-vc.sh +++ b/parse-vc.sh @@ -51,6 +51,12 @@ if [ -z "$devnet" ] || [ "$devnet" == "null" ]; then devnet="" fi +# Automatically extract isAggregator flag using yq (defaults to false if not set) +isAggregator=$(yq eval ".validators[] | select(.name == \"$item\") | .isAggregator // false" "$validator_config_file") +if [ -z "$isAggregator" ] || [ "$isAggregator" == "null" ]; then + isAggregator="false" +fi + # Automatically extract private key using yq privKey=$(yq eval ".validators[] | select(.name == \"$item\") | .privkey" "$validator_config_file") @@ -99,10 +105,12 @@ if [ "$keyType" == "hash-sig" ] && [ "$hashSigKeyIndex" != "null" ] && [ -n "$ha echo "Hash-Sig Key Index: $hashSigKeyIndex" echo "Hash-Sig Public Key: $hashSigPkPath" echo "Hash-Sig Secret Key: $hashSigSkPath" + echo "Is Aggregator: $isAggregator" else echo "Node: $item" echo "QUIC Port: $quicPort" echo "Metrics Port: $metricsPort" echo "Devnet: ${devnet:-}" echo "Private Key File: $privKeyPath" + echo "Is Aggregator: $isAggregator" fi From da7b5e480266d8213f927914665ba262c97607f7 Mon Sep 17 00:00:00 2001 From: ch4r10t33r Date: Sun, 17 May 2026 09:19:45 +0100 Subject: [PATCH 02/20] ansible-devnet: 8-subnet homogeneous validator layout and full leanpoint upstreams Regenerate validator-config.yaml for 64 validators across 8 attestation subnets (one client family per subnet). Aggregators sit on dedicated aggregator hosts; regular validators use the Validator_servers IP pool. Leanpoint convert/sync now emits one upstream per validator by default (removed per-subnet cap of two). Optional --subnet-sample restores the legacy subset behavior. Ansible localhost plays that use add_host force strategy: linear so they work with ansible.cfg strategy=free on large devnets. --- README.md | 2 +- ansible-devnet/genesis/validator-config.yaml | 572 ++++++++++++++++--- ansible/playbooks/clean-node-data.yml | 4 + ansible/playbooks/copy-genesis.yml | 2 + ansible/playbooks/deploy-nodes.yml | 2 + ansible/playbooks/stop-nodes.yml | 2 + convert-validator-config.py | 135 ++++- sync-leanpoint-upstreams.sh | 12 +- 8 files changed, 656 insertions(+), 75 deletions(-) diff --git a/README.md b/README.md index 71700c3..ee1e4a7 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ After validator nodes are spun up, leanpoint is deployed so it can monitor them. 1. `convert-validator-config.py` reads `validator-config.yaml` and generates `upstreams.json` (validator URLs for health checks). 2. `sync-leanpoint-upstreams.sh` either deploys leanpoint locally (local devnet) or syncs to the tooling server and recreates the remote container (Ansible). -**Remote defaults:** Tooling server `46.225.10.32`, user `root`, remote path `/etc/leanpoint/upstreams.json`, container name `leanpoint`, published port **5555→5555** (`LEANPOINT_HOST_PORT`, default **5555**). Override with env vars (see script header in `sync-leanpoint-upstreams.sh`). +**Remote defaults:** Tooling server `46.225.10.32`, user `root`, remote path `/etc/leanpoint/upstreams.json`, container name `leanpoint`, published port **5555→5555** (`LEANPOINT_HOST_PORT`, default **5555**). **`LEANPOINT_IMAGE`** defaults to **`0xpartha/leanpoint:latest`** (do not use **`blockblaz/leanpoint:latest`** on the shared tooling VM unless that image is verified on the host CPU). Override with env vars (see script header in `sync-leanpoint-upstreams.sh`). **SSH key for remote sync:** When using Ansible deployment, the tooling server may require a specific SSH key. Pass `--sshKey ~/.ssh/id_ed25519_github` (or `--private-key`) so the sync can succeed. diff --git a/ansible-devnet/genesis/validator-config.yaml b/ansible-devnet/genesis/validator-config.yaml index cc283cf..4b1ca25 100644 --- a/ansible-devnet/genesis/validator-config.yaml +++ b/ansible-devnet/genesis/validator-config.yaml @@ -1,159 +1,597 @@ +# 8-subnet ansible devnet — homogeneous clients (one client family per subnet). +# +# Genesis order = YAML list order → validator_index = row index (0..63). +# attestation_committee_count: 8 → subnet = index % 8. +# Row pattern repeats qlean → lantern → ream → zeam → ethlambda → gean → grandine → nlean. +# Aggregators: first validator on each subnet (indices 0–7), on dedicated aggregator hosts. +# +# Regular validators use IPs from lean_ethereum_servers.txt (Validator_servers). +# Aggregators use dedicated Aggregator_servers (one IP per client type). +# Tooling host (46.225.10.32) is not used. +# +# Ports (per IP): up to four processes per host — quic 9001..9004, api 5055..5058, +# metrics 9102..9105. Avoid observability TCP ports on ansible hosts: 9090 prometheus, +# 9080 promtail, 9098 cadvisor, 9100 node_exporter. shuffle: roundrobin -# devnet-4: 16 nodes, one validator each (genesis order = list order). -# Attestation subnet per validator: validator_index % attestation_committee_count (2). -# Optional YAML `subnet:` is not read by clients — only spin-node / Ansible use -# inferred subnets (same formula) for grouping and one-aggregator-per-subnet. -# One validator per server (ports 9001 / 9095 / 5055). Preset aggregators: -# ethlambda_0 (index 5, committee 1), qlean_1 (index 10, committee 0). deployment_mode: ansible config: activeEpoch: 18 keyType: "hash-sig" - attestation_committee_count: 2 + attestation_committee_count: 8 validators: - # ── genesis validator indices 0–7 ───────────────────────────────────────── + - name: "qlean_0" + privkey: "8e9f81c9caa9e29d26a7327311ca63a38254efdfccf3ce1362bae47eae0b18b3" + enrFields: + ip: "178.105.22.187" + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: true + count: 1 + - name: "lantern_0" + privkey: "d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5" + enrFields: + ip: "178.105.124.135" + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: true + count: 1 + - name: "ream_0" + privkey: "af27950128b49cda7e7bc9fcb7b0270f7a3945aa7543326f3bfdbd57d2a97a32" + enrFields: + ip: "178.105.117.56" + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: true + count: 1 - name: "zeam_0" privkey: "bdf953adc161873ba026330c56450453f582e3c4ee6cb713644794bcfdd85fe5" + enrFields: + ip: "178.105.121.214" + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: true + count: 1 + - name: "ethlambda_0" + privkey: "299550529a79bc2dce003747c52fb0639465c893e00b0440ac66144d625e066a" + enrFields: + ip: "178.105.22.193" + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: true + count: 1 + - name: "gean_0" + privkey: "69c251cdb06039dd99d87e5a1439fa3720615be98c293ec9bcfd041877a2e8ca" + enrFields: + ip: "178.105.70.184" + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: true + count: 1 + - name: "grandine_0" + privkey: "c05937b251889e35c58d4601c29bed8153dc22c548448f85e0ab9ca436d4b904" + enrFields: + ip: "178.105.113.166" + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: true + count: 1 + - name: "nlean_0" + privkey: "2e9be3f1b0d32ca3a4d62017fbfafe3950b7e90fed6802ff8bd2e0f8c4e2ca91" + enrFields: + ip: "178.105.102.228" + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: true + count: 1 + - name: "qlean_1" + privkey: "0e190e06a62db01bf566af4348277463d26eebab8f6badbcb989242ea4fee050" + enrFields: + ip: "37.27.220.14" + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 + - name: "lantern_1" + privkey: "1176609530e568ec12ad227e60ae71c618bfb13fde5a3257e4b0627216e78e04" + enrFields: + ip: "157.180.116.162" + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 + - name: "ream_1" + privkey: "fc2f11f90dd90df33bfb5f3467c2cbc37cf93dbb41c7ac556f9eea117418e73d" + enrFields: + ip: "204.168.178.179" + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 + - name: "zeam_1" + privkey: "8d14c7b02d55ca050ef97a3961aa16828837fb363e4e19e4dd0060f58670a2b3" + enrFields: + ip: "204.168.190.188" + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 + - name: "ethlambda_1" + privkey: "eb8533a0d5071d4dbcb0c4fcf9b8ac6edc3d1a260d2bb348fafc5cdb455aa1d4" + enrFields: + ip: "135.181.82.109" + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 + - name: "gean_1" + privkey: "5408a68b960a7932e367b32489498a13c339f87ff8090ea54213524b3d76fcff" + enrFields: + ip: "157.90.254.146" + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 + - name: "grandine_1" + privkey: "ab6edee1173379f647b4022d74d4b3342d547e7bb6954664a1a489b95b7c9b60" + enrFields: + ip: "37.27.89.135" + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 + - name: "nlean_1" + privkey: "74388a2487b943d947c17bf65ca08470f3ad5f045ea0f5aed38a98bbaba1bd49" + enrFields: + ip: "157.180.20.55" + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 + - name: "qlean_2" + privkey: "7fdceb8166b9d7ab4169d80345ab85fd042b3684ba307c4c02e851c1520975b5" + enrFields: + ip: "178.104.151.50" + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 + - name: "lantern_2" + privkey: "943151f2327ea91357632dd8a3a498242b07eeeebad973ea82d6bead80627d3e" + enrFields: + ip: "178.104.133.162" + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 + - name: "ream_2" + privkey: "d723f0cc94645848bbacf8aea4e01944e2f4a6f9f2c69b17ed1868f492374a7a" + enrFields: + ip: "178.104.149.208" + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 + - name: "zeam_2" + privkey: "11ff3a52375898532680c86739dbe0b632088545d0b901181d92053b5fab8d38" + enrFields: + ip: "178.104.149.91" + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 + - name: "ethlambda_2" + privkey: "4a7c5b2077d0e3c6f5322d0f2fd98cb2efffbceb41de2799e520b48872dc4102" enrFields: ip: "95.216.154.185" quic: 9001 - metricsPort: 9095 + metricsPort: 9102 apiPort: 5055 isAggregator: false count: 1 - - name: "ream_0" - privkey: "af27950128b49cda7e7bc9fcb7b0270f7a3945aa7543326f3bfdbd57d2a97a32" + - name: "gean_2" + privkey: "14da65451d15aee0cbdbcd847cfc3474f106c4595bf716306832fc078858f458" enrFields: ip: "204.168.135.7" quic: 9001 - metricsPort: 9095 + metricsPort: 9102 apiPort: 5055 isAggregator: false count: 1 - - name: "qlean_0" - privkey: "8e9f81c9caa9e29d26a7327311ca63a38254efdfccf3ce1362bae47eae0b18b3" + - name: "grandine_2" + privkey: "b4fdf4bc8ea6e74845b19823510c590406ff63371e0573132f1ddb6b455b7bf7" enrFields: ip: "65.21.182.45" quic: 9001 - metricsPort: 9095 + metricsPort: 9102 apiPort: 5055 isAggregator: false count: 1 - - name: "lantern_0" - privkey: "d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5" + - name: "nlean_2" + privkey: "53f1956c292eca6aba53bfd1089682828130db1c0ef980746e5e3e1d94f98360" enrFields: ip: "65.109.131.177" quic: 9001 - metricsPort: 9095 + metricsPort: 9102 apiPort: 5055 isAggregator: false count: 1 - - name: "grandine_0" - privkey: "c05937b251889e35c58d4601c29bed8153dc22c548448f85e0ab9ca436d4b904" + - name: "qlean_3" + privkey: "1027ce52661ef76b5ada68bc8cd45c13ebefb09c99dcad251b9bd4893c71815f" enrFields: ip: "65.109.138.213" quic: 9001 - metricsPort: 9095 + metricsPort: 9102 apiPort: 5055 isAggregator: false count: 1 - - name: "ethlambda_0" - privkey: "299550529a79bc2dce003747c52fb0639465c893e00b0440ac66144d625e066a" + - name: "lantern_3" + privkey: "31258c1b84e4a64954fb18dbb04ec9de7079ad4d0273b40380aa1f4ca2f20804" enrFields: ip: "204.168.134.201" quic: 9001 - metricsPort: 9095 + metricsPort: 9102 apiPort: 5055 - isAggregator: true + isAggregator: false count: 1 - - name: "gean_0" - privkey: "69c251cdb06039dd99d87e5a1439fa3720615be98c293ec9bcfd041877a2e8ca" + - name: "ream_3" + privkey: "4701dd38aa11a1affd69b5d95e535210f8c29e720222b906c46bd64cf018f9c7" enrFields: ip: "95.217.19.42" quic: 9001 - metricsPort: 9095 + metricsPort: 9102 apiPort: 5055 isAggregator: false count: 1 - - name: "nlean_0" - privkey: "2e9be3f1b0d32ca3a4d62017fbfafe3950b7e90fed6802ff8bd2e0f8c4e2ca91" + - name: "zeam_3" + privkey: "877c1489e75914bd46dccb71e0ee2d32af337fbb82f2c9caea73818e00ea9a61" enrFields: ip: "95.216.173.151" quic: 9001 - metricsPort: 9095 + metricsPort: 9102 apiPort: 5055 isAggregator: false count: 1 - # ── genesis validator indices 8–15 ──────────────────────────────────────── - - name: "zeam_1" - privkey: "8d14c7b02d55ca050ef97a3961aa16828837fb363e4e19e4dd0060f58670a2b3" + - name: "ethlambda_3" + privkey: "1c454b0399cd2178b46e42c8b599e34f2e7ddab71df96b1fbf510d5951335ea0" enrFields: ip: "95.216.164.165" quic: 9001 - metricsPort: 9095 + metricsPort: 9102 apiPort: 5055 isAggregator: false count: 1 - - name: "ream_1" - privkey: "fc2f11f90dd90df33bfb5f3467c2cbc37cf93dbb41c7ac556f9eea117418e73d" + - name: "gean_3" + privkey: "51832ccf0a189ed39a6fe13833894445b1fa60fc008fcb7fd7e30405318068d1" enrFields: ip: "95.216.165.186" quic: 9001 - metricsPort: 9095 + metricsPort: 9102 apiPort: 5055 isAggregator: false count: 1 - - name: "qlean_1" - privkey: "0e190e06a62db01bf566af4348277463d26eebab8f6badbcb989242ea4fee050" + - name: "grandine_3" + privkey: "27b642b91b25a4dd55638d7936469270f0fc4b77ceba768617291415eeb3ab1a" enrFields: ip: "37.27.250.20" quic: 9001 - metricsPort: 9095 + metricsPort: 9102 apiPort: 5055 - isAggregator: true + isAggregator: false count: 1 - - name: "lantern_1" - privkey: "1176609530e568ec12ad227e60ae71c618bfb13fde5a3257e4b0627216e78e04" + - name: "nlean_3" + privkey: "f52f1216a9a20f5b2c200ed64f1a2a8e7427772574147a7edf4602b40f820e9b" + enrFields: + ip: "37.27.220.14" + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 + - name: "qlean_4" + privkey: "0763e1a9f1909df7eae31ef4e93d5aaf45a506624ef81df893fc02e08a4ab94f" + enrFields: + ip: "157.180.116.162" + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 + - name: "lantern_4" + privkey: "a5dac14f317084d255c57e5080985b1dbd79cf8fc625a9c7fba781abf07001bd" + enrFields: + ip: "204.168.178.179" + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 + - name: "ream_4" + privkey: "62e6c79311ce4ee1b95ff52c8d6b7cccea0bf0b927fbed595d454858b32c49d6" + enrFields: + ip: "204.168.190.188" + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 + - name: "zeam_4" + privkey: "2aed04fc149298af1376aee0161d5dc67f9d7a47e625a58ae8e44f210d45c2a2" enrFields: ip: "135.181.82.109" - quic: 9001 - metricsPort: 9095 - apiPort: 5055 + quic: 9002 + metricsPort: 9103 + apiPort: 5056 isAggregator: false count: 1 - - name: "grandine_1" - privkey: "ab6edee1173379f647b4022d74d4b3342d547e7bb6954664a1a489b95b7c9b60" + - name: "ethlambda_4" + privkey: "91c2c932a6589e130bfa5281cc27f6b9930ba7f99aa043d334f14578ef7492f1" + enrFields: + ip: "157.90.254.146" + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 + - name: "gean_4" + privkey: "9b678649edf603b73913e8957fd7c26fe6af08656b8fe14b0d47aa80d3b59c9f" enrFields: ip: "37.27.89.135" - quic: 9001 - metricsPort: 9095 - apiPort: 5055 + quic: 9002 + metricsPort: 9103 + apiPort: 5056 isAggregator: false count: 1 - - name: "ethlambda_1" - privkey: "eb8533a0d5071d4dbcb0c4fcf9b8ac6edc3d1a260d2bb348fafc5cdb455aa1d4" + - name: "grandine_4" + privkey: "936fa39e4f1ce548d977f852f84ee900bbd2789bf85779e0e958d966a1ba7386" enrFields: ip: "157.180.20.55" - quic: 9001 - metricsPort: 9095 - apiPort: 5055 + quic: 9002 + metricsPort: 9103 + apiPort: 5056 isAggregator: false count: 1 - - name: "gean_1" - privkey: "5408a68b960a7932e367b32489498a13c339f87ff8090ea54213524b3d76fcff" + - name: "nlean_4" + privkey: "6cc68faf32d824996ecec2644c61cdb479da9bc1f957f7d09913fac8c49fe578" enrFields: ip: "178.104.151.50" - quic: 9001 - metricsPort: 9095 - apiPort: 5055 + quic: 9002 + metricsPort: 9103 + apiPort: 5056 isAggregator: false count: 1 - - name: "nlean_1" - privkey: "74388a2487b943d947c17bf65ca08470f3ad5f045ea0f5aed38a98bbaba1bd49" + - name: "qlean_5" + privkey: "ce66f19a179bd74e01b775c696b1744060078fe4d84abb5761a3f1d7cef15562" enrFields: ip: "178.104.133.162" - quic: 9001 - metricsPort: 9095 - apiPort: 5055 + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 + - name: "lantern_5" + privkey: "12bb3c043f2600f27be6bf8916368e9ff01b9ef5c4a7ccdf6aab8e2a24ba55c3" + enrFields: + ip: "178.104.149.208" + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 + - name: "ream_5" + privkey: "4caa321fc7598a3da98623a03774ee177c0025fa783fc2fc9c082b297ed96433" + enrFields: + ip: "178.104.149.91" + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 + - name: "zeam_5" + privkey: "34cfea8f2e6d6f8a28ae8869220450b8069ba0c9cca436a1079d44fa40989820" + enrFields: + ip: "95.216.154.185" + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 + - name: "ethlambda_5" + privkey: "793a5d9add590b890085b440ceefe3e8c805785f7271feee09d35511e7060056" + enrFields: + ip: "204.168.135.7" + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 + - name: "gean_5" + privkey: "b55c8e411d54ca0ef5ce69e86cba09ae110add1c5d598ed3356adfa5182b82e9" + enrFields: + ip: "65.21.182.45" + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 + - name: "grandine_5" + privkey: "99f1df1cbc7429360d52142a8ec5df6a3ad38a4a39897d2ee356b88fd3a69385" + enrFields: + ip: "65.109.131.177" + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 + - name: "nlean_5" + privkey: "9cf22e42d463e751958eceace0a3201c661b4326076027c900a04aa845a9e683" + enrFields: + ip: "65.109.138.213" + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 + - name: "qlean_6" + privkey: "aaf15d4a12fec7409b136d022790d9aaaca3d5679c9e28babe23a424cd80729b" + enrFields: + ip: "204.168.134.201" + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 + - name: "lantern_6" + privkey: "7536676b6ee5f0062030c0b6784c44a175d66cdc39277354c1b9014b419cbc30" + enrFields: + ip: "95.217.19.42" + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 + - name: "ream_6" + privkey: "5401bb0acca1620ffddd776c98b5de53034b960eb84a1fe0ce381161789fd980" + enrFields: + ip: "95.216.173.151" + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 + - name: "zeam_6" + privkey: "2ac17fb2c419558e349400f45fd5077eac9ca7bcd527b0833ac7f630465c5457" + enrFields: + ip: "95.216.164.165" + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 + - name: "ethlambda_6" + privkey: "e673e0a9b6ea018ae0b8039019bcee829037dd5638c2b5a5f085385e46231b6a" + enrFields: + ip: "95.216.165.186" + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 + - name: "gean_6" + privkey: "0db45359918ec742bd1c354af1c5614969985139d2617790dc2ada928e95a731" + enrFields: + ip: "37.27.250.20" + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 + - name: "grandine_6" + privkey: "eadfc14b6231cb9a207f96f436f7d55c733ba3781306307b7f81f564e2b53f0a" + enrFields: + ip: "37.27.220.14" + quic: 9003 + metricsPort: 9104 + apiPort: 5057 + isAggregator: false + count: 1 + - name: "nlean_6" + privkey: "c9dcdad85cfffc846c9771f950f4198ae4791972cf86841ee01004f8375b9332" + enrFields: + ip: "157.180.116.162" + quic: 9003 + metricsPort: 9104 + apiPort: 5057 + isAggregator: false + count: 1 + - name: "qlean_7" + privkey: "2907bc1d1e0a3dee4cc317dfda39a09b5e23596671d7108482edb897616c6b6c" + enrFields: + ip: "204.168.178.179" + quic: 9003 + metricsPort: 9104 + apiPort: 5057 + isAggregator: false + count: 1 + - name: "lantern_7" + privkey: "98da18b276aa5a54ceba15e78260286b9dbe03c4bf8b9db061d024ba4446cfff" + enrFields: + ip: "204.168.190.188" + quic: 9003 + metricsPort: 9104 + apiPort: 5057 + isAggregator: false + count: 1 + - name: "ream_7" + privkey: "a8a9cb1f243380d915e4997b3bc5058295d94704870885d5ed6cb1dfea24d77e" + enrFields: + ip: "135.181.82.109" + quic: 9003 + metricsPort: 9104 + apiPort: 5057 + isAggregator: false + count: 1 + - name: "zeam_7" + privkey: "97ae5f1d63fa7c05783f7ae5af4ce1fc43a030833c174e430080a8695ce91d56" + enrFields: + ip: "157.90.254.146" + quic: 9003 + metricsPort: 9104 + apiPort: 5057 + isAggregator: false + count: 1 + - name: "ethlambda_7" + privkey: "17bea5ada768f12d6f0c95ab9618277527e9691e2e4169e73bc92248da122e68" + enrFields: + ip: "37.27.89.135" + quic: 9003 + metricsPort: 9104 + apiPort: 5057 + isAggregator: false + count: 1 + - name: "gean_7" + privkey: "32e4b103c4b349df214d265db9cb1e6d49346e14093fee656828a5e1ef050b8b" + enrFields: + ip: "157.180.20.55" + quic: 9003 + metricsPort: 9104 + apiPort: 5057 + isAggregator: false + count: 1 + - name: "grandine_7" + privkey: "7dd9ca62d79af8e01c95c743ef1a39155a42eded7096599a7f67b3932b387e6c" + enrFields: + ip: "178.104.151.50" + quic: 9003 + metricsPort: 9104 + apiPort: 5057 + isAggregator: false + count: 1 + - name: "nlean_7" + privkey: "c99cd68f66a7460a8aca095d6dd4b6ce953e175d91077f1e7aba27858f68e301" + enrFields: + ip: "178.104.133.162" + quic: 9003 + metricsPort: 9104 + apiPort: 5057 isAggregator: false count: 1 diff --git a/ansible/playbooks/clean-node-data.yml b/ansible/playbooks/clean-node-data.yml index adfbe0c..3b9ec70 100644 --- a/ansible/playbooks/clean-node-data.yml +++ b/ansible/playbooks/clean-node-data.yml @@ -4,6 +4,10 @@ - name: Parse and validate node names hosts: localhost connection: local + # ansible.cfg sets strategy=free for large devnets; add_host is not + # supported under free (it would run once per inventory host). Linear + # keeps this localhost-only play correct while building clean_targets. + strategy: linear gather_facts: no vars: validator_config_file: "{{ local_validator_config_path | default(genesis_dir + '/validator-config.yaml') }}" diff --git a/ansible/playbooks/copy-genesis.yml b/ansible/playbooks/copy-genesis.yml index 41051a3..dee7e71 100644 --- a/ansible/playbooks/copy-genesis.yml +++ b/ansible/playbooks/copy-genesis.yml @@ -15,6 +15,8 @@ - name: Copy Genesis Files to Remote Hosts hosts: all:!local + # ansible.cfg uses strategy=free; add_host is not valid under free. + strategy: linear gather_facts: yes vars: validator_config_file: "{{ genesis_dir }}/validator-config.yaml" diff --git a/ansible/playbooks/deploy-nodes.yml b/ansible/playbooks/deploy-nodes.yml index 30ac134..c70a9d0 100644 --- a/ansible/playbooks/deploy-nodes.yml +++ b/ansible/playbooks/deploy-nodes.yml @@ -13,6 +13,8 @@ - name: Parse and validate node names hosts: localhost connection: local + # ansible.cfg uses strategy=free; add_host is not valid under free. + strategy: linear gather_facts: yes tags: - deploy diff --git a/ansible/playbooks/stop-nodes.yml b/ansible/playbooks/stop-nodes.yml index 8c6c880..aae148d 100644 --- a/ansible/playbooks/stop-nodes.yml +++ b/ansible/playbooks/stop-nodes.yml @@ -5,6 +5,8 @@ - name: Parse and validate node names hosts: localhost connection: local + # ansible.cfg uses strategy=free; add_host is not valid under free. + strategy: linear gather_facts: yes vars: validator_config_file: "{{ local_validator_config_path | default(genesis_dir + '/validator-config.yaml') }}" diff --git a/convert-validator-config.py b/convert-validator-config.py index 823ad5a..38111cc 100644 --- a/convert-validator-config.py +++ b/convert-validator-config.py @@ -18,6 +18,10 @@ Options: --docker Use host.docker.internal so a container on the host can reach validators on the host (local devnet + Docker). + --all-upstreams Emit one upstream per validator (default; kept for compatibility). + --subnet-sample Legacy: at most N validators per attestation subnet (see + LEANPOINT_UPSTREAMS_PER_SUBNET, default 2). Not used unless this flag + is set. Examples: python3 convert-validator-config.py \\ @@ -33,9 +37,81 @@ upstreams-local-docker.json --docker """ +import os import sys import json import yaml +from collections import defaultdict +from typing import Any, Optional + + +def _attestation_committee_count(config: dict) -> Optional[int]: + """Subnet count from config.config.attestation_committee_count, or None if unset/invalid.""" + cfg = config.get("config") + if not isinstance(cfg, dict): + return None + raw = cfg.get("attestation_committee_count") + if raw is None: + return None + try: + n = int(raw) + except (TypeError, ValueError): + return None + if n < 1: + return None + return n + + +def _select_validators_for_leanpoint( + validators: list[dict[str, Any]], + subnet_count: Optional[int], + *, + per_subnet: int, + all_upstreams: bool, +) -> list[tuple[int, dict[str, Any]]]: + """ + Return (global_index, validator) rows to expose as leanpoint upstreams. + + When subnet_count is set and all_upstreams is False, keep at most `per_subnet` + validators per attestation subnet (index % subnet_count). Each subnet includes + the first validator with isAggregator: true in YAML order when present, then + fills remaining slots with the next validators in that subnet not yet chosen. + """ + if all_upstreams or subnet_count is None: + return list(enumerate(validators)) + + by_subnet: dict[int, list[tuple[int, dict[str, Any]]]] = defaultdict(list) + for i, v in enumerate(validators): + by_subnet[i % subnet_count].append((i, v)) + + selected: list[tuple[int, dict[str, Any]]] = [] + for s in sorted(by_subnet.keys()): + group = by_subnet[s] + chosen: list[tuple[int, dict[str, Any]]] = [] + chosen_idx: set[int] = set() + + for idx, val in group: + if val.get("isAggregator") is True: + chosen.append((idx, val)) + chosen_idx.add(idx) + break + else: + if group: + idx, val = group[0] + chosen.append((idx, val)) + chosen_idx.add(idx) + + for idx, val in group: + if len(chosen) >= per_subnet: + break + if idx in chosen_idx: + continue + chosen.append((idx, val)) + chosen_idx.add(idx) + + selected.extend(chosen) + + return selected def convert_validator_config( @@ -43,6 +119,10 @@ def convert_validator_config( output_path: str, base_port: int = 8081, docker_host: bool = False, + *, + all_upstreams: bool = True, + subnet_sample: bool = False, + per_subnet: Optional[int] = None, ): """ Convert validator-config.yaml to upstreams.json. @@ -53,6 +133,11 @@ def convert_validator_config( base_port: Base HTTP port for beacon API (default: 8081) docker_host: If True, use host.docker.internal so leanpoint in Docker can reach a devnet running on the host (Docker Desktop/Orbstack). + all_upstreams: If True (default), emit one upstream per validator. + subnet_sample: If True, keep at most per_subnet validators per attestation + subnet instead of the full list. + per_subnet: Max upstreams per attestation subnet when subnet_sample is True; + default from LEANPOINT_UPSTREAMS_PER_SUBNET env or 2. """ with open(yaml_path, 'r') as f: config = yaml.safe_load(f) @@ -61,9 +146,38 @@ def convert_validator_config( print("Error: No 'validators' key found in config", file=sys.stderr) sys.exit(1) + validators = config["validators"] + committee = _attestation_committee_count(config) + + use_subnet_sample = subnet_sample and not all_upstreams + if use_subnet_sample: + if per_subnet is None: + try: + per_subnet = int(os.environ.get("LEANPOINT_UPSTREAMS_PER_SUBNET", "2")) + except ValueError: + per_subnet = 2 + if per_subnet < 1: + per_subnet = 1 + rows = _select_validators_for_leanpoint( + validators, + committee, + per_subnet=per_subnet, + all_upstreams=False, + ) + if committee is not None: + print( + f"Info: Leanpoint upstream subset (--subnet-sample): " + f"attestation_committee_count={committee}, up to {per_subnet} " + f"validator(s) per subnet ({len(rows)} upstreams from " + f"{len(validators)} validators).", + file=sys.stderr, + ) + else: + rows = list(enumerate(validators)) + upstreams = [] - for idx, validator in enumerate(config['validators']): + for idx, validator in rows: name = validator.get('name', f'validator_{idx}') # Try to get IP from enrFields, default to localhost @@ -90,7 +204,7 @@ def convert_validator_config( with open(output_path, 'w') as f: json.dump(output, f, indent=2) - print(f"✅ Converted {len(upstreams)} validators to {output_path}") + print(f"✅ Wrote {len(upstreams)} leanpoint upstream(s) to {output_path}") print(f"\nGenerated upstreams:") for u in upstreams: print(f" - {u['name']}: {u['url']}{u['path']}") @@ -161,8 +275,15 @@ def write_nemo_env_file( def main(): - argv = [a for a in sys.argv[1:] if a != "--docker"] docker_host = "--docker" in sys.argv + subnet_sample = "--subnet-sample" in sys.argv + # Default: every validator is an upstream. --all-upstreams is a no-op (compat). + all_upstreams = "--subnet-sample" not in sys.argv + argv = [ + a + for a in sys.argv[1:] + if a not in ("--docker", "--all-upstreams", "--subnet-sample") + ] if "--print-lean-api-url" in argv: argv = [a for a in argv if a != "--print-lean-api-url"] @@ -213,7 +334,13 @@ def main(): output_path = args[1] try: - convert_validator_config(yaml_path, output_path, docker_host=docker_host) + convert_validator_config( + yaml_path, + output_path, + docker_host=docker_host, + all_upstreams=all_upstreams, + subnet_sample=subnet_sample, + ) except FileNotFoundError as e: print(f"Error: File not found: {e}", file=sys.stderr) sys.exit(1) diff --git a/sync-leanpoint-upstreams.sh b/sync-leanpoint-upstreams.sh index 1313186..5cb2319 100755 --- a/sync-leanpoint-upstreams.sh +++ b/sync-leanpoint-upstreams.sh @@ -20,8 +20,14 @@ # LEANPOINT_DIR Path containing convert-validator-config.py (default: script_dir) # REMOTE_UPSTREAMS_PATH Remote path for upstreams.json (default: /etc/leanpoint/upstreams.json) # LEANPOINT_CONTAINER Docker container name (default: leanpoint) -# LEANPOINT_IMAGE Docker image to pull and run (default: 0xpartha/leanpoint:latest) +# LEANPOINT_IMAGE Docker image to pull and run (default: 0xpartha/leanpoint:latest). +# Use 0xpartha/leanpoint:latest for shared tooling hosts; do not use +# blockblaz/leanpoint:latest there unless the image is built for baseline +# x86_64 (older hosts may SIGILL / exit 132). # LEANPOINT_HOST_PORT Host port published for leanpoint HTTP (default: 5555). +# LEANPOINT_UPSTREAMS_PER_SUBNET Only with convert-validator-config.py --subnet-sample: +# max upstreams per attestation subnet (default: 2). +# By default every validator in validator-config.yaml is polled. # NEMO_HOST_PORT Used only for clash check (default 5053); must differ from LEANPOINT_HOST_PORT. # LEANPOINT_SYNC_DISABLED Set to 1 to skip (e.g. when tooling server is not used) @@ -67,7 +73,7 @@ fi if [ -n "$local_data_dir" ]; then mkdir -p "$local_data_dir" local_upstreams="$local_data_dir/upstreams.json" - python3 "$convert_script" "$validator_config_file" "$local_upstreams" --docker || { + python3 "$convert_script" "$validator_config_file" "$local_upstreams" --docker --all-upstreams || { echo "Warning: convert-validator-config.py failed, skipping local leanpoint deploy." exit 0 } @@ -94,7 +100,7 @@ fi out_file=$(mktemp) trap "rm -f $out_file" EXIT -python3 "$convert_script" "$validator_config_file" "$out_file" || { +python3 "$convert_script" "$validator_config_file" "$out_file" --all-upstreams || { echo "Warning: convert-validator-config.py failed, skipping leanpoint sync." exit 0 } From f95ae4983d7148411dcaeef391ac71e7b571087c Mon Sep 17 00:00:00 2001 From: ch4r10t33r Date: Sun, 17 May 2026 09:46:55 +0100 Subject: [PATCH 03/20] ansible: fold observability into --prepare and harden apt installs --prepare now installs tools, opens firewall ports, and starts Prometheus, Promtail, node_exporter, and cadvisor on every host. Add apt retries/throttle, prepare fork cap, and a single retry pass for transient lock failures. --- ansible/playbooks/deploy-observability.yml | 51 +++++++++++++++++++++ ansible/playbooks/prepare.yml | 52 +++++++++++++++++++++- ansible/roles/observability/tasks/main.yml | 36 ++++++++++----- parse-env.sh | 5 +++ run-ansible.sh | 48 +++++++++++++++++--- spin-node.sh | 4 +- 6 files changed, 175 insertions(+), 21 deletions(-) create mode 100644 ansible/playbooks/deploy-observability.yml diff --git a/ansible/playbooks/deploy-observability.yml b/ansible/playbooks/deploy-observability.yml new file mode 100644 index 0000000..9a9b9b6 --- /dev/null +++ b/ansible/playbooks/deploy-observability.yml @@ -0,0 +1,51 @@ +--- +# Deploy Prometheus, Promtail, node_exporter, and cadvisor on every remote host +# (one play per unique enrFields.ip — uses inventory/hosts-prepare.yml). +# +# Imported by prepare.yml after tools and firewall are configured. +# Can also run standalone via run-ansible.sh action=observability. + +- name: Deploy observability stack on all remote hosts + hosts: prepare_hosts + gather_facts: yes + vars: + observability_force_primary: true + genesis_dir: "{{ remote_genesis_dir | default('/opt/lean-quickstart/genesis') }}" + # Role defaults are not in play scope after include_role; set ports for health checks. + prometheus_port: 9090 + promtail_port: 9080 + cadvisor_port: 9098 + node_exporter_port: 9100 + + tasks: + - name: Deploy observability stack + include_role: + name: observability + + - name: Verify observability containers are running + command: >- + docker inspect -f + {% raw %}'{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{else}}no-healthcheck{{end}}'{% endraw %} + {{ item }} + register: obs_container_status + changed_when: false + loop: + - cadvisor + - node_exporter + - prometheus + - promtail + failed_when: obs_container_status.stdout.split()[0] != 'running' + + - name: Wait for observability HTTP endpoints + uri: + url: "http://127.0.0.1:{{ item.port }}{{ item.path }}" + status_code: 200 + loop: + - { port: "{{ prometheus_port }}", path: "/-/healthy" } + - { port: "{{ node_exporter_port }}", path: "/metrics" } + - { port: "{{ cadvisor_port }}", path: "/healthz" } + - { port: "{{ promtail_port }}", path: "/ready" } + register: obs_http_check + until: obs_http_check is succeeded + retries: 6 + delay: 10 diff --git a/ansible/playbooks/prepare.yml b/ansible/playbooks/prepare.yml index b217902..eb6c60b 100644 --- a/ansible/playbooks/prepare.yml +++ b/ansible/playbooks/prepare.yml @@ -1,6 +1,7 @@ --- # Prepare playbook: Verify and install the software required to run lean nodes, -# then open and persist the firewall rules each host needs. +# open and persist firewall rules, then deploy the per-host observability stack +# (Prometheus, Promtail, node_exporter, cadvisor). # # Three software prerequisites on every remote server: # @@ -37,6 +38,9 @@ # Usage (via spin-node.sh): # ./spin-node.sh --prepare [--sshKey ~/.ssh/id_ed25519] [--useRoot] # +# Observability containers are started in a second play (deploy-observability.yml) +# after tools and firewall rules are in place. +# # Only runs on remote hosts (localhost is excluded). # # run-ansible.sh uses inventory/hosts-prepare.yml (one host per unique enrFields.ip) @@ -83,6 +87,9 @@ when: - ansible_os_family == "Debian" - docker_pre.rc != 0 + # Limit concurrent apt/docker installs — parallel prepare on 30+ hosts often + # hits transient apt lock or download.docker.com errors on a single machine. + throttle: "{{ prepare_apt_throttle | default(5) }}" block: - name: Install apt prerequisites for Docker repository apt: @@ -93,7 +100,12 @@ - lsb-release state: present update_cache: yes + lock_timeout: 120 become: yes + register: docker_apt_prereq + until: docker_apt_prereq is succeeded + retries: 3 + delay: 15 - name: Create /etc/apt/keyrings directory file: @@ -109,6 +121,10 @@ mode: '0644' force: no become: yes + register: docker_gpg_download + until: docker_gpg_download is succeeded + retries: 3 + delay: 10 - name: Add Docker apt repository apt_repository: @@ -120,6 +136,10 @@ state: present filename: docker become: yes + register: docker_apt_repo + until: docker_apt_repo is succeeded + retries: 3 + delay: 15 - name: Install Docker packages apt: @@ -130,7 +150,18 @@ - docker-compose-plugin state: present update_cache: yes + lock_timeout: 300 become: yes + register: docker_apt_install + until: docker_apt_install is succeeded + retries: 3 + delay: 20 + + - name: Re-check docker is on PATH after install attempt + command: which docker + register: docker_post + changed_when: false + failed_when: false - name: Ensure Docker service is started and enabled systemd: @@ -138,7 +169,9 @@ state: started enabled: yes become: yes - when: ansible_os_family == "Debian" + when: + - ansible_os_family == "Debian" + - docker_post.rc == 0 # ────────────────────────────────────────────────────────────────────────── # 3. yq — the common role hard-fails at every deploy if this is absent @@ -181,7 +214,13 @@ name: jq state: present update_cache: yes + lock_timeout: 120 become: yes + register: jq_apt_install + until: jq_apt_install is succeeded + retries: 3 + delay: 15 + throttle: "{{ prepare_apt_throttle | default(5) }}" # ────────────────────────────────────────────────────────────────────────── # 5. Firewall rules (iptables + ufw) @@ -222,8 +261,14 @@ - ufw state: present update_cache: "{{ 'yes' if (iptables_pre.rc != 0 or ufw_pre.rc != 0) else 'no' }}" + lock_timeout: 120 become: yes when: ansible_os_family == "Debian" + register: fw_apt_install + until: fw_apt_install is succeeded + retries: 3 + delay: 15 + throttle: "{{ prepare_apt_throttle | default(5) }}" - name: Read all node entries for this host from the active validator config vars: @@ -355,3 +400,6 @@ loop_control: label: "{{ item.item.name }}" when: item.rc != 0 + +- name: Deploy observability stack on all remote hosts + import_playbook: deploy-observability.yml diff --git a/ansible/roles/observability/tasks/main.yml b/ansible/roles/observability/tasks/main.yml index 545942b..9e1359f 100644 --- a/ansible/roles/observability/tasks/main.yml +++ b/ansible/roles/observability/tasks/main.yml @@ -13,7 +13,10 @@ - name: Set colocated nodes fact set_fact: colocated_nodes: "{{ colocated_nodes_raw.stdout | from_json }}" - is_primary_node: "{{ (colocated_nodes_raw.stdout | from_json)[0].name == node_name }}" + node_name: "{{ node_name | default((colocated_nodes_raw.stdout | from_json)[0].name) }}" + observability_manage_host: >- + {{ observability_force_primary | default(false) | bool + or ((colocated_nodes_raw.stdout | from_json)[0].name == node_name) }} - name: Create observability config directory file: @@ -34,13 +37,23 @@ mode: '0644' # --- cadvisor (always recreate to ensure correct flags) --- -# Only the first colocated node on each machine manages infra containers -# to avoid parallel rm+run races when multiple nodes share a host. +# Only one play per machine manages infra containers (primary deploy node or +# deploy-observability.yml with observability_force_primary). + +- name: Pull observability container images + command: docker pull {{ item }} + loop: + - "{{ cadvisor_image }}" + - "{{ node_exporter_image }}" + - "{{ prometheus_image }}" + - "{{ promtail_image }}" + register: observability_pull + when: observability_manage_host | bool - name: Remove existing cadvisor container command: docker rm -f cadvisor failed_when: false - when: is_primary_node + when: observability_manage_host | bool - name: Start cadvisor container command: >- @@ -60,14 +73,14 @@ -v /var/lib/docker/:/var/lib/docker:ro {{ cadvisor_image }} --port={{ cadvisor_port }} - when: is_primary_node + when: observability_manage_host | bool # --- node_exporter (always recreate to ensure correct flags) --- - name: Remove existing node_exporter container command: docker rm -f node_exporter failed_when: false - when: is_primary_node + when: observability_manage_host | bool - name: Start node_exporter container command: >- @@ -84,7 +97,7 @@ --path.sysfs=/host/sys --path.rootfs=/rootfs --web.listen-address=0.0.0.0:{{ node_exporter_port }} - when: is_primary_node + when: observability_manage_host | bool # --- prometheus (always recreate to pick up config/mount changes, data persists on host) --- @@ -94,11 +107,12 @@ state: directory mode: '0777' recurse: yes + when: observability_manage_host | bool - name: Remove existing prometheus container command: docker rm -f prometheus failed_when: false - when: is_primary_node + when: observability_manage_host | bool - name: Start prometheus container command: >- @@ -112,14 +126,14 @@ --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.retention.time=15d --web.listen-address=0.0.0.0:{{ prometheus_port }} - when: is_primary_node + when: observability_manage_host | bool # --- promtail (always recreate to pick up config/mount changes) --- - name: Remove existing promtail container command: docker rm -f promtail failed_when: false - when: is_primary_node + when: observability_manage_host | bool - name: Start promtail container command: >- @@ -133,4 +147,4 @@ {{ promtail_image }} -config.file=/etc/promtail/config.yml -server.http-listen-port={{ promtail_port }} - when: is_primary_node + when: observability_manage_host | bool diff --git a/parse-env.sh b/parse-env.sh index 50fc46c..041c217 100755 --- a/parse-env.sh +++ b/parse-env.sh @@ -111,6 +111,11 @@ while [[ $# -gt 0 ]]; do prepareMode=true shift ;; + --deploy-observability) + echo "Warning: --deploy-observability is deprecated; use --prepare (observability is included)." + prepareMode=true + shift + ;; --subnets) subnets="$2" shift # past argument diff --git a/run-ansible.sh b/run-ansible.sh index 2348451..5c277fd 100755 --- a/run-ansible.sh +++ b/run-ansible.sh @@ -74,7 +74,7 @@ fi # prepare.yml: one play per physical host (deduped by IP). Deploy still uses full hosts.yml. EFFECTIVE_INVENTORY="$INVENTORY_FILE" -if [ "$action" == "prepare" ] && [ -f "$PREPARE_INVENTORY" ]; then +if { [ "$action" == "prepare" ] || [ "$action" == "observability" ]; } && [ -f "$PREPARE_INVENTORY" ]; then EFFECTIVE_INVENTORY="$PREPARE_INVENTORY" fi @@ -164,6 +164,10 @@ fi EXTRA_VARS="$EXTRA_VARS network_name=$networkName" +if [ "$action" == "prepare" ]; then + EXTRA_VARS="$EXTRA_VARS prepare_apt_throttle=${LEAN_PREPARE_APT_THROTTLE:-5}" +fi + # Determine deployment mode (docker/binary) - read default from group_vars/all.yml # Default to 'docker' if not specified in group_vars GROUP_VARS_FILE="$ANSIBLE_DIR/inventory/group_vars/all.yml" @@ -183,6 +187,9 @@ if [ "$action" == "stop" ]; then elif [ "$action" == "prepare" ]; then PLAYBOOK="$ANSIBLE_DIR/playbooks/prepare.yml" ACTION_MSG="preparing servers" +elif [ "$action" == "observability" ]; then + PLAYBOOK="$ANSIBLE_DIR/playbooks/deploy-observability.yml" + ACTION_MSG="deploying observability stack" else PLAYBOOK="$ANSIBLE_DIR/playbooks/site.yml" ACTION_MSG="deploying nodes" @@ -196,15 +203,25 @@ ANSIBLE_CMD="$ANSIBLE_CMD -e \"$EXTRA_VARS\"" # Forks: honor ANSIBLE_FORKS when set; else derive from unique enrFields.ip in validator-config # (Ansible cannot set forks from inside a playbook for the same run.) +_play_forks="" if [ -n "${ANSIBLE_FORKS:-}" ]; then - ANSIBLE_CMD="$ANSIBLE_CMD -f ${ANSIBLE_FORKS}" + _play_forks="${ANSIBLE_FORKS}" elif [ -f "$_local_vc_path" ] && command -v yq &>/dev/null; then - _ansible_forks="$("$ANSIBLE_DIR/compute-forks-from-validator-config.sh" "$_local_vc_path")" - ANSIBLE_CMD="$ANSIBLE_CMD -f ${_ansible_forks}" - echo "Ansible forks: ${_ansible_forks} (unique IPs in validator-config; set ANSIBLE_FORKS to override)" + _play_forks="$("$ANSIBLE_DIR/compute-forks-from-validator-config.sh" "$_local_vc_path")" elif [ -f "$_local_vc_path" ]; then echo "Warning: yq not found; omitting -f (using ansible.cfg forks default)" fi +if [ -n "$_play_forks" ] && { [ "$action" == "prepare" ] || [ "$action" == "observability" ]; }; then + _prepare_forks_max="${LEAN_PREPARE_FORKS_MAX:-15}" + if (( _play_forks > _prepare_forks_max )); then + echo "Prepare: capping forks from ${_play_forks} to ${_prepare_forks_max} (LEAN_PREPARE_FORKS_MAX)" + _play_forks="$_prepare_forks_max" + fi +fi +if [ -n "$_play_forks" ]; then + ANSIBLE_CMD="$ANSIBLE_CMD -f ${_play_forks}" + echo "Ansible forks: ${_play_forks} (set ANSIBLE_FORKS to override)" +fi # Dry-run: show what Ansible would change without applying anything. if [ "$dryRun" == "true" ]; then @@ -220,6 +237,21 @@ cd "$ANSIBLE_DIR" eval $ANSIBLE_CMD EXIT_CODE=$? +# One idempotent retry: a single host often fails on first pass due to apt lock +# or download.docker.com under high prepare parallelism. +if [ $EXIT_CODE -ne 0 ] && [ "$action" == "prepare" ] && [ "$dryRun" != "true" ]; then + _retry_forks="${LEAN_PREPARE_RETRY_FORKS:-5}" + echo "" + echo "Prepare had failures; retrying all hosts once (forks=${_retry_forks})..." + _retry_cmd="$ANSIBLE_CMD" + if [ -n "$_play_forks" ]; then + _retry_cmd=$(echo "$_retry_cmd" | sed -E "s/ -f ${_play_forks}/ -f ${_retry_forks}/") + else + _retry_cmd="$_retry_cmd -f ${_retry_forks}" + fi + eval $_retry_cmd + EXIT_CODE=$? +fi _dry_tag="" [ "$dryRun" == "true" ] && _dry_tag=" (dry-run — no changes applied)" if [ $EXIT_CODE -eq 0 ]; then @@ -227,7 +259,9 @@ if [ $EXIT_CODE -eq 0 ]; then if [ "$action" == "stop" ]; then echo "✅ Ansible stop operation completed successfully!${_dry_tag}" elif [ "$action" == "prepare" ]; then - echo "✅ Server preparation completed successfully!${_dry_tag}" + echo "✅ Server preparation completed (tools, firewall, observability)!${_dry_tag}" + elif [ "$action" == "observability" ]; then + echo "✅ Observability stack deployed on all hosts!${_dry_tag}" else echo "✅ Ansible deployment completed successfully!${_dry_tag}" fi @@ -237,6 +271,8 @@ else echo "❌ Ansible stop operation failed with exit code $EXIT_CODE" elif [ "$action" == "prepare" ]; then echo "❌ Server preparation failed with exit code $EXIT_CODE" + elif [ "$action" == "observability" ]; then + echo "❌ Observability deployment failed with exit code $EXIT_CODE" else echo "❌ Ansible deployment failed with exit code $EXIT_CODE" fi diff --git a/spin-node.sh b/spin-node.sh index 7270190..39c2e5f 100755 --- a/spin-node.sh +++ b/spin-node.sh @@ -207,7 +207,7 @@ if [ -n "$prepareMode" ] && [ "$prepareMode" == "true" ]; then if [ "$dryRun" == "true" ]; then echo "[DRY RUN] Would prepare remote servers — running Ansible with --check --diff" else - echo "Preparing remote servers (verifying and installing required software)..." + echo "Preparing remote servers (tools, firewall, observability stack)..." fi if ! "$scriptDir/run-ansible.sh" "$configDir" "" "" "" "$validator_config_file" "$sshKeyFile" "$useRoot" "prepare" "" "" "" "$dryRun" "" "$networkName"; then @@ -215,7 +215,7 @@ if [ -n "$prepareMode" ] && [ "$prepareMode" == "true" ]; then exit 1 fi - [ "$dryRun" == "true" ] && echo "✅ Dry-run complete — no changes were made." || echo "✅ All remote servers are prepared." + [ "$dryRun" == "true" ] && echo "✅ Dry-run complete — no changes were made." || echo "✅ All remote servers are prepared (tools, firewall, Prometheus, Promtail, node_exporter, cadvisor)." exit 0 fi From 05d39a2ce70b7b19a3cc3d55cf990796c5710f3c Mon Sep 17 00:00:00 2001 From: ch4r10t33r Date: Sun, 17 May 2026 10:59:15 +0100 Subject: [PATCH 04/20] ansible-devnet: per-subnet aggregator gossip and replace qlean/lantern slots Aggregators now get --aggregate-subnet-ids for their committee only (validator_index % attestation_committee_count) via parse-vc.sh and ansible zeam/ethlambda roles, not the full 0..N-1 CSV. Client cmd scripts pass a single subnet id; peam allowed_topics match the same rule. Rename former qlean_* / lantern_* validator nodes to zeam_8..15 and ethlambda_8..15 in ansible and local genesis configs to avoid clashing with existing zeam_0..7 / ethlambda_0..7 names. --- .../test-validator-config-subnet2.yaml | 18 ++++---- .../genesis/validator-config-expanded.yaml | 20 ++++----- ansible-devnet/genesis/validator-config.yaml | 45 ++++++++++--------- ansible/roles/ethlambda/tasks/main.yml | 39 +++++++++------- ansible/roles/zeam/tasks/main.yml | 39 +++++++++------- client-cmds/ethlambda-cmd.sh | 7 +-- client-cmds/gean-cmd.sh | 7 +-- client-cmds/grandine-cmd.sh | 7 +-- client-cmds/lantern-cmd.sh | 7 +-- client-cmds/lighthouse-cmd.sh | 7 +-- client-cmds/nlean-cmd.sh | 7 +-- client-cmds/peam-cmd.sh | 20 +++++---- client-cmds/qlean-cmd.sh | 7 +-- client-cmds/ream-cmd.sh | 7 +-- client-cmds/zeam-cmd.sh | 9 ++-- local-devnet/genesis/validator-config.yaml | 14 ++---- parse-vc.sh | 19 +++----- 17 files changed, 128 insertions(+), 151 deletions(-) diff --git a/ansible-devnet/genesis/test-validator-config-subnet2.yaml b/ansible-devnet/genesis/test-validator-config-subnet2.yaml index 168e828..fd3041d 100644 --- a/ansible-devnet/genesis/test-validator-config-subnet2.yaml +++ b/ansible-devnet/genesis/test-validator-config-subnet2.yaml @@ -1,14 +1,14 @@ shuffle: roundrobin -# Test layout: 2 subnets × 7 nodes each (1 zeam + 1 grandine + 1 gean + 2 ethlambda + 1 qlean + 1 lantern per subnet +# Test layout: 2 subnets × 7 nodes each (2 zeam + 1 grandine + 1 gean + 3 ethlambda per subnet) # except subnet 1 which has 1 ream instead of ethlambda_3). # Each server hosts one node per subnet per client type on distinct ports. # # Server layout (IP → nodes): -# 37.27.89.135 grandine_0 (s0,9001) zeam_3 (s1,9002) qlean_0 (s0,9003) -# 157.180.20.55 zeam_1 (s0,9001) zeam_4 (s1,9002) qlean_1 (s1,9003) +# 37.27.89.135 grandine_0 (s0,9001) zeam_3 (s1,9002) zeam_8 (s0,9003) +# 157.180.20.55 zeam_1 (s0,9001) zeam_4 (s1,9002) zeam_9 (s1,9003) # 178.104.133.162 gean_0 (s0,9001) ethlambda_2 (s1,9002) -# 178.104.151.50 ethlambda_0 (s0,9001) ream_0 (s1,9002) lantern_0 (s0,9003) -# 178.104.149.91 ethlambda_1 (s0,9001) gean_1 (s1,9002) lantern_1 (s1,9003) +# 178.104.151.50 ethlambda_0 (s0,9001) ream_0 (s1,9002) ethlambda_8 (s0,9003) +# 178.104.149.91 ethlambda_1 (s0,9001) gean_1 (s1,9002) ethlambda_9 (s1,9003) # # ./spin-node.sh --validatorConfig ansible-devnet/genesis/test-validator-config-subnet2.yaml deployment_mode: ansible @@ -68,7 +68,7 @@ validators: subnet: 0 isAggregator: false count: 1 - - name: "qlean_0" + - name: "zeam_8" privkey: "8e9f81c9caa9e29d26a7327311ca63a38254efdfccf3ce1362bae47eae0b18b3" enrFields: ip: "37.27.89.135" @@ -78,7 +78,7 @@ validators: subnet: 0 isAggregator: false count: 1 - - name: "lantern_0" + - name: "ethlambda_8" privkey: "d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5" enrFields: ip: "178.104.151.50" @@ -139,7 +139,7 @@ validators: subnet: 1 isAggregator: false count: 1 - - name: "qlean_1" + - name: "zeam_9" privkey: "1991db904e455114bc836a0860689e77c1d2fa313a15567c6c7aa81e180925ac" enrFields: ip: "157.180.20.55" @@ -149,7 +149,7 @@ validators: subnet: 1 isAggregator: false count: 1 - - name: "lantern_1" + - name: "ethlambda_9" privkey: "582644c8e66b61df3dafb33fa496cee97060566542fb8594fca661be3332710f" enrFields: ip: "178.104.149.91" diff --git a/ansible-devnet/genesis/validator-config-expanded.yaml b/ansible-devnet/genesis/validator-config-expanded.yaml index c0238c0..fb5c6bb 100644 --- a/ansible-devnet/genesis/validator-config-expanded.yaml +++ b/ansible-devnet/genesis/validator-config-expanded.yaml @@ -155,7 +155,7 @@ validators: isAggregator: false count: 1 subnet: 4 -- name: qlean_0 +- name: zeam_8 privkey: c2bbdac5e876b3e9d4b8b6b8c2bbdac5e876b3e9d4b8b6b8c2bbdac5e876b3e9 enrFields: ip: 178.104.133.162 @@ -165,7 +165,7 @@ validators: isAggregator: false count: 1 subnet: 0 -- name: qlean_1 +- name: zeam_9 privkey: 927eed3828e0af660ab7525d0971f7f29a3055f8832b947b03eab0bc5683bd97 enrFields: ip: 178.104.133.162 @@ -175,7 +175,7 @@ validators: isAggregator: false count: 1 subnet: 1 -- name: qlean_2 +- name: zeam_10 privkey: 7fdceb8166b9d7ab4169d80345ab85fd042b3684ba307c4c02e851c1520975b5 enrFields: ip: 178.104.133.162 @@ -185,7 +185,7 @@ validators: isAggregator: false count: 1 subnet: 2 -- name: qlean_3 +- name: zeam_11 privkey: 1027ce52661ef76b5ada68bc8cd45c13ebefb09c99dcad251b9bd4893c71815f enrFields: ip: 178.104.133.162 @@ -195,7 +195,7 @@ validators: isAggregator: false count: 1 subnet: 3 -- name: qlean_4 +- name: zeam_12 privkey: 0763e1a9f1909df7eae31ef4e93d5aaf45a506624ef81df893fc02e08a4ab94f enrFields: ip: 178.104.133.162 @@ -205,7 +205,7 @@ validators: isAggregator: false count: 1 subnet: 4 -- name: lantern_0 +- name: ethlambda_8 privkey: d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5 enrFields: ip: 178.104.149.91 @@ -215,7 +215,7 @@ validators: isAggregator: false count: 1 subnet: 0 -- name: lantern_1 +- name: ethlambda_9 privkey: 582644c8e66b61df3dafb33fa496cee97060566542fb8594fca661be3332710f enrFields: ip: 178.104.149.91 @@ -225,7 +225,7 @@ validators: isAggregator: false count: 1 subnet: 1 -- name: lantern_2 +- name: ethlambda_10 privkey: 943151f2327ea91357632dd8a3a498242b07eeeebad973ea82d6bead80627d3e enrFields: ip: 178.104.149.91 @@ -235,7 +235,7 @@ validators: isAggregator: false count: 1 subnet: 2 -- name: lantern_3 +- name: ethlambda_11 privkey: 31258c1b84e4a64954fb18dbb04ec9de7079ad4d0273b40380aa1f4ca2f20804 enrFields: ip: 178.104.149.91 @@ -245,7 +245,7 @@ validators: isAggregator: false count: 1 subnet: 3 -- name: lantern_4 +- name: ethlambda_12 privkey: a5dac14f317084d255c57e5080985b1dbd79cf8fc625a9c7fba781abf07001bd enrFields: ip: 178.104.149.91 diff --git a/ansible-devnet/genesis/validator-config.yaml b/ansible-devnet/genesis/validator-config.yaml index 4b1ca25..bb0a838 100644 --- a/ansible-devnet/genesis/validator-config.yaml +++ b/ansible-devnet/genesis/validator-config.yaml @@ -2,8 +2,9 @@ # # Genesis order = YAML list order → validator_index = row index (0..63). # attestation_committee_count: 8 → subnet = index % 8. -# Row pattern repeats qlean → lantern → ream → zeam → ethlambda → gean → grandine → nlean. -# Aggregators: first validator on each subnet (indices 0–7), on dedicated aggregator hosts. +# Row pattern repeats zeam → ethlambda → ream → zeam → ethlambda → gean → grandine → nlean +# (former qlean/lantern slots are zeam_8..15 and ethlambda_8..15). +# Aggregators: one per committee subnet (validator_index % 8); each subscribes only to its subnet. # # Regular validators use IPs from lean_ethereum_servers.txt (Validator_servers). # Aggregators use dedicated Aggregator_servers (one IP per client type). @@ -19,7 +20,7 @@ config: keyType: "hash-sig" attestation_committee_count: 8 validators: - - name: "qlean_0" + - name: "zeam_8" privkey: "8e9f81c9caa9e29d26a7327311ca63a38254efdfccf3ce1362bae47eae0b18b3" enrFields: ip: "178.105.22.187" @@ -28,7 +29,7 @@ validators: apiPort: 5055 isAggregator: true count: 1 - - name: "lantern_0" + - name: "ethlambda_8" privkey: "d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5" enrFields: ip: "178.105.124.135" @@ -53,7 +54,7 @@ validators: quic: 9001 metricsPort: 9102 apiPort: 5055 - isAggregator: true + isAggregator: false count: 1 - name: "ethlambda_0" privkey: "299550529a79bc2dce003747c52fb0639465c893e00b0440ac66144d625e066a" @@ -62,7 +63,7 @@ validators: quic: 9001 metricsPort: 9102 apiPort: 5055 - isAggregator: true + isAggregator: false count: 1 - name: "gean_0" privkey: "69c251cdb06039dd99d87e5a1439fa3720615be98c293ec9bcfd041877a2e8ca" @@ -91,7 +92,7 @@ validators: apiPort: 5055 isAggregator: true count: 1 - - name: "qlean_1" + - name: "zeam_9" privkey: "0e190e06a62db01bf566af4348277463d26eebab8f6badbcb989242ea4fee050" enrFields: ip: "37.27.220.14" @@ -100,7 +101,7 @@ validators: apiPort: 5055 isAggregator: false count: 1 - - name: "lantern_1" + - name: "ethlambda_9" privkey: "1176609530e568ec12ad227e60ae71c618bfb13fde5a3257e4b0627216e78e04" enrFields: ip: "157.180.116.162" @@ -163,7 +164,7 @@ validators: apiPort: 5055 isAggregator: false count: 1 - - name: "qlean_2" + - name: "zeam_10" privkey: "7fdceb8166b9d7ab4169d80345ab85fd042b3684ba307c4c02e851c1520975b5" enrFields: ip: "178.104.151.50" @@ -172,7 +173,7 @@ validators: apiPort: 5055 isAggregator: false count: 1 - - name: "lantern_2" + - name: "ethlambda_10" privkey: "943151f2327ea91357632dd8a3a498242b07eeeebad973ea82d6bead80627d3e" enrFields: ip: "178.104.133.162" @@ -235,7 +236,7 @@ validators: apiPort: 5055 isAggregator: false count: 1 - - name: "qlean_3" + - name: "zeam_11" privkey: "1027ce52661ef76b5ada68bc8cd45c13ebefb09c99dcad251b9bd4893c71815f" enrFields: ip: "65.109.138.213" @@ -244,7 +245,7 @@ validators: apiPort: 5055 isAggregator: false count: 1 - - name: "lantern_3" + - name: "ethlambda_11" privkey: "31258c1b84e4a64954fb18dbb04ec9de7079ad4d0273b40380aa1f4ca2f20804" enrFields: ip: "204.168.134.201" @@ -269,7 +270,7 @@ validators: quic: 9001 metricsPort: 9102 apiPort: 5055 - isAggregator: false + isAggregator: true count: 1 - name: "ethlambda_3" privkey: "1c454b0399cd2178b46e42c8b599e34f2e7ddab71df96b1fbf510d5951335ea0" @@ -307,7 +308,7 @@ validators: apiPort: 5056 isAggregator: false count: 1 - - name: "qlean_4" + - name: "zeam_12" privkey: "0763e1a9f1909df7eae31ef4e93d5aaf45a506624ef81df893fc02e08a4ab94f" enrFields: ip: "157.180.116.162" @@ -316,7 +317,7 @@ validators: apiPort: 5056 isAggregator: false count: 1 - - name: "lantern_4" + - name: "ethlambda_12" privkey: "a5dac14f317084d255c57e5080985b1dbd79cf8fc625a9c7fba781abf07001bd" enrFields: ip: "204.168.178.179" @@ -379,7 +380,7 @@ validators: apiPort: 5056 isAggregator: false count: 1 - - name: "qlean_5" + - name: "zeam_13" privkey: "ce66f19a179bd74e01b775c696b1744060078fe4d84abb5761a3f1d7cef15562" enrFields: ip: "178.104.133.162" @@ -388,7 +389,7 @@ validators: apiPort: 5056 isAggregator: false count: 1 - - name: "lantern_5" + - name: "ethlambda_13" privkey: "12bb3c043f2600f27be6bf8916368e9ff01b9ef5c4a7ccdf6aab8e2a24ba55c3" enrFields: ip: "178.104.149.208" @@ -451,7 +452,7 @@ validators: apiPort: 5056 isAggregator: false count: 1 - - name: "qlean_6" + - name: "zeam_14" privkey: "aaf15d4a12fec7409b136d022790d9aaaca3d5679c9e28babe23a424cd80729b" enrFields: ip: "204.168.134.201" @@ -460,7 +461,7 @@ validators: apiPort: 5056 isAggregator: false count: 1 - - name: "lantern_6" + - name: "ethlambda_14" privkey: "7536676b6ee5f0062030c0b6784c44a175d66cdc39277354c1b9014b419cbc30" enrFields: ip: "95.217.19.42" @@ -523,7 +524,7 @@ validators: apiPort: 5057 isAggregator: false count: 1 - - name: "qlean_7" + - name: "zeam_15" privkey: "2907bc1d1e0a3dee4cc317dfda39a09b5e23596671d7108482edb897616c6b6c" enrFields: ip: "204.168.178.179" @@ -532,7 +533,7 @@ validators: apiPort: 5057 isAggregator: false count: 1 - - name: "lantern_7" + - name: "ethlambda_15" privkey: "98da18b276aa5a54ceba15e78260286b9dbe03c4bf8b9db061d024ba4446cfff" enrFields: ip: "204.168.190.188" @@ -566,7 +567,7 @@ validators: quic: 9003 metricsPort: 9104 apiPort: 5057 - isAggregator: false + isAggregator: true count: 1 - name: "gean_7" privkey: "32e4b103c4b349df214d265db9cb1e6d49346e14093fee656828a5e1ef050b8b" diff --git a/ansible/roles/ethlambda/tasks/main.yml b/ansible/roles/ethlambda/tasks/main.yml index 28e222d..8e80f29 100644 --- a/ansible/roles/ethlambda/tasks/main.yml +++ b/ansible/roles/ethlambda/tasks/main.yml @@ -50,32 +50,37 @@ ethlambda_is_aggregator: "{{ 'true' if (ethlambda_node_config.results[3].stdout | default('') | trim) == 'true' else 'false' }}" when: ethlambda_node_config is defined -# Compute the full set of subnet ids in the network so aggregators can subscribe -# to attestations from every subnet (not just the one their validators live in). -# Required in multi-subnet deployments for cross-subnet attestation aggregation. -- name: Compute aggregate subnet ids from attestation_committee_count +# Aggregators subscribe only to their committee subnet (validator_index % committee_count). +- name: Compute aggregate subnet id for this aggregator node shell: | set -e + idx=$(yq eval '.validators | to_entries | .[] | select(.value.name == "{{ node_name }}") | .key' "{{ local_validator_config_path }}") ac=$(yq eval '.config.attestation_committee_count // 1' "{{ local_validator_config_path }}") + idx=$(echo "$idx" | tr -d '\r\n' | head -1) ac=$(echo "$ac" | tr -d '\r\n' | head -1) + case "$idx" in ''|*[!0-9]*) echo ""; exit 0;; esac case "$ac" in ''|*[!0-9]*) ac=1;; esac if [ "$ac" -lt 1 ]; then ac=1; fi - out="0" - i=1 - while [ "$i" -lt "$ac" ]; do - out="$out,$i" - i=$((i + 1)) - done - echo "$out" - register: ethlambda_all_subnets_raw + echo $((idx % ac)) + register: ethlambda_node_subnet_raw changed_when: false delegate_to: localhost - run_once: true + when: + - node_name is defined + - (ethlambda_is_aggregator | default('false')) == 'true' -- name: Set aggregate subnet ids csv +- name: Set aggregate subnet id for aggregator set_fact: - ethlambda_aggregate_subnet_ids: "{{ ethlambda_all_subnets_raw.stdout | trim }}" - run_once: true + ethlambda_aggregate_subnet_ids: "{{ ethlambda_node_subnet_raw.stdout | trim }}" + when: + - (ethlambda_is_aggregator | default('false')) == 'true' + - ethlambda_node_subnet_raw is defined + - ethlambda_node_subnet_raw.stdout | trim | length > 0 + +- name: Clear aggregate subnet ids for non-aggregator + set_fact: + ethlambda_aggregate_subnet_ids: "" + when: (ethlambda_is_aggregator | default('false')) != 'true' - name: Ensure node key file exists stat: @@ -135,7 +140,7 @@ --api-port {{ ethlambda_api_port }} --metrics-port {{ ethlambda_metrics_port }} {{ '--is-aggregator' if (ethlambda_is_aggregator | default('false')) == 'true' else '' }} - {{ ('--aggregate-subnet-ids ' + ethlambda_aggregate_subnet_ids) if (ethlambda_is_aggregator | default('false')) == 'true' and (',' in ethlambda_aggregate_subnet_ids | default('')) else '' }} + {{ ('--aggregate-subnet-ids ' + ethlambda_aggregate_subnet_ids) if (ethlambda_is_aggregator | default('false')) == 'true' and (ethlambda_aggregate_subnet_ids | default('') | length > 0) else '' }} {{ ('--checkpoint-sync-url ' + checkpoint_sync_url) if (checkpoint_sync_url is defined and checkpoint_sync_url | length > 0) else '' }} register: ethlambda_container changed_when: ethlambda_container.rc == 0 diff --git a/ansible/roles/zeam/tasks/main.yml b/ansible/roles/zeam/tasks/main.yml index d855d77..618f176 100644 --- a/ansible/roles/zeam/tasks/main.yml +++ b/ansible/roles/zeam/tasks/main.yml @@ -56,32 +56,37 @@ zeam_is_aggregator: "{{ 'true' if (node_config.results[3].stdout | default('') | trim) == 'true' else 'false' }}" when: node_config is defined -# Compute the full set of subnet ids in the network so aggregators can subscribe -# to attestations from every subnet (not just the one their validators live in). -# Required in multi-subnet deployments for cross-subnet attestation aggregation. -- name: Compute aggregate subnet ids from attestation_committee_count +# Aggregators subscribe only to their committee subnet (validator_index % committee_count). +- name: Compute aggregate subnet id for this aggregator node shell: | set -e + idx=$(yq eval '.validators | to_entries | .[] | select(.value.name == "{{ node_name }}") | .key' "{{ local_validator_config_path }}") ac=$(yq eval '.config.attestation_committee_count // 1' "{{ local_validator_config_path }}") + idx=$(echo "$idx" | tr -d '\r\n' | head -1) ac=$(echo "$ac" | tr -d '\r\n' | head -1) + case "$idx" in ''|*[!0-9]*) echo ""; exit 0;; esac case "$ac" in ''|*[!0-9]*) ac=1;; esac if [ "$ac" -lt 1 ]; then ac=1; fi - out="0" - i=1 - while [ "$i" -lt "$ac" ]; do - out="$out,$i" - i=$((i + 1)) - done - echo "$out" - register: zeam_all_subnets_raw + echo $((idx % ac)) + register: zeam_node_subnet_raw changed_when: false delegate_to: localhost - run_once: true + when: + - node_name is defined + - (zeam_is_aggregator | default('false')) == 'true' -- name: Set aggregate subnet ids csv +- name: Set aggregate subnet id for aggregator set_fact: - zeam_aggregate_subnet_ids: "{{ zeam_all_subnets_raw.stdout | trim }}" - run_once: true + zeam_aggregate_subnet_ids: "{{ zeam_node_subnet_raw.stdout | trim }}" + when: + - (zeam_is_aggregator | default('false')) == 'true' + - zeam_node_subnet_raw is defined + - zeam_node_subnet_raw.stdout | trim | length > 0 + +- name: Clear aggregate subnet ids for non-aggregator + set_fact: + zeam_aggregate_subnet_ids: "" + when: (zeam_is_aggregator | default('false')) != 'true' - name: Ensure node key file exists stat: @@ -134,7 +139,7 @@ --api-port {{ zeam_api_port }} --metrics-port {{ zeam_metrics_port }} {{ '--is-aggregator' if (zeam_is_aggregator | default('false')) == 'true' else '' }} - {{ ('--aggregate-subnet-ids ' + zeam_aggregate_subnet_ids) if (zeam_is_aggregator | default('false')) == 'true' and (',' in zeam_aggregate_subnet_ids | default('')) else '' }} + {{ ('--aggregate-subnet-ids ' + zeam_aggregate_subnet_ids) if (zeam_is_aggregator | default('false')) == 'true' and (zeam_aggregate_subnet_ids | default('') | length > 0) else '' }} {{ ('--checkpoint-sync-url ' + checkpoint_sync_url) if (checkpoint_sync_url is defined and checkpoint_sync_url | length > 0) else '' }} {{ ('--db-backend ' + zeam_db_backend) if (zeam_db_backend | default('') | length > 0) else '' }} {{ ('--chain-worker ' + zeam_chain_worker) if (zeam_chain_worker | default('') | length > 0) else '' }} diff --git a/client-cmds/ethlambda-cmd.sh b/client-cmds/ethlambda-cmd.sh index 1ea79ee..3326ee9 100644 --- a/client-cmds/ethlambda-cmd.sh +++ b/client-cmds/ethlambda-cmd.sh @@ -10,12 +10,9 @@ if [ "$isAggregator" == "true" ]; then aggregator_flag="--is-aggregator" fi -# In multi-subnet deployments, an aggregator must subscribe to every subnet's -# attestation topics so it can aggregate votes from all committees. The caller -# (spin-node.sh / ansible roles) exports aggregateSubnetIds as a CSV of the -# full subnet id set for the network. +# Aggregators subscribe only to their committee subnet (parse-vc.sh exports aggregateSubnetIds). aggregate_subnet_ids_flag="" -if [ "$isAggregator" == "true" ] && [ -n "${aggregateSubnetIds:-}" ] && [[ "$aggregateSubnetIds" == *,* ]]; then +if [ "$isAggregator" == "true" ] && [ -n "${aggregateSubnetIds:-}" ]; then aggregate_subnet_ids_flag="--aggregate-subnet-ids $aggregateSubnetIds" fi diff --git a/client-cmds/gean-cmd.sh b/client-cmds/gean-cmd.sh index 53af74f..b068ac5 100644 --- a/client-cmds/gean-cmd.sh +++ b/client-cmds/gean-cmd.sh @@ -9,12 +9,9 @@ if [ "$isAggregator" == "true" ]; then aggregator_flag="--is-aggregator" fi -# In multi-subnet deployments, an aggregator must subscribe to every subnet's -# attestation topics so it can aggregate votes from all committees. The caller -# (spin-node.sh / ansible roles) exports aggregateSubnetIds as a CSV of the -# full subnet id set for the network. +# Aggregators subscribe only to their committee subnet (parse-vc.sh exports aggregateSubnetIds). aggregate_subnet_ids_flag="" -if [ "$isAggregator" == "true" ] && [ -n "${aggregateSubnetIds:-}" ] && [[ "$aggregateSubnetIds" == *,* ]]; then +if [ "$isAggregator" == "true" ] && [ -n "${aggregateSubnetIds:-}" ]; then aggregate_subnet_ids_flag="--aggregate-subnet-ids $aggregateSubnetIds" fi diff --git a/client-cmds/grandine-cmd.sh b/client-cmds/grandine-cmd.sh index 8828ed6..300f67e 100644 --- a/client-cmds/grandine-cmd.sh +++ b/client-cmds/grandine-cmd.sh @@ -6,12 +6,9 @@ if [ "$isAggregator" == "true" ]; then aggregator_flag="--is-aggregator" fi -# In multi-subnet deployments, an aggregator must subscribe to every subnet's -# attestation topics so it can aggregate votes from all committees. The caller -# (spin-node.sh / ansible roles) exports aggregateSubnetIds as a CSV of the -# full subnet id set for the network. +# Aggregators subscribe only to their committee subnet (parse-vc.sh exports aggregateSubnetIds). aggregate_subnet_ids_flag="" -if [ "$isAggregator" == "true" ] && [ -n "${aggregateSubnetIds:-}" ] && [[ "$aggregateSubnetIds" == *,* ]]; then +if [ "$isAggregator" == "true" ] && [ -n "${aggregateSubnetIds:-}" ]; then aggregate_subnet_ids_flag="--aggregate-subnet-ids $aggregateSubnetIds" fi diff --git a/client-cmds/lantern-cmd.sh b/client-cmds/lantern-cmd.sh index 19f1b86..2b60c3a 100755 --- a/client-cmds/lantern-cmd.sh +++ b/client-cmds/lantern-cmd.sh @@ -14,12 +14,9 @@ if [ "$isAggregator" == "true" ]; then aggregator_flag="--is-aggregator" fi -# In multi-subnet deployments, an aggregator must subscribe to every subnet's -# attestation topics so it can aggregate votes from all committees. The caller -# (spin-node.sh / ansible roles) exports aggregateSubnetIds as a CSV of the -# full subnet id set for the network. +# Aggregators subscribe only to their committee subnet (parse-vc.sh exports aggregateSubnetIds). aggregate_subnet_ids_flag="" -if [ "$isAggregator" == "true" ] && [ -n "${aggregateSubnetIds:-}" ] && [[ "$aggregateSubnetIds" == *,* ]]; then +if [ "$isAggregator" == "true" ] && [ -n "${aggregateSubnetIds:-}" ]; then aggregate_subnet_ids_flag="--aggregate-subnet-ids $aggregateSubnetIds" fi diff --git a/client-cmds/lighthouse-cmd.sh b/client-cmds/lighthouse-cmd.sh index 4d25b83..664bcd9 100644 --- a/client-cmds/lighthouse-cmd.sh +++ b/client-cmds/lighthouse-cmd.sh @@ -9,12 +9,9 @@ if [ "$isAggregator" == "true" ]; then aggregator_flag="--is-aggregator" fi -# In multi-subnet deployments, an aggregator must subscribe to every subnet's -# attestation topics so it can aggregate votes from all committees. The caller -# (spin-node.sh / ansible roles) exports aggregateSubnetIds as a CSV of the -# full subnet id set for the network. +# Aggregators subscribe only to their committee subnet (parse-vc.sh exports aggregateSubnetIds). aggregate_subnet_ids_flag="" -if [ "$isAggregator" == "true" ] && [ -n "${aggregateSubnetIds:-}" ] && [[ "$aggregateSubnetIds" == *,* ]]; then +if [ "$isAggregator" == "true" ] && [ -n "${aggregateSubnetIds:-}" ]; then aggregate_subnet_ids_flag="--aggregate-subnet-ids $aggregateSubnetIds" fi diff --git a/client-cmds/nlean-cmd.sh b/client-cmds/nlean-cmd.sh index 9d28a5c..718e325 100755 --- a/client-cmds/nlean-cmd.sh +++ b/client-cmds/nlean-cmd.sh @@ -35,12 +35,9 @@ if [[ "${isAggregator:-false}" == "true" ]]; then aggregator_flag="--is-aggregator" fi -# In multi-subnet deployments, an aggregator must subscribe to every subnet's -# attestation topics so it can aggregate votes from all committees. The caller -# (spin-node.sh / ansible roles) exports aggregateSubnetIds as a CSV of the -# full subnet id set for the network. +# Aggregators subscribe only to their committee subnet (parse-vc.sh exports aggregateSubnetIds). aggregate_subnet_ids_flag="" -if [[ "${isAggregator:-false}" == "true" ]] && [[ -n "${aggregateSubnetIds:-}" ]] && [[ "$aggregateSubnetIds" == *,* ]]; then +if [[ "${isAggregator:-false}" == "true" ]] && [[ -n "${aggregateSubnetIds:-}" ]]; then aggregate_subnet_ids_flag="--aggregate-subnet-ids $aggregateSubnetIds" fi diff --git a/client-cmds/peam-cmd.sh b/client-cmds/peam-cmd.sh index d1863b1..153047f 100644 --- a/client-cmds/peam-cmd.sh +++ b/client-cmds/peam-cmd.sh @@ -21,7 +21,14 @@ allowed_topics="/leanconsensus/$topic_domain/block/ssz_snappy,/leanconsensus/$to topic_scores="/leanconsensus/$topic_domain/block/ssz_snappy:2,/leanconsensus/$topic_domain/aggregation/ssz_snappy:1" topic_validators="/leanconsensus/$topic_domain/block/ssz_snappy=block,/leanconsensus/$topic_domain/aggregation/ssz_snappy=aggregation" -for ((subnet = 0; subnet < committee_count; subnet++)); do +# Attestation topics: one committee subnet per node (aggregator or validator). +# parse-vc.sh sets aggregateSubnetIds for aggregators; otherwise use local index. +attestation_subnets="${aggregateSubnetIds:-}" +if [ -z "$attestation_subnets" ]; then + attestation_subnets=$((local_validator_index % committee_count)) +fi +IFS=',' read -r -a _attestation_subnet_arr <<< "$attestation_subnets" +for subnet in "${_attestation_subnet_arr[@]}"; do att_topic="/leanconsensus/$topic_domain/attestation_${subnet}/ssz_snappy" allowed_topics="$allowed_topics,$att_topic" topic_scores="$topic_scores,$att_topic:1" @@ -53,15 +60,10 @@ if [ "$isAggregator" == "true" ]; then aggregator_flag="--is-aggregator" fi -# In multi-subnet deployments, an aggregator must subscribe to every subnet's -# attestation topics so it can aggregate votes from all committees. The caller -# (spin-node.sh / ansible roles) exports aggregateSubnetIds as a CSV of the -# full subnet id set for the network. Note: peam already subscribes to all -# subnets in [0, committee_count) via allowed_topics above; this flag exists -# for contract parity with other clients and is a no-op unless the binary -# recognises it. +# Aggregators: parse-vc.sh exports a single committee subnet in aggregateSubnetIds. +# Peam allowed_topics above should match that subnet when isAggregator is true. aggregate_subnet_ids_flag="" -if [ "$isAggregator" == "true" ] && [ -n "${aggregateSubnetIds:-}" ] && [[ "$aggregateSubnetIds" == *,* ]]; then +if [ "$isAggregator" == "true" ] && [ -n "${aggregateSubnetIds:-}" ]; then aggregate_subnet_ids_flag="--aggregate-subnet-ids $aggregateSubnetIds" fi diff --git a/client-cmds/qlean-cmd.sh b/client-cmds/qlean-cmd.sh index b228e6b..4784350 100644 --- a/client-cmds/qlean-cmd.sh +++ b/client-cmds/qlean-cmd.sh @@ -21,12 +21,9 @@ if [ "$isAggregator" == "true" ]; then aggregator_flag="--is-aggregator" fi -# In multi-subnet deployments, an aggregator must subscribe to every subnet's -# attestation topics so it can aggregate votes from all committees. The caller -# (spin-node.sh / ansible roles) exports aggregateSubnetIds as a CSV of the -# full subnet id set for the network. +# Aggregators subscribe only to their committee subnet (parse-vc.sh exports aggregateSubnetIds). aggregate_subnet_ids_flag="" -if [ "$isAggregator" == "true" ] && [ -n "${aggregateSubnetIds:-}" ] && [[ "$aggregateSubnetIds" == *,* ]]; then +if [ "$isAggregator" == "true" ] && [ -n "${aggregateSubnetIds:-}" ]; then aggregate_subnet_ids_flag="--aggregate-subnet-ids $aggregateSubnetIds" fi diff --git a/client-cmds/ream-cmd.sh b/client-cmds/ream-cmd.sh index 41315c4..f3a8416 100755 --- a/client-cmds/ream-cmd.sh +++ b/client-cmds/ream-cmd.sh @@ -10,12 +10,9 @@ if [ "$isAggregator" == "true" ]; then aggregator_flag="--is-aggregator" fi -# In multi-subnet deployments, an aggregator must subscribe to every subnet's -# attestation topics so it can aggregate votes from all committees. The caller -# (spin-node.sh / ansible roles) exports aggregateSubnetIds as a CSV of the -# full subnet id set for the network. +# Aggregators subscribe only to their committee subnet (parse-vc.sh exports aggregateSubnetIds). aggregate_subnet_ids_flag="" -if [ "$isAggregator" == "true" ] && [ -n "${aggregateSubnetIds:-}" ] && [[ "$aggregateSubnetIds" == *,* ]]; then +if [ "$isAggregator" == "true" ] && [ -n "${aggregateSubnetIds:-}" ]; then aggregate_subnet_ids_flag="--aggregate-subnet-ids $aggregateSubnetIds" fi diff --git a/client-cmds/zeam-cmd.sh b/client-cmds/zeam-cmd.sh index dbe7207..969036b 100644 --- a/client-cmds/zeam-cmd.sh +++ b/client-cmds/zeam-cmd.sh @@ -17,12 +17,11 @@ if [ "$isAggregator" == "true" ]; then aggregator_flag="--is-aggregator" fi -# In multi-subnet deployments, an aggregator must subscribe to every subnet's -# attestation topics so it can aggregate votes from all committees. The caller -# (spin-node.sh / ansible roles) exports aggregateSubnetIds as a CSV of the -# full subnet id set for the network. +# Aggregators subscribe only to their committee subnet (validator_index % +# attestation_committee_count). parse-vc.sh exports that single id in +# aggregateSubnetIds when isAggregator is true. aggregate_subnet_ids_flag="" -if [ "$isAggregator" == "true" ] && [ -n "${aggregateSubnetIds:-}" ] && [[ "$aggregateSubnetIds" == *,* ]]; then +if [ "$isAggregator" == "true" ] && [ -n "${aggregateSubnetIds:-}" ]; then aggregate_subnet_ids_flag="--aggregate-subnet-ids $aggregateSubnetIds" fi diff --git a/local-devnet/genesis/validator-config.yaml b/local-devnet/genesis/validator-config.yaml index 9547cd3..61303c2 100644 --- a/local-devnet/genesis/validator-config.yaml +++ b/local-devnet/genesis/validator-config.yaml @@ -27,27 +27,21 @@ validators: apiPort: 5052 isAggregator: false count: 1 - - name: "qlean_0" - # node id f0af4dcd8864372ca01ae984b9a386f86e5cb582331394db3434fea59ad78bde - # peer id 16Uiu2HAmQj1RDNAxopeeeCFPRr3zhJYmH6DEPHYKmxLViLahWcFE - privkey: "c2bbdac5e876b3e9d4b8b6b8c2bbdac5e876b3e9d4b8b6b8c2bbdac5e876b3e9" + - name: "zeam_1" + privkey: "8d14c7b02d55ca050ef97a3961aa16828837fb363e4e19e4dd0060f58670a2b3" enrFields: - #verify /ip4/127.0.0.1/udp/9003/quic-v1/p2p/16Uiu2HAmQj1RDNAxopeeeCFPRr3zhJYmH6DEPHYKmxLViLahWcFE ip: "127.0.0.1" quic: 9003 metricsPort: 8083 apiPort: 5053 isAggregator: false count: 1 - - name: "lantern_0" - # node id 974c5fd0a680f5e576189123d5808cefd6c748c75effde28c5c3aacbb8d4c652 - # peer id 16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv - privkey: "d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5" + - name: "ethlambda_1" + privkey: "eb8533a0d5071d4dbcb0c4fcf9b8ac6edc3d1a260d2bb348fafc5cdb455aa1d4" enrFields: ip: "127.0.0.1" quic: 9004 metricsPort: 8084 - httpPort: 5054 apiPort: 5054 isAggregator: false count: 1 diff --git a/parse-vc.sh b/parse-vc.sh index a3b0341..5a690ed 100644 --- a/parse-vc.sh +++ b/parse-vc.sh @@ -69,20 +69,17 @@ if [ -z "$isAggregator" ] || [ "$isAggregator" == "null" ]; then isAggregator="false" fi -# CSV of all attestation subnet ids (e.g. "0,1"). Clients do not read a YAML -# `subnet:` field for consensus — subnets are validator_index % committee_count. -# Aggregators must still hear every subnet, so derive ids from -# config.attestation_committee_count (not from per-validator subnet metadata). +# Subnet = validator_index % attestation_committee_count (no per-validator YAML field). +# Aggregators subscribe only to their own committee subnet via --aggregate-subnet-ids. +hashSigKeyIndex=$(yq eval ".validators | to_entries | .[] | select(.value.name == \"$item\") | .key" "$validator_config_file") _ac=$(yq eval '.config.attestation_committee_count // 1' "$validator_config_file") _ac=$(echo "$_ac" | tr -d '\r\n' | head -1) case "$_ac" in ''|*[!0-9]*) _ac=1;; esac if [ "$_ac" -lt 1 ] 2>/dev/null; then _ac=1; fi -aggregateSubnetIds="0" -_i=1 -while [ "$_i" -lt "$_ac" ] 2>/dev/null; do - aggregateSubnetIds+=",$_i" - _i=$((_i + 1)) -done +aggregateSubnetIds="" +if [ "$isAggregator" == "true" ] && [ -n "$hashSigKeyIndex" ] && [ "$hashSigKeyIndex" != "null" ]; then + aggregateSubnetIds=$((hashSigKeyIndex % _ac)) +fi export aggregateSubnetIds # Extract attestation_committee_count from config section (optional - only if explicitly set) @@ -106,8 +103,6 @@ echo "$privKey" > "$configDir/$privKeyPath" # Extract hash-sig key configuration from top-level config keyType=$(yq eval ".config.keyType" "$validator_config_file") -hashSigKeyIndex=$(yq eval ".validators | to_entries | .[] | select(.value.name == \"$item\") | .key" "$validator_config_file") - # Load hash-sig keys if configured if [ "$keyType" == "hash-sig" ] && [ "$hashSigKeyIndex" != "null" ] && [ -n "$hashSigKeyIndex" ]; then # devnet4+: separate proposer + attester keys (hash-sig-cli); legacy: single pk/sk per index (SSZ only) From 5fb131f697e29ffd2fd4821982016f30dc626d87 Mon Sep 17 00:00:00 2001 From: ch4r10t33r Date: Sun, 17 May 2026 23:29:06 +0100 Subject: [PATCH 05/20] spin-node: add --stop-all-containers for ansible hosts Stop every Docker container on each unique validator-config IP except the per-host observability stack (prometheus, promtail, cadvisor, node_exporter). Document in README. --- README.md | 13 ++++ ansible/playbooks/stop-all-containers.yml | 36 ++++++++++++ parse-env.sh | 8 ++- run-ansible.sh | 11 +++- spin-node.sh | 72 +++++++++++++++++++++++ 5 files changed, 136 insertions(+), 4 deletions(-) create mode 100644 ansible/playbooks/stop-all-containers.yml diff --git a/README.md b/README.md index ee1e4a7..e84587b 100644 --- a/README.md +++ b/README.md @@ -252,6 +252,12 @@ Every Ansible deployment automatically deploys an observability stack alongside - `tmp/ansible-run-DD-MM-YYYY-HH-MM.log` for Ansible deployments - Example: `NETWORK_DIR=local-devnet ./spin-node.sh --node all --logs` 19. `--network` sets the network name label attached to every metric and log stream scraped by the observability stack. +20. `--stop-all-containers` stops every Docker container on each unique `enrFields.ip` in the active `validator-config.yaml`, except the per-host observability stack (`prometheus`, `promtail`, `cadvisor`, `node_exporter`). Use this to tear down stray or leftover lean client containers without touching metrics collection. + - **Ansible mode only** — fails if `deployment_mode` is not `ansible` + - Does not require `--node`; runs one SSH session per unique host IP (same deduplicated inventory as `--prepare`) + - Unlike `--stop --node all`, this removes **all** non-observability containers on each host (not only containers whose names match validator rows) + - **Allowed with `--stop-all-containers`:** `--validatorConfig`, `--subnets N`, `--sshKey` / `--private-key`, `--useRoot`, `--deploymentMode ansible`, `--network`, `--dry-run`, `--logs`, and `NETWORK_DIR` + - Example: `NETWORK_DIR=ansible-devnet ./spin-node.sh --stop-all-containers --network devnet-4 --sshKey ~/.ssh/id_ed25519 --useRoot` - **Required for Ansible deployments** — the script exits with an error if omitted when `deployment_mode: ansible` - For local deployments, falls back to the default network name if not specified - Propagated to Ansible as the `network_name` variable, used in `prometheus.yml.j2` and `promtail.yml.j2` templates @@ -281,6 +287,13 @@ NETWORK_DIR=ansible-devnet ./spin-node.sh --prepare --sshKey ~/.ssh/id_ed25519 - - Passing deploy-only flags (e.g. `--node`, `--generateGenesis`, `--stop`, `--metrics`) alongside `--prepare` produces a prominent error. Use `--validatorConfig`, `--subnets N`, `--sshKey`, `--useRoot`, `--deploymentMode ansible`, `--network`, `--dry-run`, or `--logs` when needed so inventory and firewall match your deploy. - `--node` is not required; the prepare playbook runs on **one play per unique IP** (deduplicated inventory) so parallel prepares do not fight over the same host +To stop all lean client containers on every validator host while keeping observability running: + +```bash +NETWORK_DIR=ansible-devnet ./spin-node.sh --stop-all-containers \ + --network devnet-4 --sshKey ~/.ssh/id_ed25519 --useRoot +``` + Once preparation succeeds, proceed with the normal deploy command: ```sh diff --git a/ansible/playbooks/stop-all-containers.yml b/ansible/playbooks/stop-all-containers.yml new file mode 100644 index 0000000..53ba2dc --- /dev/null +++ b/ansible/playbooks/stop-all-containers.yml @@ -0,0 +1,36 @@ +--- +# Stop every Docker container on each unique validator host IP, except the +# per-host observability stack (prometheus, promtail, cadvisor, node_exporter). +# Uses hosts-prepare.yml (one play per physical host). + +- name: Stop non-observability containers on all validator hosts + hosts: prepare_hosts + gather_facts: no + vars: + observability_containers: + - prometheus + - promtail + - cadvisor + - node_exporter + + tasks: + - name: List Docker container names + command: docker ps -a --format '{{ "{{" }}.Names{{ "}}" }}' + register: all_containers + changed_when: false + + - name: Stop and remove non-observability containers + command: docker rm -f {{ item }} + loop: "{{ all_containers.stdout_lines | difference(observability_containers) }}" + when: item | length > 0 + register: stop_result + failed_when: false + changed_when: stop_result.rc == 0 + + - name: Report stopped containers + debug: + msg: "Stopped: {{ item.item }}" + loop: "{{ stop_result.results | default([]) }}" + when: + - item.rc is defined + - item.rc == 0 diff --git a/parse-env.sh b/parse-env.sh index 041c217..8a833d7 100755 --- a/parse-env.sh +++ b/parse-env.sh @@ -111,6 +111,10 @@ while [[ $# -gt 0 ]]; do prepareMode=true shift ;; + --stop-all-containers) + stopAllContainers=true + shift + ;; --deploy-observability) echo "Warning: --deploy-observability is deprecated; use --prepare (observability is included)." prepareMode=true @@ -145,8 +149,8 @@ while [[ $# -gt 0 ]]; do esac done -# if no node and no restart-client specified, exit (unless --prepare mode) -if [[ ! -n "$node" ]] && [[ ! -n "$restartClient" ]] && [[ "$prepareMode" != "true" ]]; +# if no node and no restart-client specified, exit (unless --prepare or --stop-all-containers) +if [[ ! -n "$node" ]] && [[ ! -n "$restartClient" ]] && [[ "$prepareMode" != "true" ]] && [[ "$stopAllContainers" != "true" ]]; then echo "no node or restart-client specified, exiting..." exit diff --git a/run-ansible.sh b/run-ansible.sh index 5c277fd..819ac79 100755 --- a/run-ansible.sh +++ b/run-ansible.sh @@ -74,7 +74,7 @@ fi # prepare.yml: one play per physical host (deduped by IP). Deploy still uses full hosts.yml. EFFECTIVE_INVENTORY="$INVENTORY_FILE" -if { [ "$action" == "prepare" ] || [ "$action" == "observability" ]; } && [ -f "$PREPARE_INVENTORY" ]; then +if { [ "$action" == "prepare" ] || [ "$action" == "observability" ] || [ "$action" == "stop-all-containers" ]; } && [ -f "$PREPARE_INVENTORY" ]; then EFFECTIVE_INVENTORY="$PREPARE_INVENTORY" fi @@ -190,6 +190,9 @@ elif [ "$action" == "prepare" ]; then elif [ "$action" == "observability" ]; then PLAYBOOK="$ANSIBLE_DIR/playbooks/deploy-observability.yml" ACTION_MSG="deploying observability stack" +elif [ "$action" == "stop-all-containers" ]; then + PLAYBOOK="$ANSIBLE_DIR/playbooks/stop-all-containers.yml" + ACTION_MSG="stopping all non-observability containers" else PLAYBOOK="$ANSIBLE_DIR/playbooks/site.yml" ACTION_MSG="deploying nodes" @@ -211,7 +214,7 @@ elif [ -f "$_local_vc_path" ] && command -v yq &>/dev/null; then elif [ -f "$_local_vc_path" ]; then echo "Warning: yq not found; omitting -f (using ansible.cfg forks default)" fi -if [ -n "$_play_forks" ] && { [ "$action" == "prepare" ] || [ "$action" == "observability" ]; }; then +if [ -n "$_play_forks" ] && { [ "$action" == "prepare" ] || [ "$action" == "observability" ] || [ "$action" == "stop-all-containers" ]; }; then _prepare_forks_max="${LEAN_PREPARE_FORKS_MAX:-15}" if (( _play_forks > _prepare_forks_max )); then echo "Prepare: capping forks from ${_play_forks} to ${_prepare_forks_max} (LEAN_PREPARE_FORKS_MAX)" @@ -262,6 +265,8 @@ if [ $EXIT_CODE -eq 0 ]; then echo "✅ Server preparation completed (tools, firewall, observability)!${_dry_tag}" elif [ "$action" == "observability" ]; then echo "✅ Observability stack deployed on all hosts!${_dry_tag}" + elif [ "$action" == "stop-all-containers" ]; then + echo "✅ Stopped all non-observability containers on validator hosts!${_dry_tag}" else echo "✅ Ansible deployment completed successfully!${_dry_tag}" fi @@ -273,6 +278,8 @@ else echo "❌ Server preparation failed with exit code $EXIT_CODE" elif [ "$action" == "observability" ]; then echo "❌ Observability deployment failed with exit code $EXIT_CODE" + elif [ "$action" == "stop-all-containers" ]; then + echo "❌ Stop-all-containers operation failed with exit code $EXIT_CODE" else echo "❌ Ansible deployment failed with exit code $EXIT_CODE" fi diff --git a/spin-node.sh b/spin-node.sh index 39c2e5f..8330a7a 100755 --- a/spin-node.sh +++ b/spin-node.sh @@ -219,6 +219,78 @@ if [ -n "$prepareMode" ] && [ "$prepareMode" == "true" ]; then exit 0 fi +# Handle --stop-all-containers: stop every Docker container on each unique validator +# host IP except the per-host observability stack (prometheus, promtail, cadvisor, +# node_exporter). Does not require --node. +if [ -n "$stopAllContainers" ] && [ "$stopAllContainers" == "true" ]; then + if [ "$deployment_mode" != "ansible" ]; then + echo "Error: --stop-all-containers can only be used in ansible mode." + echo "Set deployment_mode: ansible in your validator-config.yaml or pass --deploymentMode ansible" + exit 1 + fi + + ignored_flags=() + [ -n "$node" ] && ignored_flags+=("--node") + [ -n "$cleanData" ] && ignored_flags+=("--cleanData") + [ -n "$generateGenesis" ] && ignored_flags+=("--generateGenesis") + [ -n "$FORCE_KEYGEN_FLAG" ] && ignored_flags+=("--forceKeyGen") + [ -n "$stopNodes" ] && ignored_flags+=("--stop") + [ -n "$restartClient" ] && ignored_flags+=("--restart-client") + [ -n "$checkpointSyncUrl" ] && ignored_flags+=("--checkpoint-sync-url") + [ -n "$dockerTag" ] && ignored_flags+=("--tag") + [ -n "$aggregatorNode" ] && ignored_flags+=("--aggregator") + [ -n "$coreDumps" ] && ignored_flags+=("--coreDumps") + [ -n "$enableMetrics" ] && ignored_flags+=("--metrics") + [ -n "$popupTerminal" ] && ignored_flags+=("--popupTerminal") + [ -n "$dockerWithSudo" ] && ignored_flags+=("--dockerWithSudo") + [ -n "$skipLeanpoint" ] && ignored_flags+=("--skip-leanpoint") + [ -n "$skipNemo" ] && ignored_flags+=("--skip-nemo") + [ -n "$replaceWith" ] && ignored_flags+=("--replace-with") + + if [ ${#ignored_flags[@]} -gt 0 ]; then + echo "" + echo "╔══════════════════════════════════════════════════════════════╗" + echo "║ ❌ ERROR ║" + echo "╠══════════════════════════════════════════════════════════════╣" + echo "║ --stop-all-containers does not accept the following flag(s): ║" + for flag in "${ignored_flags[@]}"; do + printf "║ %-60s║\n" "• $flag" + done + echo "╠══════════════════════════════════════════════════════════════╣" + echo "║ Allowed flags with --stop-all-containers: ║" + echo "║ • --validatorConfig ║" + echo "║ • --subnets N ║" + echo "║ • --sshKey / --private-key ║" + echo "║ • --useRoot ║" + echo "║ • --deploymentMode ansible ║" + echo "║ • --network, --dry-run, --logs ║" + echo "╚══════════════════════════════════════════════════════════════╝" + echo "" + exit 1 + fi + + if ! command -v ansible-playbook &> /dev/null; then + echo "Error: ansible-playbook is not installed." + echo "Install Ansible: brew install ansible (macOS) or pip install ansible" + exit 1 + fi + + if [ "$dryRun" == "true" ]; then + echo "[DRY RUN] Would stop all non-observability containers on validator hosts" + else + echo "Stopping all non-observability containers on every IP in validator-config.yaml..." + echo "Preserving: prometheus, promtail, cadvisor, node_exporter" + fi + + if ! "$scriptDir/run-ansible.sh" "$configDir" "" "" "" "$validator_config_file" "$sshKeyFile" "$useRoot" "stop-all-containers" "" "" "" "$dryRun" "" "$networkName"; then + echo "❌ Stop-all-containers operation failed." + exit 1 + fi + + [ "$dryRun" == "true" ] && echo "✅ Dry-run complete — no changes were made." || echo "✅ Stopped all non-observability containers on validator hosts." + exit 0 +fi + #1. setup genesis params and run genesis generator if [ "$dryRun" == "true" ]; then echo "[DRY RUN] Skipping genesis generation (set-up.sh would run here)" From 60df7cdbe8ddbcad0e9f77eb4a8fda7a2b4af32b Mon Sep 17 00:00:00 2001 From: ch4r10t33r Date: Sun, 17 May 2026 23:33:17 +0100 Subject: [PATCH 06/20] spin-node: fix stop-all-containers for /bin/sh hosts Use ansible command+loop instead of bash process substitution so the playbook runs under /bin/sh. Clarify that all stale containers are removed, not only validator-config names. --- README.md | 11 +++-------- ansible/playbooks/stop-all-containers.yml | 20 ++++++++++---------- spin-node.sh | 4 ++-- 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index e84587b..ef591d1 100644 --- a/README.md +++ b/README.md @@ -252,17 +252,12 @@ Every Ansible deployment automatically deploys an observability stack alongside - `tmp/ansible-run-DD-MM-YYYY-HH-MM.log` for Ansible deployments - Example: `NETWORK_DIR=local-devnet ./spin-node.sh --node all --logs` 19. `--network` sets the network name label attached to every metric and log stream scraped by the observability stack. -20. `--stop-all-containers` stops every Docker container on each unique `enrFields.ip` in the active `validator-config.yaml`, except the per-host observability stack (`prometheus`, `promtail`, `cadvisor`, `node_exporter`). Use this to tear down stray or leftover lean client containers without touching metrics collection. +20. `--stop-all-containers` runs `docker ps -a` on each unique `enrFields.ip` in the active `validator-config.yaml` and removes **every** container except the per-host observability stack (`prometheus`, `promtail`, `cadvisor`, `node_exporter`). Stale containers from old deploys, manual runs, or other clients are removed even when their names are not listed in `validator-config.yaml`. - **Ansible mode only** — fails if `deployment_mode` is not `ansible` - Does not require `--node`; runs one SSH session per unique host IP (same deduplicated inventory as `--prepare`) - - Unlike `--stop --node all`, this removes **all** non-observability containers on each host (not only containers whose names match validator rows) + - Unlike `--stop --node all`, this is not limited to validator row names — it clears the whole Docker namespace on each host except observability - **Allowed with `--stop-all-containers`:** `--validatorConfig`, `--subnets N`, `--sshKey` / `--private-key`, `--useRoot`, `--deploymentMode ansible`, `--network`, `--dry-run`, `--logs`, and `NETWORK_DIR` - Example: `NETWORK_DIR=ansible-devnet ./spin-node.sh --stop-all-containers --network devnet-4 --sshKey ~/.ssh/id_ed25519 --useRoot` - - **Required for Ansible deployments** — the script exits with an error if omitted when `deployment_mode: ansible` - - For local deployments, falls back to the default network name if not specified - - Propagated to Ansible as the `network_name` variable, used in `prometheus.yml.j2` and `promtail.yml.j2` templates - - Appears as the `network` label on all Prometheus scrape targets and Promtail log streams, so you can filter by network in Grafana - - Example: `--network ` ### Preparing remote servers @@ -287,7 +282,7 @@ NETWORK_DIR=ansible-devnet ./spin-node.sh --prepare --sshKey ~/.ssh/id_ed25519 - - Passing deploy-only flags (e.g. `--node`, `--generateGenesis`, `--stop`, `--metrics`) alongside `--prepare` produces a prominent error. Use `--validatorConfig`, `--subnets N`, `--sshKey`, `--useRoot`, `--deploymentMode ansible`, `--network`, `--dry-run`, or `--logs` when needed so inventory and firewall match your deploy. - `--node` is not required; the prepare playbook runs on **one play per unique IP** (deduplicated inventory) so parallel prepares do not fight over the same host -To stop all lean client containers on every validator host while keeping observability running: +To stop every non-observability container on each validator host (including stale containers not in `validator-config.yaml`) while keeping observability running: ```bash NETWORK_DIR=ansible-devnet ./spin-node.sh --stop-all-containers \ diff --git a/ansible/playbooks/stop-all-containers.yml b/ansible/playbooks/stop-all-containers.yml index 53ba2dc..d9ab873 100644 --- a/ansible/playbooks/stop-all-containers.yml +++ b/ansible/playbooks/stop-all-containers.yml @@ -2,6 +2,9 @@ # Stop every Docker container on each unique validator host IP, except the # per-host observability stack (prometheus, promtail, cadvisor, node_exporter). # Uses hosts-prepare.yml (one play per physical host). +# +# This is intentionally not limited to validator names from validator-config.yaml: +# any container on the host is removed unless it is part of the observability stack. - name: Stop non-observability containers on all validator hosts hosts: prepare_hosts @@ -14,23 +17,20 @@ - node_exporter tasks: - - name: List Docker container names + - name: List all Docker container names command: docker ps -a --format '{{ "{{" }}.Names{{ "}}" }}' register: all_containers changed_when: false - - name: Stop and remove non-observability containers + - name: Stop and remove every container except observability stack command: docker rm -f {{ item }} - loop: "{{ all_containers.stdout_lines | difference(observability_containers) }}" - when: item | length > 0 + loop: "{{ all_containers.stdout_lines | difference(observability_containers) | select('length') | list }}" register: stop_result failed_when: false - changed_when: stop_result.rc == 0 - - name: Report stopped containers + - name: Report removed containers debug: - msg: "Stopped: {{ item.item }}" - loop: "{{ stop_result.results | default([]) }}" + msg: "Removed on {{ inventory_hostname }} ({{ ansible_host }}): {{ stop_result.results | selectattr('rc', 'defined') | selectattr('rc', 'equalto', 0) | map(attribute='item') | list | join(', ') }}" when: - - item.rc is defined - - item.rc == 0 + - stop_result.results is defined + - stop_result.results | selectattr('rc', 'defined') | selectattr('rc', 'equalto', 0) | list | length > 0 diff --git a/spin-node.sh b/spin-node.sh index 8330a7a..08286ea 100755 --- a/spin-node.sh +++ b/spin-node.sh @@ -278,8 +278,8 @@ if [ -n "$stopAllContainers" ] && [ "$stopAllContainers" == "true" ]; then if [ "$dryRun" == "true" ]; then echo "[DRY RUN] Would stop all non-observability containers on validator hosts" else - echo "Stopping all non-observability containers on every IP in validator-config.yaml..." - echo "Preserving: prometheus, promtail, cadvisor, node_exporter" + echo "Stopping every Docker container on each validator host IP (including stale containers not in validator-config.yaml)..." + echo "Preserving observability only: prometheus, promtail, cadvisor, node_exporter" fi if ! "$scriptDir/run-ansible.sh" "$configDir" "" "" "" "$validator_config_file" "$sshKeyFile" "$useRoot" "stop-all-containers" "" "" "" "$dryRun" "" "$networkName"; then From a1297444a2b82a5049bafebab0c94493b2daadf0 Mon Sep 17 00:00:00 2001 From: ch4r10t33r Date: Mon, 18 May 2026 22:20:31 +0100 Subject: [PATCH 07/20] ansible: auto-restart containers and docker daemon on OOM/crash Configure unless-stopped on all containers via prepare/deploy, systemd Restart=always for docker.service, and a shared group_vars policy for new docker run invocations. --- ansible/inventory/group_vars/all.yml | 7 ++++ ansible/playbooks/prepare.yml | 8 ++++ ansible/roles/common/defaults/main.yml | 2 + ansible/roles/common/handlers/main.yml | 11 +++++ .../common/tasks/docker-restart-policy.yml | 41 +++++++++++++++++++ ansible/roles/common/tasks/main.yml | 4 ++ ansible/roles/ethlambda/tasks/main.yml | 2 +- ansible/roles/gean/tasks/main.yml | 2 +- ansible/roles/grandine/tasks/main.yml | 2 +- ansible/roles/lantern/tasks/main.yml | 2 +- ansible/roles/lighthouse/tasks/main.yml | 2 +- ansible/roles/nlean/tasks/main.yml | 2 +- ansible/roles/observability/tasks/main.yml | 8 ++-- ansible/roles/peam/tasks/main.yml | 2 +- ansible/roles/qlean/tasks/main.yml | 2 +- ansible/roles/ream/tasks/main.yml | 2 +- ansible/roles/zeam/tasks/main.yml | 4 +- 17 files changed, 88 insertions(+), 15 deletions(-) create mode 100644 ansible/roles/common/handlers/main.yml create mode 100644 ansible/roles/common/tasks/docker-restart-policy.yml diff --git a/ansible/inventory/group_vars/all.yml b/ansible/inventory/group_vars/all.yml index 461977a..3930fbe 100644 --- a/ansible/inventory/group_vars/all.yml +++ b/ansible/inventory/group_vars/all.yml @@ -21,6 +21,13 @@ remote_data_dir: "/opt/lean-quickstart/data" docker_with_sudo: false docker_network_mode: host +# Restart node/observability containers after OOM kill or crash (Docker exit != 0). +# Applied on --prepare and deploy via the common role; new containers use this in docker run. +docker_container_restart_policy: unless-stopped + +# Keep the Docker daemon running if systemd stops it (host-level). +docker_systemd_restart: always + # Node deployment mode: 'docker' or 'binary' deployment_mode: docker diff --git a/ansible/playbooks/prepare.yml b/ansible/playbooks/prepare.yml index eb6c60b..ec6b533 100644 --- a/ansible/playbooks/prepare.yml +++ b/ansible/playbooks/prepare.yml @@ -173,6 +173,14 @@ - ansible_os_family == "Debian" - docker_post.rc == 0 + - name: Configure Docker and container auto-restart (OOM/crash) + include_role: + name: common + tasks_from: docker-restart-policy.yml + vars: + is_localhost: false + when: docker_post.rc == 0 + # ────────────────────────────────────────────────────────────────────────── # 3. yq — the common role hard-fails at every deploy if this is absent # ────────────────────────────────────────────────────────────────────────── diff --git a/ansible/roles/common/defaults/main.yml b/ansible/roles/common/defaults/main.yml index d153504..0fa8f70 100644 --- a/ansible/roles/common/defaults/main.yml +++ b/ansible/roles/common/defaults/main.yml @@ -3,4 +3,6 @@ docker_version: latest yq_install_method: "auto" # auto, brew, binary +docker_container_restart_policy: unless-stopped +docker_systemd_restart: always diff --git a/ansible/roles/common/handlers/main.yml b/ansible/roles/common/handlers/main.yml new file mode 100644 index 0000000..f3aedcb --- /dev/null +++ b/ansible/roles/common/handlers/main.yml @@ -0,0 +1,11 @@ +--- +- name: Reload systemd + systemd: + daemon_reload: yes + become: yes + +- name: Restart docker + systemd: + name: docker + state: restarted + become: yes diff --git a/ansible/roles/common/tasks/docker-restart-policy.yml b/ansible/roles/common/tasks/docker-restart-policy.yml new file mode 100644 index 0000000..3173e19 --- /dev/null +++ b/ansible/roles/common/tasks/docker-restart-policy.yml @@ -0,0 +1,41 @@ +--- +# Host-level auto-restart for Docker workloads (OOM kill, crash, daemon recovery). +# Configures systemd for the Docker service and sets restart policy on all containers. + +- name: Ensure Docker systemd drop-in directory exists + file: + path: /etc/systemd/system/docker.service.d + state: directory + mode: '0755' + become: yes + when: not is_localhost | default(false) + +- name: Configure Docker daemon systemd auto-restart + copy: + dest: /etc/systemd/system/docker.service.d/restart-on-failure.conf + mode: '0644' + content: | + [Service] + Restart={{ docker_systemd_restart }} + RestartSec=5 + become: yes + notify: + - Reload systemd + - Restart docker + when: not is_localhost | default(false) + +- name: List all Docker container names + command: docker ps -a --format '{{ "{{" }}.Names{{ "}}" }}' + register: docker_all_container_names + changed_when: false + when: not is_localhost | default(false) + +- name: Set restart policy on all containers (OOM/crash recovery) + command: docker update --restart={{ docker_container_restart_policy }} {{ item }} + loop: "{{ docker_all_container_names.stdout_lines | default([]) | reject('equalto', '') | list }}" + register: docker_restart_policy_update + changed_when: docker_restart_policy_update.rc == 0 + failed_when: false + when: + - not is_localhost | default(false) + - docker_all_container_names.stdout_lines | default([]) | length > 0 diff --git a/ansible/roles/common/tasks/main.yml b/ansible/roles/common/tasks/main.yml index e82f1f3..7936380 100644 --- a/ansible/roles/common/tasks/main.yml +++ b/ansible/roles/common/tasks/main.yml @@ -142,3 +142,7 @@ mode: '0755' when: node_name is defined +- name: Configure Docker auto-restart on remote hosts + include_tasks: docker-restart-policy.yml + when: not is_localhost | default(false) + diff --git a/ansible/roles/ethlambda/tasks/main.yml b/ansible/roles/ethlambda/tasks/main.yml index 8e80f29..0a37dfd 100644 --- a/ansible/roles/ethlambda/tasks/main.yml +++ b/ansible/roles/ethlambda/tasks/main.yml @@ -121,7 +121,7 @@ docker run -d --pull=always --name {{ node_name }} - --restart unless-stopped + --restart {{ docker_container_restart_policy }} --network host {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} -v {{ genesis_dir }}:/config:ro diff --git a/ansible/roles/gean/tasks/main.yml b/ansible/roles/gean/tasks/main.yml index 8ef1617..bed127e 100644 --- a/ansible/roles/gean/tasks/main.yml +++ b/ansible/roles/gean/tasks/main.yml @@ -85,7 +85,7 @@ docker run -d --pull=always --name {{ node_name }} - --restart unless-stopped + --restart {{ docker_container_restart_policy }} --network host {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} -v {{ genesis_dir }}:/config:ro diff --git a/ansible/roles/grandine/tasks/main.yml b/ansible/roles/grandine/tasks/main.yml index 4941385..6bb7a93 100644 --- a/ansible/roles/grandine/tasks/main.yml +++ b/ansible/roles/grandine/tasks/main.yml @@ -91,7 +91,7 @@ docker run -d --pull=always --name {{ node_name }} - --restart unless-stopped + --restart {{ docker_container_restart_policy }} --network host {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} -v {{ genesis_dir }}:/config:ro diff --git a/ansible/roles/lantern/tasks/main.yml b/ansible/roles/lantern/tasks/main.yml index 292f395..f3c18f3 100644 --- a/ansible/roles/lantern/tasks/main.yml +++ b/ansible/roles/lantern/tasks/main.yml @@ -76,7 +76,7 @@ docker run -d --pull=always --name {{ node_name }} - --restart unless-stopped + --restart {{ docker_container_restart_policy }} --network host {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} -v {{ genesis_dir }}:/config:ro diff --git a/ansible/roles/lighthouse/tasks/main.yml b/ansible/roles/lighthouse/tasks/main.yml index 1491202..c80eab9 100644 --- a/ansible/roles/lighthouse/tasks/main.yml +++ b/ansible/roles/lighthouse/tasks/main.yml @@ -88,7 +88,7 @@ command: >- docker run -d --name {{ node_name }} - --restart unless-stopped + --restart {{ docker_container_restart_policy }} --network host {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} -v {{ genesis_dir }}:/config:ro diff --git a/ansible/roles/nlean/tasks/main.yml b/ansible/roles/nlean/tasks/main.yml index 6dbd41b..03c713e 100644 --- a/ansible/roles/nlean/tasks/main.yml +++ b/ansible/roles/nlean/tasks/main.yml @@ -85,7 +85,7 @@ docker run -d --pull=always --name {{ node_name }} - --restart unless-stopped + --restart {{ docker_container_restart_policy }} --network host {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} -v {{ genesis_dir }}:/config:ro diff --git a/ansible/roles/observability/tasks/main.yml b/ansible/roles/observability/tasks/main.yml index 9e1359f..a3c2664 100644 --- a/ansible/roles/observability/tasks/main.yml +++ b/ansible/roles/observability/tasks/main.yml @@ -59,7 +59,7 @@ command: >- docker run -d --name cadvisor - --restart unless-stopped + --restart {{ docker_container_restart_policy }} --network host --privileged --device=/dev/kmsg @@ -86,7 +86,7 @@ command: >- docker run -d --name node_exporter - --restart unless-stopped + --restart {{ docker_container_restart_policy }} --network host --pid host -v /proc:/host/proc:ro @@ -118,7 +118,7 @@ command: >- docker run -d --name prometheus - --restart unless-stopped + --restart {{ docker_container_restart_policy }} --network host -v {{ observability_dir }}/prometheus.yml:/etc/prometheus/prometheus.yml:ro -v {{ observability_dir }}/prometheus-data:/prometheus @@ -139,7 +139,7 @@ command: >- docker run -d --name promtail - --restart unless-stopped + --restart {{ docker_container_restart_policy }} --network host -v {{ observability_dir }}/promtail.yml:/etc/promtail/config.yml:ro -v /var/run/docker.sock:/var/run/docker.sock:ro diff --git a/ansible/roles/peam/tasks/main.yml b/ansible/roles/peam/tasks/main.yml index e45d6fe..60066f6 100644 --- a/ansible/roles/peam/tasks/main.yml +++ b/ansible/roles/peam/tasks/main.yml @@ -136,7 +136,7 @@ docker run -d --pull=always --name {{ node_name }} - --restart unless-stopped + --restart {{ docker_container_restart_policy }} --network host {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} -v {{ genesis_dir }}:/config:ro diff --git a/ansible/roles/qlean/tasks/main.yml b/ansible/roles/qlean/tasks/main.yml index bfb5995..9a6a1b6 100644 --- a/ansible/roles/qlean/tasks/main.yml +++ b/ansible/roles/qlean/tasks/main.yml @@ -83,7 +83,7 @@ docker run -d --pull=always --name {{ node_name }} - --restart unless-stopped + --restart {{ docker_container_restart_policy }} --network host --platform {{ qlean_docker_platform }} {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} diff --git a/ansible/roles/ream/tasks/main.yml b/ansible/roles/ream/tasks/main.yml index c01cb93..b5b64ed 100644 --- a/ansible/roles/ream/tasks/main.yml +++ b/ansible/roles/ream/tasks/main.yml @@ -84,7 +84,7 @@ docker run -d --pull=always --name {{ node_name }} - --restart unless-stopped + --restart {{ docker_container_restart_policy }} --network host {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} -v {{ genesis_dir }}:/config:ro diff --git a/ansible/roles/zeam/tasks/main.yml b/ansible/roles/zeam/tasks/main.yml index 618f176..f7cbb9b 100644 --- a/ansible/roles/zeam/tasks/main.yml +++ b/ansible/roles/zeam/tasks/main.yml @@ -31,7 +31,7 @@ - name: Set docker image and deployment mode from client-cmd.sh set_fact: - zeam_docker_image: "{{ zeam_docker_image_raw.stdout | trim | default('blockblaz/zeam:devnet4') }}" + zeam_docker_image: "{{ zeam_docker_image_raw.stdout | trim | default('0xpartha/zeam:local') }}" deployment_mode: "{{ zeam_deployment_mode_raw.stdout | trim | default('docker') }}" delegate_to: localhost run_once: true @@ -121,7 +121,7 @@ docker run -d --pull=always --name {{ node_name }} - --restart unless-stopped + --restart {{ docker_container_restart_policy }} --network host --security-opt seccomp=unconfined {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} From f6bdddbc7b7b12ca11f839f00f96519ce897cc82 Mon Sep 17 00:00:00 2001 From: ch4r10t33r Date: Mon, 18 May 2026 22:58:23 +0100 Subject: [PATCH 08/20] ansible: cap container memory to stop ream/zeam host OOM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kernel logs showed ream using up to ~15GiB RSS on 16GiB hosts with 2–3 validators per IP. Add per-client docker --memory limits, tighter limits on the 8GiB host, and run docker-restart-policy only on prepare (not mid-deploy). --- ansible/inventory/group_vars/all.yml | 18 +++++++++++++++++- ansible/inventory/host_vars/157.90.254.146.yml | 14 ++++++++++++++ ansible/roles/common/tasks/main.yml | 4 ---- ansible/roles/ethlambda/tasks/main.yml | 2 ++ ansible/roles/gean/tasks/main.yml | 2 ++ ansible/roles/grandine/tasks/main.yml | 2 ++ ansible/roles/lantern/tasks/main.yml | 2 ++ ansible/roles/lighthouse/tasks/main.yml | 2 ++ ansible/roles/nlean/tasks/main.yml | 2 ++ ansible/roles/peam/tasks/main.yml | 2 ++ ansible/roles/qlean/tasks/main.yml | 2 ++ ansible/roles/ream/tasks/main.yml | 2 ++ ansible/roles/zeam/tasks/main.yml | 2 ++ 13 files changed, 51 insertions(+), 5 deletions(-) create mode 100644 ansible/inventory/host_vars/157.90.254.146.yml diff --git a/ansible/inventory/group_vars/all.yml b/ansible/inventory/group_vars/all.yml index 3930fbe..0a25071 100644 --- a/ansible/inventory/group_vars/all.yml +++ b/ansible/inventory/group_vars/all.yml @@ -22,12 +22,28 @@ docker_with_sudo: false docker_network_mode: host # Restart node/observability containers after OOM kill or crash (Docker exit != 0). -# Applied on --prepare and deploy via the common role; new containers use this in docker run. +# Applied on --prepare (host config) and in docker run on deploy. docker_container_restart_policy: unless-stopped # Keep the Docker daemon running if systemd stops it (host-level). docker_systemd_restart: always +# Per-client cgroup memory caps (issue: ream was ~15GiB RSS on 16GiB hosts → kernel OOM). +# --memory-swap matches --memory so containers cannot expand into swap and evict neighbors. +# Budget for 16GiB host: observability ~0.3GiB + up to 3 validators; 8GiB hosts need ≤2 validators +# or lower limits via host_vars. +docker_memory_limits: + zeam: "4g" + ream: "5g" + ethlambda: "1536m" + gean: "3g" + nlean: "3g" + qlean: "3g" + lantern: "3g" + grandine: "3g" + lighthouse: "3g" + peam: "3g" + # Node deployment mode: 'docker' or 'binary' deployment_mode: docker diff --git a/ansible/inventory/host_vars/157.90.254.146.yml b/ansible/inventory/host_vars/157.90.254.146.yml new file mode 100644 index 0000000..e6ad270 --- /dev/null +++ b/ansible/inventory/host_vars/157.90.254.146.yml @@ -0,0 +1,14 @@ +--- +# 8GiB Hetzner host (ubuntu-8gb-fsn1-1): three validators in validator-config.yaml +# cannot use the default 16GiB limits without kernel OOM. +docker_memory_limits: + zeam: "2g" + ream: "3g" + ethlambda: "1200m" + gean: "2g" + nlean: "2g" + qlean: "2g" + lantern: "2g" + grandine: "2g" + lighthouse: "2g" + peam: "2g" diff --git a/ansible/roles/common/tasks/main.yml b/ansible/roles/common/tasks/main.yml index 7936380..e82f1f3 100644 --- a/ansible/roles/common/tasks/main.yml +++ b/ansible/roles/common/tasks/main.yml @@ -142,7 +142,3 @@ mode: '0755' when: node_name is defined -- name: Configure Docker auto-restart on remote hosts - include_tasks: docker-restart-policy.yml - when: not is_localhost | default(false) - diff --git a/ansible/roles/ethlambda/tasks/main.yml b/ansible/roles/ethlambda/tasks/main.yml index 0a37dfd..b6c974e 100644 --- a/ansible/roles/ethlambda/tasks/main.yml +++ b/ansible/roles/ethlambda/tasks/main.yml @@ -122,6 +122,8 @@ --pull=always --name {{ node_name }} --restart {{ docker_container_restart_policy }} + --memory {{ docker_memory_limits.ethlambda }} + --memory-swap {{ docker_memory_limits.ethlambda }} --network host {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} -v {{ genesis_dir }}:/config:ro diff --git a/ansible/roles/gean/tasks/main.yml b/ansible/roles/gean/tasks/main.yml index bed127e..770cf27 100644 --- a/ansible/roles/gean/tasks/main.yml +++ b/ansible/roles/gean/tasks/main.yml @@ -86,6 +86,8 @@ --pull=always --name {{ node_name }} --restart {{ docker_container_restart_policy }} + --memory {{ docker_memory_limits.gean }} + --memory-swap {{ docker_memory_limits.gean }} --network host {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} -v {{ genesis_dir }}:/config:ro diff --git a/ansible/roles/grandine/tasks/main.yml b/ansible/roles/grandine/tasks/main.yml index 6bb7a93..b40fb71 100644 --- a/ansible/roles/grandine/tasks/main.yml +++ b/ansible/roles/grandine/tasks/main.yml @@ -92,6 +92,8 @@ --pull=always --name {{ node_name }} --restart {{ docker_container_restart_policy }} + --memory {{ docker_memory_limits.grandine }} + --memory-swap {{ docker_memory_limits.grandine }} --network host {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} -v {{ genesis_dir }}:/config:ro diff --git a/ansible/roles/lantern/tasks/main.yml b/ansible/roles/lantern/tasks/main.yml index f3c18f3..c7a49ca 100644 --- a/ansible/roles/lantern/tasks/main.yml +++ b/ansible/roles/lantern/tasks/main.yml @@ -77,6 +77,8 @@ --pull=always --name {{ node_name }} --restart {{ docker_container_restart_policy }} + --memory {{ docker_memory_limits.lantern }} + --memory-swap {{ docker_memory_limits.lantern }} --network host {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} -v {{ genesis_dir }}:/config:ro diff --git a/ansible/roles/lighthouse/tasks/main.yml b/ansible/roles/lighthouse/tasks/main.yml index c80eab9..d8b3add 100644 --- a/ansible/roles/lighthouse/tasks/main.yml +++ b/ansible/roles/lighthouse/tasks/main.yml @@ -89,6 +89,8 @@ docker run -d --name {{ node_name }} --restart {{ docker_container_restart_policy }} + --memory {{ docker_memory_limits.lighthouse }} + --memory-swap {{ docker_memory_limits.lighthouse }} --network host {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} -v {{ genesis_dir }}:/config:ro diff --git a/ansible/roles/nlean/tasks/main.yml b/ansible/roles/nlean/tasks/main.yml index 03c713e..ca2cfa5 100644 --- a/ansible/roles/nlean/tasks/main.yml +++ b/ansible/roles/nlean/tasks/main.yml @@ -86,6 +86,8 @@ --pull=always --name {{ node_name }} --restart {{ docker_container_restart_policy }} + --memory {{ docker_memory_limits.nlean }} + --memory-swap {{ docker_memory_limits.nlean }} --network host {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} -v {{ genesis_dir }}:/config:ro diff --git a/ansible/roles/peam/tasks/main.yml b/ansible/roles/peam/tasks/main.yml index 60066f6..43bbf83 100644 --- a/ansible/roles/peam/tasks/main.yml +++ b/ansible/roles/peam/tasks/main.yml @@ -137,6 +137,8 @@ --pull=always --name {{ node_name }} --restart {{ docker_container_restart_policy }} + --memory {{ docker_memory_limits.peam }} + --memory-swap {{ docker_memory_limits.peam }} --network host {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} -v {{ genesis_dir }}:/config:ro diff --git a/ansible/roles/qlean/tasks/main.yml b/ansible/roles/qlean/tasks/main.yml index 9a6a1b6..3c4fa02 100644 --- a/ansible/roles/qlean/tasks/main.yml +++ b/ansible/roles/qlean/tasks/main.yml @@ -84,6 +84,8 @@ --pull=always --name {{ node_name }} --restart {{ docker_container_restart_policy }} + --memory {{ docker_memory_limits.qlean }} + --memory-swap {{ docker_memory_limits.qlean }} --network host --platform {{ qlean_docker_platform }} {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} diff --git a/ansible/roles/ream/tasks/main.yml b/ansible/roles/ream/tasks/main.yml index b5b64ed..6bf729a 100644 --- a/ansible/roles/ream/tasks/main.yml +++ b/ansible/roles/ream/tasks/main.yml @@ -85,6 +85,8 @@ --pull=always --name {{ node_name }} --restart {{ docker_container_restart_policy }} + --memory {{ docker_memory_limits.ream }} + --memory-swap {{ docker_memory_limits.ream }} --network host {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} -v {{ genesis_dir }}:/config:ro diff --git a/ansible/roles/zeam/tasks/main.yml b/ansible/roles/zeam/tasks/main.yml index f7cbb9b..f813f2f 100644 --- a/ansible/roles/zeam/tasks/main.yml +++ b/ansible/roles/zeam/tasks/main.yml @@ -122,6 +122,8 @@ --pull=always --name {{ node_name }} --restart {{ docker_container_restart_policy }} + --memory {{ docker_memory_limits.zeam }} + --memory-swap {{ docker_memory_limits.zeam }} --network host --security-opt seccomp=unconfined {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} From 713dfe67db33dbc2afd7eac34f9db0eaa5dade52 Mon Sep 17 00:00:00 2001 From: ch4r10t33r Date: Mon, 18 May 2026 23:27:29 +0100 Subject: [PATCH 09/20] ansible: use uniform 3g memory limits on 8gib host Set all client docker_memory_limits to 3g on 157.90.254.146. --- ansible/inventory/host_vars/157.90.254.146.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ansible/inventory/host_vars/157.90.254.146.yml b/ansible/inventory/host_vars/157.90.254.146.yml index e6ad270..7de5ce7 100644 --- a/ansible/inventory/host_vars/157.90.254.146.yml +++ b/ansible/inventory/host_vars/157.90.254.146.yml @@ -2,13 +2,13 @@ # 8GiB Hetzner host (ubuntu-8gb-fsn1-1): three validators in validator-config.yaml # cannot use the default 16GiB limits without kernel OOM. docker_memory_limits: - zeam: "2g" + zeam: "3g" ream: "3g" - ethlambda: "1200m" - gean: "2g" - nlean: "2g" - qlean: "2g" - lantern: "2g" - grandine: "2g" - lighthouse: "2g" - peam: "2g" + ethlambda: "3g" + gean: "3g" + nlean: "3g" + qlean: "3g" + lantern: "3g" + grandine: "3g" + lighthouse: "3g" + peam: "3g" From f43e360843a518f27d781f6f7339e3f635ecdfa6 Mon Sep 17 00:00:00 2001 From: ch4r10t33r Date: Mon, 18 May 2026 23:35:14 +0100 Subject: [PATCH 10/20] ansible: remove host_vars for deleted 8gib server Drop 157.90.254.146 overrides; all devnet hosts use 16gib defaults. --- ansible/inventory/group_vars/all.yml | 3 +-- ansible/inventory/host_vars/157.90.254.146.yml | 14 -------------- 2 files changed, 1 insertion(+), 16 deletions(-) delete mode 100644 ansible/inventory/host_vars/157.90.254.146.yml diff --git a/ansible/inventory/group_vars/all.yml b/ansible/inventory/group_vars/all.yml index 0a25071..5bbc98e 100644 --- a/ansible/inventory/group_vars/all.yml +++ b/ansible/inventory/group_vars/all.yml @@ -30,8 +30,7 @@ docker_systemd_restart: always # Per-client cgroup memory caps (issue: ream was ~15GiB RSS on 16GiB hosts → kernel OOM). # --memory-swap matches --memory so containers cannot expand into swap and evict neighbors. -# Budget for 16GiB host: observability ~0.3GiB + up to 3 validators; 8GiB hosts need ≤2 validators -# or lower limits via host_vars. +# Budget for 16GiB host: observability ~0.3GiB + up to 3 validators per IP. docker_memory_limits: zeam: "4g" ream: "5g" diff --git a/ansible/inventory/host_vars/157.90.254.146.yml b/ansible/inventory/host_vars/157.90.254.146.yml deleted file mode 100644 index 7de5ce7..0000000 --- a/ansible/inventory/host_vars/157.90.254.146.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -# 8GiB Hetzner host (ubuntu-8gb-fsn1-1): three validators in validator-config.yaml -# cannot use the default 16GiB limits without kernel OOM. -docker_memory_limits: - zeam: "3g" - ream: "3g" - ethlambda: "3g" - gean: "3g" - nlean: "3g" - qlean: "3g" - lantern: "3g" - grandine: "3g" - lighthouse: "3g" - peam: "3g" From 6307816bf4d154b499124b9210334b3da6dab57c Mon Sep 17 00:00:00 2001 From: ch4r10t33r Date: Tue, 19 May 2026 18:04:13 +0100 Subject: [PATCH 11/20] ansible: 4g memory cap for shared validators; aggregators exempt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Set every per-client docker_memory_limits entry to a uniform 4g and make each role's docker run skip --memory/--memory-swap when the node is an aggregator (one container per IP, no co-tenant memory pressure). Previously ream (5g), ethlambda (1.5g) and the rest (3g) were inconsistent, and the cap was applied unconditionally — so aggregators were also being throttled even though they own their host. --- ansible/inventory/group_vars/all.yml | 28 ++++++++++++++----------- ansible/roles/ethlambda/tasks/main.yml | 4 ++-- ansible/roles/gean/tasks/main.yml | 4 ++-- ansible/roles/grandine/tasks/main.yml | 4 ++-- ansible/roles/lantern/tasks/main.yml | 4 ++-- ansible/roles/lighthouse/tasks/main.yml | 4 ++-- ansible/roles/nlean/tasks/main.yml | 4 ++-- ansible/roles/peam/tasks/main.yml | 4 ++-- ansible/roles/qlean/tasks/main.yml | 4 ++-- ansible/roles/ream/tasks/main.yml | 4 ++-- ansible/roles/zeam/tasks/main.yml | 4 ++-- 11 files changed, 36 insertions(+), 32 deletions(-) diff --git a/ansible/inventory/group_vars/all.yml b/ansible/inventory/group_vars/all.yml index 5bbc98e..66d80d4 100644 --- a/ansible/inventory/group_vars/all.yml +++ b/ansible/inventory/group_vars/all.yml @@ -28,20 +28,24 @@ docker_container_restart_policy: unless-stopped # Keep the Docker daemon running if systemd stops it (host-level). docker_systemd_restart: always -# Per-client cgroup memory caps (issue: ream was ~15GiB RSS on 16GiB hosts → kernel OOM). -# --memory-swap matches --memory so containers cannot expand into swap and evict neighbors. -# Budget for 16GiB host: observability ~0.3GiB + up to 3 validators per IP. +# Per-client cgroup memory caps for shared (non-aggregator) hosts. +# Aggregators are exempt: each role skips --memory/--memory-swap when +# its `*_is_aggregator` fact is "true" (one container per IP, no +# co-tenant pressure). Budget for 16GiB shared host: observability +# ~0.3GiB + up to 3 validators per IP × 4GiB cap = ~12.3GiB worst case. +# --memory-swap matches --memory so containers cannot expand into swap +# and evict neighbors. docker_memory_limits: zeam: "4g" - ream: "5g" - ethlambda: "1536m" - gean: "3g" - nlean: "3g" - qlean: "3g" - lantern: "3g" - grandine: "3g" - lighthouse: "3g" - peam: "3g" + ream: "4g" + ethlambda: "4g" + gean: "4g" + nlean: "4g" + qlean: "4g" + lantern: "4g" + grandine: "4g" + lighthouse: "4g" + peam: "4g" # Node deployment mode: 'docker' or 'binary' deployment_mode: docker diff --git a/ansible/roles/ethlambda/tasks/main.yml b/ansible/roles/ethlambda/tasks/main.yml index b6c974e..e5aa7f5 100644 --- a/ansible/roles/ethlambda/tasks/main.yml +++ b/ansible/roles/ethlambda/tasks/main.yml @@ -122,8 +122,8 @@ --pull=always --name {{ node_name }} --restart {{ docker_container_restart_policy }} - --memory {{ docker_memory_limits.ethlambda }} - --memory-swap {{ docker_memory_limits.ethlambda }} + {{ ('--memory ' + docker_memory_limits.ethlambda) if (ethlambda_is_aggregator | default('false')) != 'true' else '' }} + {{ ('--memory-swap ' + docker_memory_limits.ethlambda) if (ethlambda_is_aggregator | default('false')) != 'true' else '' }} --network host {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} -v {{ genesis_dir }}:/config:ro diff --git a/ansible/roles/gean/tasks/main.yml b/ansible/roles/gean/tasks/main.yml index 770cf27..4435d60 100644 --- a/ansible/roles/gean/tasks/main.yml +++ b/ansible/roles/gean/tasks/main.yml @@ -86,8 +86,8 @@ --pull=always --name {{ node_name }} --restart {{ docker_container_restart_policy }} - --memory {{ docker_memory_limits.gean }} - --memory-swap {{ docker_memory_limits.gean }} + {{ ('--memory ' + docker_memory_limits.gean) if (gean_is_aggregator | default('false')) != 'true' else '' }} + {{ ('--memory-swap ' + docker_memory_limits.gean) if (gean_is_aggregator | default('false')) != 'true' else '' }} --network host {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} -v {{ genesis_dir }}:/config:ro diff --git a/ansible/roles/grandine/tasks/main.yml b/ansible/roles/grandine/tasks/main.yml index b40fb71..f0ba526 100644 --- a/ansible/roles/grandine/tasks/main.yml +++ b/ansible/roles/grandine/tasks/main.yml @@ -92,8 +92,8 @@ --pull=always --name {{ node_name }} --restart {{ docker_container_restart_policy }} - --memory {{ docker_memory_limits.grandine }} - --memory-swap {{ docker_memory_limits.grandine }} + {{ ('--memory ' + docker_memory_limits.grandine) if (grandine_is_aggregator | default('false')) != 'true' else '' }} + {{ ('--memory-swap ' + docker_memory_limits.grandine) if (grandine_is_aggregator | default('false')) != 'true' else '' }} --network host {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} -v {{ genesis_dir }}:/config:ro diff --git a/ansible/roles/lantern/tasks/main.yml b/ansible/roles/lantern/tasks/main.yml index c7a49ca..90a03d5 100644 --- a/ansible/roles/lantern/tasks/main.yml +++ b/ansible/roles/lantern/tasks/main.yml @@ -77,8 +77,8 @@ --pull=always --name {{ node_name }} --restart {{ docker_container_restart_policy }} - --memory {{ docker_memory_limits.lantern }} - --memory-swap {{ docker_memory_limits.lantern }} + {{ ('--memory ' + docker_memory_limits.lantern) if (lantern_is_aggregator | default('false')) != 'true' else '' }} + {{ ('--memory-swap ' + docker_memory_limits.lantern) if (lantern_is_aggregator | default('false')) != 'true' else '' }} --network host {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} -v {{ genesis_dir }}:/config:ro diff --git a/ansible/roles/lighthouse/tasks/main.yml b/ansible/roles/lighthouse/tasks/main.yml index d8b3add..1de0b91 100644 --- a/ansible/roles/lighthouse/tasks/main.yml +++ b/ansible/roles/lighthouse/tasks/main.yml @@ -89,8 +89,8 @@ docker run -d --name {{ node_name }} --restart {{ docker_container_restart_policy }} - --memory {{ docker_memory_limits.lighthouse }} - --memory-swap {{ docker_memory_limits.lighthouse }} + {{ ('--memory ' + docker_memory_limits.lighthouse) if (lighthouse_is_aggregator | default('false')) != 'true' else '' }} + {{ ('--memory-swap ' + docker_memory_limits.lighthouse) if (lighthouse_is_aggregator | default('false')) != 'true' else '' }} --network host {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} -v {{ genesis_dir }}:/config:ro diff --git a/ansible/roles/nlean/tasks/main.yml b/ansible/roles/nlean/tasks/main.yml index ca2cfa5..9b4c70e 100644 --- a/ansible/roles/nlean/tasks/main.yml +++ b/ansible/roles/nlean/tasks/main.yml @@ -86,8 +86,8 @@ --pull=always --name {{ node_name }} --restart {{ docker_container_restart_policy }} - --memory {{ docker_memory_limits.nlean }} - --memory-swap {{ docker_memory_limits.nlean }} + {{ ('--memory ' + docker_memory_limits.nlean) if (nlean_is_aggregator | default('false')) != 'true' else '' }} + {{ ('--memory-swap ' + docker_memory_limits.nlean) if (nlean_is_aggregator | default('false')) != 'true' else '' }} --network host {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} -v {{ genesis_dir }}:/config:ro diff --git a/ansible/roles/peam/tasks/main.yml b/ansible/roles/peam/tasks/main.yml index 43bbf83..584983e 100644 --- a/ansible/roles/peam/tasks/main.yml +++ b/ansible/roles/peam/tasks/main.yml @@ -137,8 +137,8 @@ --pull=always --name {{ node_name }} --restart {{ docker_container_restart_policy }} - --memory {{ docker_memory_limits.peam }} - --memory-swap {{ docker_memory_limits.peam }} + {{ ('--memory ' + docker_memory_limits.peam) if (peam_is_aggregator | default('false')) != 'true' else '' }} + {{ ('--memory-swap ' + docker_memory_limits.peam) if (peam_is_aggregator | default('false')) != 'true' else '' }} --network host {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} -v {{ genesis_dir }}:/config:ro diff --git a/ansible/roles/qlean/tasks/main.yml b/ansible/roles/qlean/tasks/main.yml index 3c4fa02..d02c639 100644 --- a/ansible/roles/qlean/tasks/main.yml +++ b/ansible/roles/qlean/tasks/main.yml @@ -84,8 +84,8 @@ --pull=always --name {{ node_name }} --restart {{ docker_container_restart_policy }} - --memory {{ docker_memory_limits.qlean }} - --memory-swap {{ docker_memory_limits.qlean }} + {{ ('--memory ' + docker_memory_limits.qlean) if (qlean_is_aggregator | default('false')) != 'true' else '' }} + {{ ('--memory-swap ' + docker_memory_limits.qlean) if (qlean_is_aggregator | default('false')) != 'true' else '' }} --network host --platform {{ qlean_docker_platform }} {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} diff --git a/ansible/roles/ream/tasks/main.yml b/ansible/roles/ream/tasks/main.yml index 6bf729a..6eb61ea 100644 --- a/ansible/roles/ream/tasks/main.yml +++ b/ansible/roles/ream/tasks/main.yml @@ -85,8 +85,8 @@ --pull=always --name {{ node_name }} --restart {{ docker_container_restart_policy }} - --memory {{ docker_memory_limits.ream }} - --memory-swap {{ docker_memory_limits.ream }} + {{ ('--memory ' + docker_memory_limits.ream) if (ream_is_aggregator | default('false')) != 'true' else '' }} + {{ ('--memory-swap ' + docker_memory_limits.ream) if (ream_is_aggregator | default('false')) != 'true' else '' }} --network host {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} -v {{ genesis_dir }}:/config:ro diff --git a/ansible/roles/zeam/tasks/main.yml b/ansible/roles/zeam/tasks/main.yml index f813f2f..cf5af40 100644 --- a/ansible/roles/zeam/tasks/main.yml +++ b/ansible/roles/zeam/tasks/main.yml @@ -122,8 +122,8 @@ --pull=always --name {{ node_name }} --restart {{ docker_container_restart_policy }} - --memory {{ docker_memory_limits.zeam }} - --memory-swap {{ docker_memory_limits.zeam }} + {{ ('--memory ' + docker_memory_limits.zeam) if (zeam_is_aggregator | default('false')) != 'true' else '' }} + {{ ('--memory-swap ' + docker_memory_limits.zeam) if (zeam_is_aggregator | default('false')) != 'true' else '' }} --network host --security-opt seccomp=unconfined {{ '--init --ulimit core=-1 --workdir /data' if (enable_core_dumps | default('') == 'all') or (node_name in (enable_core_dumps | default('')).split(',')) or (node_name.split('_')[0] in (enable_core_dumps | default('')).split(',')) else '' }} From f60dec75d7374debe6194d51f502b69576d716bb Mon Sep 17 00:00:00 2001 From: ch4r10t33r Date: Wed, 20 May 2026 12:24:12 +0100 Subject: [PATCH 12/20] ansible: default zeam docker image to blockblaz/zeam:devnet4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the ansible fallback when client-cmd extraction fails: 0xpartha/zeam:local → blockblaz/zeam:devnet4 (matches defaults and zeam-cmd.sh). --- ansible/roles/zeam/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/zeam/tasks/main.yml b/ansible/roles/zeam/tasks/main.yml index cf5af40..8b80327 100644 --- a/ansible/roles/zeam/tasks/main.yml +++ b/ansible/roles/zeam/tasks/main.yml @@ -31,7 +31,7 @@ - name: Set docker image and deployment mode from client-cmd.sh set_fact: - zeam_docker_image: "{{ zeam_docker_image_raw.stdout | trim | default('0xpartha/zeam:local') }}" + zeam_docker_image: "{{ zeam_docker_image_raw.stdout | trim | default('blockblaz/zeam:devnet4') }}" deployment_mode: "{{ zeam_deployment_mode_raw.stdout | trim | default('docker') }}" delegate_to: localhost run_once: true From daee9d4be5050da12dff0ce87f667d085f8568cb Mon Sep 17 00:00:00 2001 From: ch4r10t33r Date: Wed, 20 May 2026 12:39:24 +0100 Subject: [PATCH 13/20] ansible-devnet: gean on subnet 5, grandine on subnet 7 Replace the former nlean column with grandine aggregators and add a second ream column on subnet 6 so gean and grandine each own one subnet. --- ansible-devnet/genesis/validator-config.yaml | 171 ++++++++++--------- 1 file changed, 88 insertions(+), 83 deletions(-) diff --git a/ansible-devnet/genesis/validator-config.yaml b/ansible-devnet/genesis/validator-config.yaml index bb0a838..5a35b3b 100644 --- a/ansible-devnet/genesis/validator-config.yaml +++ b/ansible-devnet/genesis/validator-config.yaml @@ -2,12 +2,17 @@ # # Genesis order = YAML list order → validator_index = row index (0..63). # attestation_committee_count: 8 → subnet = index % 8. -# Row pattern repeats zeam → ethlambda → ream → zeam → ethlambda → gean → grandine → nlean +# Row pattern repeats zeam → ethlambda → ream → zeam → ethlambda → gean → ream → grandine # (former qlean/lantern slots are zeam_8..15 and ethlambda_8..15). -# Aggregators: one per committee subnet (validator_index % 8); each subscribes only to its subnet. +# Aggregators: one per committee subnet (validator_index % 8); each on Aggregator_servers IP. +# Subnet → node (only these have isAggregator: true): +# 0 zeam_8 @ 178.105.22.187 1 ethlambda_8 @ 178.105.124.135 2 ream_0 @ 178.105.117.56 +# 3 zeam_4 @ 178.105.121.214 4 ethlambda_5 @ 178.105.22.193 5 gean_0 @ 178.105.70.184 +# 6 ream_13 @ 178.105.113.166 7 grandine_0 @ 178.105.102.228 # # Regular validators use IPs from lean_ethereum_servers.txt (Validator_servers). -# Aggregators use dedicated Aggregator_servers (one IP per client type). +# Aggregators use dedicated Aggregator_servers: exactly one container per IP +# (quic 9001, api 5055, metrics 9102). No regular validators on those hosts. # Tooling host (46.225.10.32) is not used. # # Ports (per IP): up to four processes per host — quic 9001..9004, api 5055..5058, @@ -50,19 +55,19 @@ validators: - name: "zeam_0" privkey: "bdf953adc161873ba026330c56450453f582e3c4ee6cb713644794bcfdd85fe5" enrFields: - ip: "178.105.121.214" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 + ip: "65.109.131.177" + quic: 9002 + metricsPort: 9103 + apiPort: 5056 isAggregator: false count: 1 - name: "ethlambda_0" privkey: "299550529a79bc2dce003747c52fb0639465c893e00b0440ac66144d625e066a" enrFields: - ip: "178.105.22.193" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 + ip: "178.104.149.208" + quic: 9003 + metricsPort: 9104 + apiPort: 5057 isAggregator: false count: 1 - name: "gean_0" @@ -74,17 +79,17 @@ validators: apiPort: 5055 isAggregator: true count: 1 - - name: "grandine_0" - privkey: "c05937b251889e35c58d4601c29bed8153dc22c548448f85e0ab9ca436d4b904" + - name: "ream_8" + privkey: "2e9be3f1b0d32ca3a4d62017fbfafe3950b7e90fed6802ff8bd2e0f8c4e2ca91" enrFields: - ip: "178.105.113.166" + ip: "157.180.20.55" quic: 9001 metricsPort: 9102 apiPort: 5055 - isAggregator: true + isAggregator: false count: 1 - - name: "nlean_0" - privkey: "2e9be3f1b0d32ca3a4d62017fbfafe3950b7e90fed6802ff8bd2e0f8c4e2ca91" + - name: "grandine_0" + privkey: "c05937b251889e35c58d4601c29bed8153dc22c548448f85e0ab9ca436d4b904" enrFields: ip: "178.105.102.228" quic: 9001 @@ -146,19 +151,19 @@ validators: apiPort: 5055 isAggregator: false count: 1 - - name: "grandine_1" - privkey: "ab6edee1173379f647b4022d74d4b3342d547e7bb6954664a1a489b95b7c9b60" + - name: "ream_9" + privkey: "74388a2487b943d947c17bf65ca08470f3ad5f045ea0f5aed38a98bbaba1bd49" enrFields: - ip: "37.27.89.135" + ip: "65.109.131.177" quic: 9001 metricsPort: 9102 apiPort: 5055 isAggregator: false count: 1 - - name: "nlean_1" - privkey: "74388a2487b943d947c17bf65ca08470f3ad5f045ea0f5aed38a98bbaba1bd49" + - name: "grandine_1" + privkey: "ab6edee1173379f647b4022d74d4b3342d547e7bb6954664a1a489b95b7c9b60" enrFields: - ip: "157.180.20.55" + ip: "37.27.89.135" quic: 9001 metricsPort: 9102 apiPort: 5055 @@ -218,19 +223,19 @@ validators: apiPort: 5055 isAggregator: false count: 1 - - name: "grandine_2" - privkey: "b4fdf4bc8ea6e74845b19823510c590406ff63371e0573132f1ddb6b455b7bf7" + - name: "ream_10" + privkey: "53f1956c292eca6aba53bfd1089682828130db1c0ef980746e5e3e1d94f98360" enrFields: - ip: "65.21.182.45" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 + ip: "37.27.220.14" + quic: 9002 + metricsPort: 9103 + apiPort: 5056 isAggregator: false count: 1 - - name: "nlean_2" - privkey: "53f1956c292eca6aba53bfd1089682828130db1c0ef980746e5e3e1d94f98360" + - name: "grandine_2" + privkey: "b4fdf4bc8ea6e74845b19823510c590406ff63371e0573132f1ddb6b455b7bf7" enrFields: - ip: "65.109.131.177" + ip: "65.21.182.45" quic: 9001 metricsPort: 9102 apiPort: 5055 @@ -270,7 +275,7 @@ validators: quic: 9001 metricsPort: 9102 apiPort: 5055 - isAggregator: true + isAggregator: false count: 1 - name: "ethlambda_3" privkey: "1c454b0399cd2178b46e42c8b599e34f2e7ddab71df96b1fbf510d5951335ea0" @@ -290,6 +295,15 @@ validators: apiPort: 5055 isAggregator: false count: 1 + - name: "ream_11" + privkey: "f52f1216a9a20f5b2c200ed64f1a2a8e7427772574147a7edf4602b40f820e9b" + enrFields: + ip: "178.104.151.50" + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 - name: "grandine_3" privkey: "27b642b91b25a4dd55638d7936469270f0fc4b77ceba768617291415eeb3ab1a" enrFields: @@ -299,15 +313,6 @@ validators: apiPort: 5055 isAggregator: false count: 1 - - name: "nlean_3" - privkey: "f52f1216a9a20f5b2c200ed64f1a2a8e7427772574147a7edf4602b40f820e9b" - enrFields: - ip: "37.27.220.14" - quic: 9002 - metricsPort: 9103 - apiPort: 5056 - isAggregator: false - count: 1 - name: "zeam_12" privkey: "0763e1a9f1909df7eae31ef4e93d5aaf45a506624ef81df893fc02e08a4ab94f" enrFields: @@ -338,16 +343,16 @@ validators: - name: "zeam_4" privkey: "2aed04fc149298af1376aee0161d5dc67f9d7a47e625a58ae8e44f210d45c2a2" enrFields: - ip: "135.181.82.109" - quic: 9002 - metricsPort: 9103 - apiPort: 5056 - isAggregator: false + ip: "178.105.121.214" + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: true count: 1 - name: "ethlambda_4" privkey: "91c2c932a6589e130bfa5281cc27f6b9930ba7f99aa043d334f14578ef7492f1" enrFields: - ip: "157.90.254.146" + ip: "95.217.158.60" quic: 9002 metricsPort: 9103 apiPort: 5056 @@ -362,19 +367,19 @@ validators: apiPort: 5056 isAggregator: false count: 1 - - name: "grandine_4" - privkey: "936fa39e4f1ce548d977f852f84ee900bbd2789bf85779e0e958d966a1ba7386" + - name: "ream_12" + privkey: "6cc68faf32d824996ecec2644c61cdb479da9bc1f957f7d09913fac8c49fe578" enrFields: - ip: "157.180.20.55" + ip: "65.109.138.213" quic: 9002 metricsPort: 9103 apiPort: 5056 isAggregator: false count: 1 - - name: "nlean_4" - privkey: "6cc68faf32d824996ecec2644c61cdb479da9bc1f957f7d09913fac8c49fe578" + - name: "grandine_4" + privkey: "936fa39e4f1ce548d977f852f84ee900bbd2789bf85779e0e958d966a1ba7386" enrFields: - ip: "178.104.151.50" + ip: "157.180.20.55" quic: 9002 metricsPort: 9103 apiPort: 5056 @@ -419,11 +424,11 @@ validators: - name: "ethlambda_5" privkey: "793a5d9add590b890085b440ceefe3e8c805785f7271feee09d35511e7060056" enrFields: - ip: "204.168.135.7" - quic: 9002 - metricsPort: 9103 - apiPort: 5056 - isAggregator: false + ip: "178.105.22.193" + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: true count: 1 - name: "gean_5" privkey: "b55c8e411d54ca0ef5ce69e86cba09ae110add1c5d598ed3356adfa5182b82e9" @@ -434,6 +439,15 @@ validators: apiPort: 5056 isAggregator: false count: 1 + - name: "ream_13" + privkey: "9cf22e42d463e751958eceace0a3201c661b4326076027c900a04aa845a9e683" + enrFields: + ip: "178.105.113.166" + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: true + count: 1 - name: "grandine_5" privkey: "99f1df1cbc7429360d52142a8ec5df6a3ad38a4a39897d2ee356b88fd3a69385" enrFields: @@ -443,15 +457,6 @@ validators: apiPort: 5056 isAggregator: false count: 1 - - name: "nlean_5" - privkey: "9cf22e42d463e751958eceace0a3201c661b4326076027c900a04aa845a9e683" - enrFields: - ip: "65.109.138.213" - quic: 9002 - metricsPort: 9103 - apiPort: 5056 - isAggregator: false - count: 1 - name: "zeam_14" privkey: "aaf15d4a12fec7409b136d022790d9aaaca3d5679c9e28babe23a424cd80729b" enrFields: @@ -506,19 +511,19 @@ validators: apiPort: 5056 isAggregator: false count: 1 - - name: "grandine_6" - privkey: "eadfc14b6231cb9a207f96f436f7d55c733ba3781306307b7f81f564e2b53f0a" + - name: "ream_14" + privkey: "c9dcdad85cfffc846c9771f950f4198ae4791972cf86841ee01004f8375b9332" enrFields: - ip: "37.27.220.14" + ip: "157.180.116.162" quic: 9003 metricsPort: 9104 apiPort: 5057 isAggregator: false count: 1 - - name: "nlean_6" - privkey: "c9dcdad85cfffc846c9771f950f4198ae4791972cf86841ee01004f8375b9332" + - name: "grandine_6" + privkey: "eadfc14b6231cb9a207f96f436f7d55c733ba3781306307b7f81f564e2b53f0a" enrFields: - ip: "157.180.116.162" + ip: "37.27.220.14" quic: 9003 metricsPort: 9104 apiPort: 5057 @@ -527,7 +532,7 @@ validators: - name: "zeam_15" privkey: "2907bc1d1e0a3dee4cc317dfda39a09b5e23596671d7108482edb897616c6b6c" enrFields: - ip: "204.168.178.179" + ip: "204.168.135.7" quic: 9003 metricsPort: 9104 apiPort: 5057 @@ -554,7 +559,7 @@ validators: - name: "zeam_7" privkey: "97ae5f1d63fa7c05783f7ae5af4ce1fc43a030833c174e430080a8695ce91d56" enrFields: - ip: "157.90.254.146" + ip: "95.217.158.60" quic: 9003 metricsPort: 9104 apiPort: 5057 @@ -567,7 +572,7 @@ validators: quic: 9003 metricsPort: 9104 apiPort: 5057 - isAggregator: true + isAggregator: false count: 1 - name: "gean_7" privkey: "32e4b103c4b349df214d265db9cb1e6d49346e14093fee656828a5e1ef050b8b" @@ -578,19 +583,19 @@ validators: apiPort: 5057 isAggregator: false count: 1 - - name: "grandine_7" - privkey: "7dd9ca62d79af8e01c95c743ef1a39155a42eded7096599a7f67b3932b387e6c" + - name: "ream_15" + privkey: "c99cd68f66a7460a8aca095d6dd4b6ce953e175d91077f1e7aba27858f68e301" enrFields: - ip: "178.104.151.50" + ip: "178.104.133.162" quic: 9003 metricsPort: 9104 apiPort: 5057 isAggregator: false count: 1 - - name: "nlean_7" - privkey: "c99cd68f66a7460a8aca095d6dd4b6ce953e175d91077f1e7aba27858f68e301" + - name: "grandine_7" + privkey: "7dd9ca62d79af8e01c95c743ef1a39155a42eded7096599a7f67b3932b387e6c" enrFields: - ip: "178.104.133.162" + ip: "178.104.151.50" quic: 9003 metricsPort: 9104 apiPort: 5057 From 1fe52385604480a8f8d19ee0e382f802055404e9 Mon Sep 17 00:00:00 2001 From: ch4r10t33r Date: Wed, 20 May 2026 13:13:54 +0100 Subject: [PATCH 14/20] ansible-devnet: align aggregator IPs and fix validator hosts Put grandine_0 on the grandine aggregator host, move ream_13 to the nlean slot, place gean_1 on 95.217.158.60, and relocate validators off hosts not in lean_ethereum_servers.txt. --- ansible-devnet/genesis/validator-config.yaml | 28 ++++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/ansible-devnet/genesis/validator-config.yaml b/ansible-devnet/genesis/validator-config.yaml index 5a35b3b..80e4ec0 100644 --- a/ansible-devnet/genesis/validator-config.yaml +++ b/ansible-devnet/genesis/validator-config.yaml @@ -8,7 +8,7 @@ # Subnet → node (only these have isAggregator: true): # 0 zeam_8 @ 178.105.22.187 1 ethlambda_8 @ 178.105.124.135 2 ream_0 @ 178.105.117.56 # 3 zeam_4 @ 178.105.121.214 4 ethlambda_5 @ 178.105.22.193 5 gean_0 @ 178.105.70.184 -# 6 ream_13 @ 178.105.113.166 7 grandine_0 @ 178.105.102.228 +# 6 ream_13 @ 178.105.102.228 7 grandine_0 @ 178.105.113.166 # # Regular validators use IPs from lean_ethereum_servers.txt (Validator_servers). # Aggregators use dedicated Aggregator_servers: exactly one container per IP @@ -91,7 +91,7 @@ validators: - name: "grandine_0" privkey: "c05937b251889e35c58d4601c29bed8153dc22c548448f85e0ab9ca436d4b904" enrFields: - ip: "178.105.102.228" + ip: "178.105.113.166" quic: 9001 metricsPort: 9102 apiPort: 5055 @@ -118,10 +118,10 @@ validators: - name: "ream_1" privkey: "fc2f11f90dd90df33bfb5f3467c2cbc37cf93dbb41c7ac556f9eea117418e73d" enrFields: - ip: "204.168.178.179" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 + ip: "37.27.220.14" + quic: 9004 + metricsPort: 9105 + apiPort: 5058 isAggregator: false count: 1 - name: "zeam_1" @@ -145,7 +145,7 @@ validators: - name: "gean_1" privkey: "5408a68b960a7932e367b32489498a13c339f87ff8090ea54213524b3d76fcff" enrFields: - ip: "157.90.254.146" + ip: "95.217.158.60" quic: 9001 metricsPort: 9102 apiPort: 5055 @@ -325,10 +325,10 @@ validators: - name: "ethlambda_12" privkey: "a5dac14f317084d255c57e5080985b1dbd79cf8fc625a9c7fba781abf07001bd" enrFields: - ip: "204.168.178.179" - quic: 9002 - metricsPort: 9103 - apiPort: 5056 + ip: "157.180.116.162" + quic: 9004 + metricsPort: 9105 + apiPort: 5058 isAggregator: false count: 1 - name: "ream_4" @@ -352,7 +352,7 @@ validators: - name: "ethlambda_4" privkey: "91c2c932a6589e130bfa5281cc27f6b9930ba7f99aa043d334f14578ef7492f1" enrFields: - ip: "95.217.158.60" + ip: "157.90.254.146" quic: 9002 metricsPort: 9103 apiPort: 5056 @@ -442,7 +442,7 @@ validators: - name: "ream_13" privkey: "9cf22e42d463e751958eceace0a3201c661b4326076027c900a04aa845a9e683" enrFields: - ip: "178.105.113.166" + ip: "178.105.102.228" quic: 9001 metricsPort: 9102 apiPort: 5055 @@ -559,7 +559,7 @@ validators: - name: "zeam_7" privkey: "97ae5f1d63fa7c05783f7ae5af4ce1fc43a030833c174e430080a8695ce91d56" enrFields: - ip: "95.217.158.60" + ip: "157.90.254.146" quic: 9003 metricsPort: 9104 apiPort: 5057 From e24fd52f913b5d794a82fb963edcdc790608efdc Mon Sep 17 00:00:00 2001 From: ch4r10t33r Date: Wed, 20 May 2026 18:59:26 +0100 Subject: [PATCH 15/20] ansible-devnet: point aggregators at new Aggregator_servers IPs Replace the eight aggregator host IPs with the Aggregator_servers list from lean_ethereum_servers.txt and keep assign-aggregator-ips.py in sync. --- ansible-devnet/genesis/validator-config.yaml | 1174 +++++++++--------- scripts/assign-aggregator-ips.py | 222 ++++ 2 files changed, 799 insertions(+), 597 deletions(-) create mode 100755 scripts/assign-aggregator-ips.py diff --git a/ansible-devnet/genesis/validator-config.yaml b/ansible-devnet/genesis/validator-config.yaml index 80e4ec0..862cdb3 100644 --- a/ansible-devnet/genesis/validator-config.yaml +++ b/ansible-devnet/genesis/validator-config.yaml @@ -1,603 +1,583 @@ -# 8-subnet ansible devnet — homogeneous clients (one client family per subnet). -# -# Genesis order = YAML list order → validator_index = row index (0..63). -# attestation_committee_count: 8 → subnet = index % 8. -# Row pattern repeats zeam → ethlambda → ream → zeam → ethlambda → gean → ream → grandine -# (former qlean/lantern slots are zeam_8..15 and ethlambda_8..15). -# Aggregators: one per committee subnet (validator_index % 8); each on Aggregator_servers IP. -# Subnet → node (only these have isAggregator: true): -# 0 zeam_8 @ 178.105.22.187 1 ethlambda_8 @ 178.105.124.135 2 ream_0 @ 178.105.117.56 -# 3 zeam_4 @ 178.105.121.214 4 ethlambda_5 @ 178.105.22.193 5 gean_0 @ 178.105.70.184 -# 6 ream_13 @ 178.105.102.228 7 grandine_0 @ 178.105.113.166 -# -# Regular validators use IPs from lean_ethereum_servers.txt (Validator_servers). -# Aggregators use dedicated Aggregator_servers: exactly one container per IP -# (quic 9001, api 5055, metrics 9102). No regular validators on those hosts. -# Tooling host (46.225.10.32) is not used. -# -# Ports (per IP): up to four processes per host — quic 9001..9004, api 5055..5058, -# metrics 9102..9105. Avoid observability TCP ports on ansible hosts: 9090 prometheus, -# 9080 promtail, 9098 cadvisor, 9100 node_exporter. shuffle: roundrobin deployment_mode: ansible config: activeEpoch: 18 - keyType: "hash-sig" + keyType: hash-sig attestation_committee_count: 8 validators: - - name: "zeam_8" - privkey: "8e9f81c9caa9e29d26a7327311ca63a38254efdfccf3ce1362bae47eae0b18b3" - enrFields: - ip: "178.105.22.187" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: true - count: 1 - - name: "ethlambda_8" - privkey: "d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5" - enrFields: - ip: "178.105.124.135" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: true - count: 1 - - name: "ream_0" - privkey: "af27950128b49cda7e7bc9fcb7b0270f7a3945aa7543326f3bfdbd57d2a97a32" - enrFields: - ip: "178.105.117.56" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: true - count: 1 - - name: "zeam_0" - privkey: "bdf953adc161873ba026330c56450453f582e3c4ee6cb713644794bcfdd85fe5" - enrFields: - ip: "65.109.131.177" - quic: 9002 - metricsPort: 9103 - apiPort: 5056 - isAggregator: false - count: 1 - - name: "ethlambda_0" - privkey: "299550529a79bc2dce003747c52fb0639465c893e00b0440ac66144d625e066a" - enrFields: - ip: "178.104.149.208" - quic: 9003 - metricsPort: 9104 - apiPort: 5057 - isAggregator: false - count: 1 - - name: "gean_0" - privkey: "69c251cdb06039dd99d87e5a1439fa3720615be98c293ec9bcfd041877a2e8ca" - enrFields: - ip: "178.105.70.184" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: true - count: 1 - - name: "ream_8" - privkey: "2e9be3f1b0d32ca3a4d62017fbfafe3950b7e90fed6802ff8bd2e0f8c4e2ca91" - enrFields: - ip: "157.180.20.55" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: false - count: 1 - - name: "grandine_0" - privkey: "c05937b251889e35c58d4601c29bed8153dc22c548448f85e0ab9ca436d4b904" - enrFields: - ip: "178.105.113.166" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: true - count: 1 - - name: "zeam_9" - privkey: "0e190e06a62db01bf566af4348277463d26eebab8f6badbcb989242ea4fee050" - enrFields: - ip: "37.27.220.14" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: false - count: 1 - - name: "ethlambda_9" - privkey: "1176609530e568ec12ad227e60ae71c618bfb13fde5a3257e4b0627216e78e04" - enrFields: - ip: "157.180.116.162" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: false - count: 1 - - name: "ream_1" - privkey: "fc2f11f90dd90df33bfb5f3467c2cbc37cf93dbb41c7ac556f9eea117418e73d" - enrFields: - ip: "37.27.220.14" - quic: 9004 - metricsPort: 9105 - apiPort: 5058 - isAggregator: false - count: 1 - - name: "zeam_1" - privkey: "8d14c7b02d55ca050ef97a3961aa16828837fb363e4e19e4dd0060f58670a2b3" - enrFields: - ip: "204.168.190.188" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: false - count: 1 - - name: "ethlambda_1" - privkey: "eb8533a0d5071d4dbcb0c4fcf9b8ac6edc3d1a260d2bb348fafc5cdb455aa1d4" - enrFields: - ip: "135.181.82.109" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: false - count: 1 - - name: "gean_1" - privkey: "5408a68b960a7932e367b32489498a13c339f87ff8090ea54213524b3d76fcff" - enrFields: - ip: "95.217.158.60" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: false - count: 1 - - name: "ream_9" - privkey: "74388a2487b943d947c17bf65ca08470f3ad5f045ea0f5aed38a98bbaba1bd49" - enrFields: - ip: "65.109.131.177" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: false - count: 1 - - name: "grandine_1" - privkey: "ab6edee1173379f647b4022d74d4b3342d547e7bb6954664a1a489b95b7c9b60" - enrFields: - ip: "37.27.89.135" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: false - count: 1 - - name: "zeam_10" - privkey: "7fdceb8166b9d7ab4169d80345ab85fd042b3684ba307c4c02e851c1520975b5" - enrFields: - ip: "178.104.151.50" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: false - count: 1 - - name: "ethlambda_10" - privkey: "943151f2327ea91357632dd8a3a498242b07eeeebad973ea82d6bead80627d3e" - enrFields: - ip: "178.104.133.162" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: false - count: 1 - - name: "ream_2" - privkey: "d723f0cc94645848bbacf8aea4e01944e2f4a6f9f2c69b17ed1868f492374a7a" - enrFields: - ip: "178.104.149.208" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: false - count: 1 - - name: "zeam_2" - privkey: "11ff3a52375898532680c86739dbe0b632088545d0b901181d92053b5fab8d38" - enrFields: - ip: "178.104.149.91" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: false - count: 1 - - name: "ethlambda_2" - privkey: "4a7c5b2077d0e3c6f5322d0f2fd98cb2efffbceb41de2799e520b48872dc4102" - enrFields: - ip: "95.216.154.185" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: false - count: 1 - - name: "gean_2" - privkey: "14da65451d15aee0cbdbcd847cfc3474f106c4595bf716306832fc078858f458" - enrFields: - ip: "204.168.135.7" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: false - count: 1 - - name: "ream_10" - privkey: "53f1956c292eca6aba53bfd1089682828130db1c0ef980746e5e3e1d94f98360" - enrFields: - ip: "37.27.220.14" - quic: 9002 - metricsPort: 9103 - apiPort: 5056 - isAggregator: false - count: 1 - - name: "grandine_2" - privkey: "b4fdf4bc8ea6e74845b19823510c590406ff63371e0573132f1ddb6b455b7bf7" - enrFields: - ip: "65.21.182.45" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: false - count: 1 - - name: "zeam_11" - privkey: "1027ce52661ef76b5ada68bc8cd45c13ebefb09c99dcad251b9bd4893c71815f" - enrFields: - ip: "65.109.138.213" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: false - count: 1 - - name: "ethlambda_11" - privkey: "31258c1b84e4a64954fb18dbb04ec9de7079ad4d0273b40380aa1f4ca2f20804" - enrFields: - ip: "204.168.134.201" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: false - count: 1 - - name: "ream_3" - privkey: "4701dd38aa11a1affd69b5d95e535210f8c29e720222b906c46bd64cf018f9c7" - enrFields: - ip: "95.217.19.42" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: false - count: 1 - - name: "zeam_3" - privkey: "877c1489e75914bd46dccb71e0ee2d32af337fbb82f2c9caea73818e00ea9a61" - enrFields: - ip: "95.216.173.151" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: false - count: 1 - - name: "ethlambda_3" - privkey: "1c454b0399cd2178b46e42c8b599e34f2e7ddab71df96b1fbf510d5951335ea0" - enrFields: - ip: "95.216.164.165" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: false - count: 1 - - name: "gean_3" - privkey: "51832ccf0a189ed39a6fe13833894445b1fa60fc008fcb7fd7e30405318068d1" - enrFields: - ip: "95.216.165.186" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: false - count: 1 - - name: "ream_11" - privkey: "f52f1216a9a20f5b2c200ed64f1a2a8e7427772574147a7edf4602b40f820e9b" - enrFields: - ip: "178.104.151.50" - quic: 9002 - metricsPort: 9103 - apiPort: 5056 - isAggregator: false - count: 1 - - name: "grandine_3" - privkey: "27b642b91b25a4dd55638d7936469270f0fc4b77ceba768617291415eeb3ab1a" - enrFields: - ip: "37.27.250.20" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: false - count: 1 - - name: "zeam_12" - privkey: "0763e1a9f1909df7eae31ef4e93d5aaf45a506624ef81df893fc02e08a4ab94f" - enrFields: - ip: "157.180.116.162" - quic: 9002 - metricsPort: 9103 - apiPort: 5056 - isAggregator: false - count: 1 - - name: "ethlambda_12" - privkey: "a5dac14f317084d255c57e5080985b1dbd79cf8fc625a9c7fba781abf07001bd" - enrFields: - ip: "157.180.116.162" - quic: 9004 - metricsPort: 9105 - apiPort: 5058 - isAggregator: false - count: 1 - - name: "ream_4" - privkey: "62e6c79311ce4ee1b95ff52c8d6b7cccea0bf0b927fbed595d454858b32c49d6" - enrFields: - ip: "204.168.190.188" - quic: 9002 - metricsPort: 9103 - apiPort: 5056 - isAggregator: false - count: 1 - - name: "zeam_4" - privkey: "2aed04fc149298af1376aee0161d5dc67f9d7a47e625a58ae8e44f210d45c2a2" - enrFields: - ip: "178.105.121.214" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: true - count: 1 - - name: "ethlambda_4" - privkey: "91c2c932a6589e130bfa5281cc27f6b9930ba7f99aa043d334f14578ef7492f1" - enrFields: - ip: "157.90.254.146" - quic: 9002 - metricsPort: 9103 - apiPort: 5056 - isAggregator: false - count: 1 - - name: "gean_4" - privkey: "9b678649edf603b73913e8957fd7c26fe6af08656b8fe14b0d47aa80d3b59c9f" - enrFields: - ip: "37.27.89.135" - quic: 9002 - metricsPort: 9103 - apiPort: 5056 - isAggregator: false - count: 1 - - name: "ream_12" - privkey: "6cc68faf32d824996ecec2644c61cdb479da9bc1f957f7d09913fac8c49fe578" - enrFields: - ip: "65.109.138.213" - quic: 9002 - metricsPort: 9103 - apiPort: 5056 - isAggregator: false - count: 1 - - name: "grandine_4" - privkey: "936fa39e4f1ce548d977f852f84ee900bbd2789bf85779e0e958d966a1ba7386" - enrFields: - ip: "157.180.20.55" - quic: 9002 - metricsPort: 9103 - apiPort: 5056 - isAggregator: false - count: 1 - - name: "zeam_13" - privkey: "ce66f19a179bd74e01b775c696b1744060078fe4d84abb5761a3f1d7cef15562" - enrFields: - ip: "178.104.133.162" - quic: 9002 - metricsPort: 9103 - apiPort: 5056 - isAggregator: false - count: 1 - - name: "ethlambda_13" - privkey: "12bb3c043f2600f27be6bf8916368e9ff01b9ef5c4a7ccdf6aab8e2a24ba55c3" - enrFields: - ip: "178.104.149.208" - quic: 9002 - metricsPort: 9103 - apiPort: 5056 - isAggregator: false - count: 1 - - name: "ream_5" - privkey: "4caa321fc7598a3da98623a03774ee177c0025fa783fc2fc9c082b297ed96433" - enrFields: - ip: "178.104.149.91" - quic: 9002 - metricsPort: 9103 - apiPort: 5056 - isAggregator: false - count: 1 - - name: "zeam_5" - privkey: "34cfea8f2e6d6f8a28ae8869220450b8069ba0c9cca436a1079d44fa40989820" - enrFields: - ip: "95.216.154.185" - quic: 9002 - metricsPort: 9103 - apiPort: 5056 - isAggregator: false - count: 1 - - name: "ethlambda_5" - privkey: "793a5d9add590b890085b440ceefe3e8c805785f7271feee09d35511e7060056" - enrFields: - ip: "178.105.22.193" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: true - count: 1 - - name: "gean_5" - privkey: "b55c8e411d54ca0ef5ce69e86cba09ae110add1c5d598ed3356adfa5182b82e9" - enrFields: - ip: "65.21.182.45" - quic: 9002 - metricsPort: 9103 - apiPort: 5056 - isAggregator: false - count: 1 - - name: "ream_13" - privkey: "9cf22e42d463e751958eceace0a3201c661b4326076027c900a04aa845a9e683" - enrFields: - ip: "178.105.102.228" - quic: 9001 - metricsPort: 9102 - apiPort: 5055 - isAggregator: true - count: 1 - - name: "grandine_5" - privkey: "99f1df1cbc7429360d52142a8ec5df6a3ad38a4a39897d2ee356b88fd3a69385" - enrFields: - ip: "65.109.131.177" - quic: 9002 - metricsPort: 9103 - apiPort: 5056 - isAggregator: false - count: 1 - - name: "zeam_14" - privkey: "aaf15d4a12fec7409b136d022790d9aaaca3d5679c9e28babe23a424cd80729b" - enrFields: - ip: "204.168.134.201" - quic: 9002 - metricsPort: 9103 - apiPort: 5056 - isAggregator: false - count: 1 - - name: "ethlambda_14" - privkey: "7536676b6ee5f0062030c0b6784c44a175d66cdc39277354c1b9014b419cbc30" - enrFields: - ip: "95.217.19.42" - quic: 9002 - metricsPort: 9103 - apiPort: 5056 - isAggregator: false - count: 1 - - name: "ream_6" - privkey: "5401bb0acca1620ffddd776c98b5de53034b960eb84a1fe0ce381161789fd980" - enrFields: - ip: "95.216.173.151" - quic: 9002 - metricsPort: 9103 - apiPort: 5056 - isAggregator: false - count: 1 - - name: "zeam_6" - privkey: "2ac17fb2c419558e349400f45fd5077eac9ca7bcd527b0833ac7f630465c5457" - enrFields: - ip: "95.216.164.165" - quic: 9002 - metricsPort: 9103 - apiPort: 5056 - isAggregator: false - count: 1 - - name: "ethlambda_6" - privkey: "e673e0a9b6ea018ae0b8039019bcee829037dd5638c2b5a5f085385e46231b6a" - enrFields: - ip: "95.216.165.186" - quic: 9002 - metricsPort: 9103 - apiPort: 5056 - isAggregator: false - count: 1 - - name: "gean_6" - privkey: "0db45359918ec742bd1c354af1c5614969985139d2617790dc2ada928e95a731" - enrFields: - ip: "37.27.250.20" - quic: 9002 - metricsPort: 9103 - apiPort: 5056 - isAggregator: false - count: 1 - - name: "ream_14" - privkey: "c9dcdad85cfffc846c9771f950f4198ae4791972cf86841ee01004f8375b9332" - enrFields: - ip: "157.180.116.162" - quic: 9003 - metricsPort: 9104 - apiPort: 5057 - isAggregator: false - count: 1 - - name: "grandine_6" - privkey: "eadfc14b6231cb9a207f96f436f7d55c733ba3781306307b7f81f564e2b53f0a" - enrFields: - ip: "37.27.220.14" - quic: 9003 - metricsPort: 9104 - apiPort: 5057 - isAggregator: false - count: 1 - - name: "zeam_15" - privkey: "2907bc1d1e0a3dee4cc317dfda39a09b5e23596671d7108482edb897616c6b6c" - enrFields: - ip: "204.168.135.7" - quic: 9003 - metricsPort: 9104 - apiPort: 5057 - isAggregator: false - count: 1 - - name: "ethlambda_15" - privkey: "98da18b276aa5a54ceba15e78260286b9dbe03c4bf8b9db061d024ba4446cfff" - enrFields: - ip: "204.168.190.188" - quic: 9003 - metricsPort: 9104 - apiPort: 5057 - isAggregator: false - count: 1 - - name: "ream_7" - privkey: "a8a9cb1f243380d915e4997b3bc5058295d94704870885d5ed6cb1dfea24d77e" - enrFields: - ip: "135.181.82.109" - quic: 9003 - metricsPort: 9104 - apiPort: 5057 - isAggregator: false - count: 1 - - name: "zeam_7" - privkey: "97ae5f1d63fa7c05783f7ae5af4ce1fc43a030833c174e430080a8695ce91d56" - enrFields: - ip: "157.90.254.146" - quic: 9003 - metricsPort: 9104 - apiPort: 5057 - isAggregator: false - count: 1 - - name: "ethlambda_7" - privkey: "17bea5ada768f12d6f0c95ab9618277527e9691e2e4169e73bc92248da122e68" - enrFields: - ip: "37.27.89.135" - quic: 9003 - metricsPort: 9104 - apiPort: 5057 - isAggregator: false - count: 1 - - name: "gean_7" - privkey: "32e4b103c4b349df214d265db9cb1e6d49346e14093fee656828a5e1ef050b8b" - enrFields: - ip: "157.180.20.55" - quic: 9003 - metricsPort: 9104 - apiPort: 5057 - isAggregator: false - count: 1 - - name: "ream_15" - privkey: "c99cd68f66a7460a8aca095d6dd4b6ce953e175d91077f1e7aba27858f68e301" - enrFields: - ip: "178.104.133.162" - quic: 9003 - metricsPort: 9104 - apiPort: 5057 - isAggregator: false - count: 1 - - name: "grandine_7" - privkey: "7dd9ca62d79af8e01c95c743ef1a39155a42eded7096599a7f67b3932b387e6c" - enrFields: - ip: "178.104.151.50" - quic: 9003 - metricsPort: 9104 - apiPort: 5057 - isAggregator: false - count: 1 +- name: zeam_8 + privkey: 8e9f81c9caa9e29d26a7327311ca63a38254efdfccf3ce1362bae47eae0b18b3 + enrFields: + ip: 77.42.121.211 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: true + count: 1 +- name: ethlambda_8 + privkey: d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5 + enrFields: + ip: 89.167.41.98 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: true + count: 1 +- name: ream_0 + privkey: af27950128b49cda7e7bc9fcb7b0270f7a3945aa7543326f3bfdbd57d2a97a32 + enrFields: + ip: 89.167.114.168 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: true + count: 1 +- name: zeam_0 + privkey: bdf953adc161873ba026330c56450453f582e3c4ee6cb713644794bcfdd85fe5 + enrFields: + ip: 65.109.131.177 + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 +- name: ethlambda_0 + privkey: 299550529a79bc2dce003747c52fb0639465c893e00b0440ac66144d625e066a + enrFields: + ip: 178.104.149.208 + quic: 9003 + metricsPort: 9104 + apiPort: 5057 + isAggregator: false + count: 1 +- name: gean_0 + privkey: 69c251cdb06039dd99d87e5a1439fa3720615be98c293ec9bcfd041877a2e8ca + enrFields: + ip: 95.217.153.36 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: true + count: 1 +- name: ream_8 + privkey: 2e9be3f1b0d32ca3a4d62017fbfafe3950b7e90fed6802ff8bd2e0f8c4e2ca91 + enrFields: + ip: 157.180.20.55 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 +- name: grandine_0 + privkey: c05937b251889e35c58d4601c29bed8153dc22c548448f85e0ab9ca436d4b904 + enrFields: + ip: 89.167.120.224 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: true + count: 1 +- name: zeam_9 + privkey: 0e190e06a62db01bf566af4348277463d26eebab8f6badbcb989242ea4fee050 + enrFields: + ip: 37.27.220.14 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 +- name: ethlambda_9 + privkey: 1176609530e568ec12ad227e60ae71c618bfb13fde5a3257e4b0627216e78e04 + enrFields: + ip: 157.180.116.162 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 +- name: ream_1 + privkey: fc2f11f90dd90df33bfb5f3467c2cbc37cf93dbb41c7ac556f9eea117418e73d + enrFields: + ip: 37.27.220.14 + quic: 9004 + metricsPort: 9105 + apiPort: 5058 + isAggregator: false + count: 1 +- name: zeam_1 + privkey: 8d14c7b02d55ca050ef97a3961aa16828837fb363e4e19e4dd0060f58670a2b3 + enrFields: + ip: 204.168.190.188 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 +- name: ethlambda_1 + privkey: eb8533a0d5071d4dbcb0c4fcf9b8ac6edc3d1a260d2bb348fafc5cdb455aa1d4 + enrFields: + ip: 135.181.82.109 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 +- name: gean_1 + privkey: 5408a68b960a7932e367b32489498a13c339f87ff8090ea54213524b3d76fcff + enrFields: + ip: 95.217.158.60 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 +- name: ream_9 + privkey: 74388a2487b943d947c17bf65ca08470f3ad5f045ea0f5aed38a98bbaba1bd49 + enrFields: + ip: 65.109.131.177 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 +- name: grandine_1 + privkey: ab6edee1173379f647b4022d74d4b3342d547e7bb6954664a1a489b95b7c9b60 + enrFields: + ip: 37.27.89.135 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 +- name: zeam_10 + privkey: 7fdceb8166b9d7ab4169d80345ab85fd042b3684ba307c4c02e851c1520975b5 + enrFields: + ip: 178.104.151.50 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 +- name: ethlambda_10 + privkey: 943151f2327ea91357632dd8a3a498242b07eeeebad973ea82d6bead80627d3e + enrFields: + ip: 178.104.133.162 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 +- name: ream_2 + privkey: d723f0cc94645848bbacf8aea4e01944e2f4a6f9f2c69b17ed1868f492374a7a + enrFields: + ip: 178.104.149.208 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 +- name: zeam_2 + privkey: 11ff3a52375898532680c86739dbe0b632088545d0b901181d92053b5fab8d38 + enrFields: + ip: 178.104.149.91 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 +- name: ethlambda_2 + privkey: 4a7c5b2077d0e3c6f5322d0f2fd98cb2efffbceb41de2799e520b48872dc4102 + enrFields: + ip: 95.216.154.185 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 +- name: gean_2 + privkey: 14da65451d15aee0cbdbcd847cfc3474f106c4595bf716306832fc078858f458 + enrFields: + ip: 204.168.135.7 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 +- name: ream_10 + privkey: 53f1956c292eca6aba53bfd1089682828130db1c0ef980746e5e3e1d94f98360 + enrFields: + ip: 37.27.220.14 + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 +- name: grandine_2 + privkey: b4fdf4bc8ea6e74845b19823510c590406ff63371e0573132f1ddb6b455b7bf7 + enrFields: + ip: 65.21.182.45 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 +- name: zeam_11 + privkey: 1027ce52661ef76b5ada68bc8cd45c13ebefb09c99dcad251b9bd4893c71815f + enrFields: + ip: 65.109.138.213 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 +- name: ethlambda_11 + privkey: 31258c1b84e4a64954fb18dbb04ec9de7079ad4d0273b40380aa1f4ca2f20804 + enrFields: + ip: 204.168.134.201 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 +- name: ream_3 + privkey: 4701dd38aa11a1affd69b5d95e535210f8c29e720222b906c46bd64cf018f9c7 + enrFields: + ip: 95.217.19.42 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 +- name: zeam_3 + privkey: 877c1489e75914bd46dccb71e0ee2d32af337fbb82f2c9caea73818e00ea9a61 + enrFields: + ip: 95.216.173.151 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 +- name: ethlambda_3 + privkey: 1c454b0399cd2178b46e42c8b599e34f2e7ddab71df96b1fbf510d5951335ea0 + enrFields: + ip: 95.216.164.165 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 +- name: gean_3 + privkey: 51832ccf0a189ed39a6fe13833894445b1fa60fc008fcb7fd7e30405318068d1 + enrFields: + ip: 95.216.165.186 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 +- name: ream_11 + privkey: f52f1216a9a20f5b2c200ed64f1a2a8e7427772574147a7edf4602b40f820e9b + enrFields: + ip: 178.104.151.50 + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 +- name: grandine_3 + privkey: 27b642b91b25a4dd55638d7936469270f0fc4b77ceba768617291415eeb3ab1a + enrFields: + ip: 37.27.250.20 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: false + count: 1 +- name: zeam_12 + privkey: 0763e1a9f1909df7eae31ef4e93d5aaf45a506624ef81df893fc02e08a4ab94f + enrFields: + ip: 157.180.116.162 + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 +- name: ethlambda_12 + privkey: a5dac14f317084d255c57e5080985b1dbd79cf8fc625a9c7fba781abf07001bd + enrFields: + ip: 157.180.116.162 + quic: 9004 + metricsPort: 9105 + apiPort: 5058 + isAggregator: false + count: 1 +- name: ream_4 + privkey: 62e6c79311ce4ee1b95ff52c8d6b7cccea0bf0b927fbed595d454858b32c49d6 + enrFields: + ip: 204.168.190.188 + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 +- name: zeam_4 + privkey: 2aed04fc149298af1376aee0161d5dc67f9d7a47e625a58ae8e44f210d45c2a2 + enrFields: + ip: 89.167.120.1 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: true + count: 1 +- name: ethlambda_4 + privkey: 91c2c932a6589e130bfa5281cc27f6b9930ba7f99aa043d334f14578ef7492f1 + enrFields: + ip: 95.217.158.60 + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 +- name: gean_4 + privkey: 9b678649edf603b73913e8957fd7c26fe6af08656b8fe14b0d47aa80d3b59c9f + enrFields: + ip: 37.27.89.135 + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 +- name: ream_12 + privkey: 6cc68faf32d824996ecec2644c61cdb479da9bc1f957f7d09913fac8c49fe578 + enrFields: + ip: 65.109.138.213 + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 +- name: grandine_4 + privkey: 936fa39e4f1ce548d977f852f84ee900bbd2789bf85779e0e958d966a1ba7386 + enrFields: + ip: 157.180.20.55 + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 +- name: zeam_13 + privkey: ce66f19a179bd74e01b775c696b1744060078fe4d84abb5761a3f1d7cef15562 + enrFields: + ip: 178.104.133.162 + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 +- name: ethlambda_13 + privkey: 12bb3c043f2600f27be6bf8916368e9ff01b9ef5c4a7ccdf6aab8e2a24ba55c3 + enrFields: + ip: 178.104.149.208 + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 +- name: ream_5 + privkey: 4caa321fc7598a3da98623a03774ee177c0025fa783fc2fc9c082b297ed96433 + enrFields: + ip: 178.104.149.91 + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 +- name: zeam_5 + privkey: 34cfea8f2e6d6f8a28ae8869220450b8069ba0c9cca436a1079d44fa40989820 + enrFields: + ip: 95.216.154.185 + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 +- name: ethlambda_5 + privkey: 793a5d9add590b890085b440ceefe3e8c805785f7271feee09d35511e7060056 + enrFields: + ip: 89.167.112.241 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: true + count: 1 +- name: gean_5 + privkey: b55c8e411d54ca0ef5ce69e86cba09ae110add1c5d598ed3356adfa5182b82e9 + enrFields: + ip: 65.21.182.45 + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 +- name: ream_13 + privkey: 9cf22e42d463e751958eceace0a3201c661b4326076027c900a04aa845a9e683 + enrFields: + ip: 89.167.3.22 + quic: 9001 + metricsPort: 9102 + apiPort: 5055 + isAggregator: true + count: 1 +- name: grandine_5 + privkey: 99f1df1cbc7429360d52142a8ec5df6a3ad38a4a39897d2ee356b88fd3a69385 + enrFields: + ip: 65.109.131.177 + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 +- name: zeam_14 + privkey: aaf15d4a12fec7409b136d022790d9aaaca3d5679c9e28babe23a424cd80729b + enrFields: + ip: 204.168.134.201 + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 +- name: ethlambda_14 + privkey: 7536676b6ee5f0062030c0b6784c44a175d66cdc39277354c1b9014b419cbc30 + enrFields: + ip: 95.217.19.42 + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 +- name: ream_6 + privkey: 5401bb0acca1620ffddd776c98b5de53034b960eb84a1fe0ce381161789fd980 + enrFields: + ip: 95.216.173.151 + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 +- name: zeam_6 + privkey: 2ac17fb2c419558e349400f45fd5077eac9ca7bcd527b0833ac7f630465c5457 + enrFields: + ip: 95.216.164.165 + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 +- name: ethlambda_6 + privkey: e673e0a9b6ea018ae0b8039019bcee829037dd5638c2b5a5f085385e46231b6a + enrFields: + ip: 95.216.165.186 + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 +- name: gean_6 + privkey: 0db45359918ec742bd1c354af1c5614969985139d2617790dc2ada928e95a731 + enrFields: + ip: 37.27.250.20 + quic: 9002 + metricsPort: 9103 + apiPort: 5056 + isAggregator: false + count: 1 +- name: ream_14 + privkey: c9dcdad85cfffc846c9771f950f4198ae4791972cf86841ee01004f8375b9332 + enrFields: + ip: 157.180.116.162 + quic: 9003 + metricsPort: 9104 + apiPort: 5057 + isAggregator: false + count: 1 +- name: grandine_6 + privkey: eadfc14b6231cb9a207f96f436f7d55c733ba3781306307b7f81f564e2b53f0a + enrFields: + ip: 37.27.220.14 + quic: 9003 + metricsPort: 9104 + apiPort: 5057 + isAggregator: false + count: 1 +- name: zeam_15 + privkey: 2907bc1d1e0a3dee4cc317dfda39a09b5e23596671d7108482edb897616c6b6c + enrFields: + ip: 204.168.135.7 + quic: 9003 + metricsPort: 9104 + apiPort: 5057 + isAggregator: false + count: 1 +- name: ethlambda_15 + privkey: 98da18b276aa5a54ceba15e78260286b9dbe03c4bf8b9db061d024ba4446cfff + enrFields: + ip: 204.168.190.188 + quic: 9003 + metricsPort: 9104 + apiPort: 5057 + isAggregator: false + count: 1 +- name: ream_7 + privkey: a8a9cb1f243380d915e4997b3bc5058295d94704870885d5ed6cb1dfea24d77e + enrFields: + ip: 135.181.82.109 + quic: 9003 + metricsPort: 9104 + apiPort: 5057 + isAggregator: false + count: 1 +- name: zeam_7 + privkey: 97ae5f1d63fa7c05783f7ae5af4ce1fc43a030833c174e430080a8695ce91d56 + enrFields: + ip: 95.217.158.60 + quic: 9003 + metricsPort: 9104 + apiPort: 5057 + isAggregator: false + count: 1 +- name: ethlambda_7 + privkey: 17bea5ada768f12d6f0c95ab9618277527e9691e2e4169e73bc92248da122e68 + enrFields: + ip: 37.27.89.135 + quic: 9003 + metricsPort: 9104 + apiPort: 5057 + isAggregator: false + count: 1 +- name: gean_7 + privkey: 32e4b103c4b349df214d265db9cb1e6d49346e14093fee656828a5e1ef050b8b + enrFields: + ip: 157.180.20.55 + quic: 9003 + metricsPort: 9104 + apiPort: 5057 + isAggregator: false + count: 1 +- name: ream_15 + privkey: c99cd68f66a7460a8aca095d6dd4b6ce953e175d91077f1e7aba27858f68e301 + enrFields: + ip: 178.104.133.162 + quic: 9003 + metricsPort: 9104 + apiPort: 5057 + isAggregator: false + count: 1 +- name: grandine_7 + privkey: 7dd9ca62d79af8e01c95c743ef1a39155a42eded7096599a7f67b3932b387e6c + enrFields: + ip: 178.104.151.50 + quic: 9003 + metricsPort: 9104 + apiPort: 5057 + isAggregator: false + count: 1 diff --git a/scripts/assign-aggregator-ips.py b/scripts/assign-aggregator-ips.py new file mode 100755 index 0000000..5e6d17e --- /dev/null +++ b/scripts/assign-aggregator-ips.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +""" +Assign dedicated aggregator-server IPs after aggregator selection (spin-node.sh). + +Each attestation subnet's aggregator (validator_index % committee_count) is placed +on the matching IP from lean_ethereum_servers.txt Aggregator_servers (one container +per IP: quic 9001, api 5055, metrics 9102). Non-aggregators are evicted from those +IPs to validator-server hosts with free port slots. +""" + +from __future__ import annotations + +import argparse +import sys +from collections import defaultdict +from typing import Any + +import yaml + +# Subnet index → Aggregator_servers IP (lean_ethereum_servers.txt lines 45–52). +SUBNET_AGGREGATOR_IPS: tuple[str, ...] = ( + "77.42.121.211", # subnet 0 + "89.167.41.98", # subnet 1 + "89.167.114.168", # subnet 2 + "89.167.120.1", # subnet 3 + "89.167.112.241", # subnet 4 + "95.217.153.36", # subnet 5 + "89.167.3.22", # subnet 6 + "89.167.120.224", # subnet 7 +) + +AGGREGATOR_IP_SET = frozenset(SUBNET_AGGREGATOR_IPS) +AGGREGATOR_QUIC = 9001 +AGGREGATOR_METRICS = 9102 +AGGREGATOR_API = 5055 + + +def _committee_count(config: dict[str, Any]) -> int: + cfg = config.get("config") or {} + raw = cfg.get("attestation_committee_count", 1) + try: + n = int(raw) + except (TypeError, ValueError): + n = 1 + return max(1, n) + + +def _rows_with_subnet(validators: list[dict[str, Any]], committee_count: int) -> list[tuple[dict[str, Any], int, int]]: + """(validator, validator_index, subnet) for each YAML row.""" + out: list[tuple[dict[str, Any], int, int]] = [] + vi = 0 + for v in validators: + count = int(v.get("count") or 1) + if "subnet" in v and v["subnet"] is not None and v["subnet"] != "": + subnet = int(v["subnet"]) + else: + subnet = vi % committee_count + out.append((v, vi, subnet)) + vi += count + return out + + +def _ports_for_quic(quic: int) -> tuple[int, int]: + offset = quic - AGGREGATOR_QUIC + return AGGREGATOR_METRICS + offset, AGGREGATOR_API + offset + + +def _set_ports(v: dict[str, Any], quic: int) -> None: + v.setdefault("enrFields", {})["quic"] = quic + metrics, api = _ports_for_quic(quic) + v["metricsPort"] = metrics + if "apiPort" in v or "httpPort" not in v: + v["apiPort"] = api + + +def _used_quic_on_ip(validators: list[dict[str, Any]], ip: str) -> set[int]: + used: set[int] = set() + for v in validators: + if v.get("enrFields", {}).get("ip") == ip: + used.add(int(v["enrFields"].get("quic", AGGREGATOR_QUIC))) + return used + + +def _find_validator_slot(validators: list[dict[str, Any]], exclude_ips: frozenset[str]) -> tuple[str, int]: + """Next free (ip, quic) on a non-aggregator host (up to 4 slots per IP).""" + by_ip: dict[str, set[int]] = defaultdict(set) + for v in validators: + ip = v.get("enrFields", {}).get("ip", "") + if not ip or ip in exclude_ips: + continue + by_ip[ip].add(int(v.get("enrFields", {}).get("quic", AGGREGATOR_QUIC))) + + for ip in sorted(by_ip.keys()): + for quic in range(AGGREGATOR_QUIC, AGGREGATOR_QUIC + 4): + if quic not in by_ip[ip]: + return ip, quic + + # No existing validator IP has room — pick any non-aggregator IP (should not happen on devnet). + for ip in sorted(by_ip.keys()): + return ip, AGGREGATOR_QUIC + raise RuntimeError("no validator-server IP with a free port slot for eviction") + + +def assign_aggregator_ips(config: dict[str, Any], *, dry_run: bool = False) -> list[str]: + validators: list[dict[str, Any]] = config.get("validators") or [] + if not validators: + return [] + + committee_count = _committee_count(config) + if committee_count > len(SUBNET_AGGREGATOR_IPS): + raise ValueError( + f"attestation_committee_count={committee_count} exceeds " + f"{len(SUBNET_AGGREGATOR_IPS)} aggregator server IPs" + ) + + rows = _rows_with_subnet(validators, committee_count) + aggregators_by_subnet: dict[int, dict[str, Any]] = {} + for v, _vi, subnet in rows: + if v.get("isAggregator") is True: + if subnet in aggregators_by_subnet: + raise ValueError( + f"subnet {subnet}: multiple aggregators " + f"({aggregators_by_subnet[subnet]['name']}, {v['name']})" + ) + aggregators_by_subnet[subnet] = v + + for subnet in range(committee_count): + if subnet not in aggregators_by_subnet: + raise ValueError(f"subnet {subnet}: no aggregator (isAggregator: true)") + + # ip -> aggregator name that must own this host + owner_by_agg_ip: dict[str, str] = {} + for subnet, agg in aggregators_by_subnet.items(): + target_ip = SUBNET_AGGREGATOR_IPS[subnet] + owner_by_agg_ip[target_ip] = agg["name"] + + changes: list[str] = [] + + def log(msg: str) -> None: + changes.append(msg) + + # Evict validators that occupy an aggregator IP but are not the designated owner. + for v in validators: + ip = v.get("enrFields", {}).get("ip", "") + if ip not in AGGREGATOR_IP_SET: + continue + owner = owner_by_agg_ip.get(ip) + if owner == v["name"]: + continue + new_ip, new_quic = _find_validator_slot(validators, AGGREGATOR_IP_SET) + log(f"evict {v['name']}: {ip} -> {new_ip} quic {new_quic}") + if not dry_run: + v["enrFields"]["ip"] = new_ip + _set_ports(v, new_quic) + + # Place each subnet aggregator on its dedicated IP (single slot). + for subnet, agg in sorted(aggregators_by_subnet.items()): + target_ip = SUBNET_AGGREGATOR_IPS[subnet] + old_ip = agg.get("enrFields", {}).get("ip", "") + if old_ip != target_ip or int(agg.get("enrFields", {}).get("quic", 0)) != AGGREGATOR_QUIC: + log( + f"aggregator {agg['name']} (subnet {subnet}): " + f"{old_ip} -> {target_ip} quic {AGGREGATOR_QUIC}" + ) + if not dry_run: + agg.setdefault("enrFields", {})["ip"] = target_ip + _set_ports(agg, AGGREGATOR_QUIC) + + # Verify: exactly one validator per aggregator IP and all aggregators use allowed IPs. + if not dry_run: + on_agg_ip: dict[str, list[str]] = defaultdict(list) + for v in validators: + ip = v.get("enrFields", {}).get("ip", "") + if ip in AGGREGATOR_IP_SET: + on_agg_ip[ip].append(v["name"]) + for ip, names in on_agg_ip.items(): + if len(names) != 1: + raise RuntimeError(f"aggregator IP {ip}: expected 1 container, got {names}") + for subnet, agg in aggregators_by_subnet.items(): + ip = agg["enrFields"]["ip"] + expected = SUBNET_AGGREGATOR_IPS[subnet] + if ip != expected: + raise RuntimeError(f"{agg['name']}: expected IP {expected}, got {ip}") + if ip not in AGGREGATOR_IP_SET: + raise RuntimeError(f"{agg['name']}: IP {ip} is not an aggregator server") + + return changes + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("validator_config", help="Path to validator-config.yaml") + parser.add_argument("--dry-run", action="store_true", help="Print changes without writing") + args = parser.parse_args() + + with open(args.validator_config) as fh: + config = yaml.safe_load(fh) + + try: + changes = assign_aggregator_ips(config, dry_run=args.dry_run) + except (ValueError, RuntimeError) as exc: + print(f"Error: {exc}", file=sys.stderr) + sys.exit(1) + + if not changes: + print("Aggregator IPs already aligned with aggregator servers.") + return + + for line in changes: + print(line) + + if args.dry_run: + print("(dry-run: no file written)") + return + + with open(args.validator_config, "w") as fh: + yaml.dump(config, fh, default_flow_style=False, sort_keys=False) + print(f"Updated {args.validator_config}") + + +if __name__ == "__main__": + main() From 50ad22e8c4704fb3f99cf3028eb36b96daafe480 Mon Sep 17 00:00:00 2001 From: ch4r10t33r Date: Wed, 20 May 2026 20:43:27 +0100 Subject: [PATCH 16/20] zeam: opt-in --rayon-threads override with aggregator/all knobs Wire the new zeam --rayon-threads CLI flag (zeam #903 / #899) into both the zeam-cmd.sh shell launcher and the ansible/roles/zeam docker run. Two knobs so non-aggregators can stay on zeam's compiled-in auto-split: ZEAM_RAYON_THREADS_AGGREGATOR / zeam_rayon_threads_aggregator aggregator-only override (wins for aggregators) ZEAM_RAYON_THREADS / zeam_rayon_threads uniform override applied to both roles Both unset (the default) is required for pre-#903 zeam images, which would refuse the flag and fail to start. The 16-vCPU recommended starting value is 12 (= cpu_count - 4 reserved system threads). --- ansible/inventory/group_vars/all.yml | 18 +++++++++++++++ ansible/roles/zeam/tasks/main.yml | 18 +++++++++++++++ client-cmds/zeam-cmd.sh | 34 ++++++++++++++++++++++++++-- 3 files changed, 68 insertions(+), 2 deletions(-) diff --git a/ansible/inventory/group_vars/all.yml b/ansible/inventory/group_vars/all.yml index 66d80d4..451fcfa 100644 --- a/ansible/inventory/group_vars/all.yml +++ b/ansible/inventory/group_vars/all.yml @@ -47,6 +47,24 @@ docker_memory_limits: lighthouse: "4g" peam: "4g" +# zeam --rayon-threads override for the multisig (XMSS) aggregate prover +# (zeam #903 / #899). Mirror of the ZEAM_RAYON_THREADS_AGGREGATOR / +# ZEAM_RAYON_THREADS env vars in client-cmds/zeam-cmd.sh. +# +# zeam_rayon_threads_aggregator wins for aggregators; zeam_rayon_threads +# applies to both roles otherwise. Both empty (the default) means no flag +# is passed and zeam picks its compiled-in auto-split. +# +# 16-vCPU recommended starting point: 12 (= cpu_count - 4 reserved system +# threads). Do not exceed cpu_count - 4, or libxev/libp2p/api/metrics +# threads start to starve — watch +# `zeam_fork_choice_tick_interval_duration_seconds` p99. +# +# REQUIRES: a zeam image built from PR #903 or later. Older images will +# refuse `--rayon-threads` and fail to start; leave both empty in that case. +zeam_rayon_threads_aggregator: "" +zeam_rayon_threads: "" + # Node deployment mode: 'docker' or 'binary' deployment_mode: docker diff --git a/ansible/roles/zeam/tasks/main.yml b/ansible/roles/zeam/tasks/main.yml index 8b80327..6b7f48f 100644 --- a/ansible/roles/zeam/tasks/main.yml +++ b/ansible/roles/zeam/tasks/main.yml @@ -88,6 +88,23 @@ zeam_aggregate_subnet_ids: "" when: (zeam_is_aggregator | default('false')) != 'true' +# Resolve --rayon-threads override for this node. Mirrors the zeam-cmd.sh +# precedence: zeam_rayon_threads_aggregator wins for aggregators, otherwise +# zeam_rayon_threads applies to both roles, otherwise the flag is omitted +# entirely so zeam picks its compiled-in auto-split (zeam #903 / #899). +# +# Both vars are typically set in group_vars/all.yml. Default unset means no +# flag is passed — required for pre-#903 images, which would reject it. +- name: Resolve effective rayon-threads override + set_fact: + zeam_effective_rayon_threads: >- + {%- if (zeam_is_aggregator | default('false')) == 'true' + and (zeam_rayon_threads_aggregator | default('') | string | length) > 0 -%} + {{ zeam_rayon_threads_aggregator }} + {%- elif (zeam_rayon_threads | default('') | string | length) > 0 -%} + {{ zeam_rayon_threads }} + {%- else -%}{%- endif -%} + - name: Ensure node key file exists stat: path: "{{ genesis_dir }}/{{ node_name }}.key" @@ -145,6 +162,7 @@ {{ ('--checkpoint-sync-url ' + checkpoint_sync_url) if (checkpoint_sync_url is defined and checkpoint_sync_url | length > 0) else '' }} {{ ('--db-backend ' + zeam_db_backend) if (zeam_db_backend | default('') | length > 0) else '' }} {{ ('--chain-worker ' + zeam_chain_worker) if (zeam_chain_worker | default('') | length > 0) else '' }} + {{ ('--rayon-threads ' + (zeam_effective_rayon_threads | trim)) if (zeam_effective_rayon_threads | default('') | trim | length > 0) else '' }} register: zeam_container_result changed_when: zeam_container_result.rc == 0 diff --git a/client-cmds/zeam-cmd.sh b/client-cmds/zeam-cmd.sh index 969036b..ce30c38 100644 --- a/client-cmds/zeam-cmd.sh +++ b/client-cmds/zeam-cmd.sh @@ -86,6 +86,34 @@ case "$zeam_chain_worker" in ;; esac +# Rayon worker count for the multisig (XMSS) aggregate prover (zeam #903 / #899). +# +# Default unset → zeam picks an auto-split that gives roughly half of the +# post-system-thread CPU budget to rayon and half to its Zig worker pool. That +# split is fine for non-aggregators (which mostly verify, also via rayon-from- +# Zig-workers) but underuses CPU on CPU-rich aggregators where the produce-path +# FFI is the per-slot bottleneck. +# +# Two knobs so non-aggregators stay on the default: +# - export ZEAM_RAYON_THREADS_AGGREGATOR=12 # aggregator-only override +# - export ZEAM_RAYON_THREADS=12 # uniform override for both roles +# The aggregator-specific value wins for aggregators when both are set. +# +# Sizing guidance for a 16-vCPU host: 12 is the recommended starting point +# (cpu_count - 4 reserved system threads: libxev/libp2p/api/metrics). Do not +# exceed cpu_count - 4 or those reserved threads start to starve, surfacing as +# `zeam_fork_choice_tick_interval_duration_seconds` p99 climbing. +# +# REQUIRES: a zeam build with PR #903 merged plus a docker image cut from it. +# Older images do not recognise `--rayon-threads` and will fail to start. Leave +# both env vars unset to suppress the flag entirely for pre-#903 images. +rayon_threads_flag="" +if [ "$isAggregator" == "true" ] && [ -n "${ZEAM_RAYON_THREADS_AGGREGATOR:-}" ]; then + rayon_threads_flag="--rayon-threads $ZEAM_RAYON_THREADS_AGGREGATOR" +elif [ -n "${ZEAM_RAYON_THREADS:-}" ]; then + rayon_threads_flag="--rayon-threads $ZEAM_RAYON_THREADS" +fi + node_binary="$scriptDir/../zig-out/bin/zeam $zeam_global_flags node \ --custom-genesis $configDir \ --validator-config $validatorConfig \ @@ -99,7 +127,8 @@ node_binary="$scriptDir/../zig-out/bin/zeam $zeam_global_flags node \ $aggregate_subnet_ids_flag \ $checkpoint_sync_flag \ $db_backend_flag \ - $chain_worker_flag" + $chain_worker_flag \ + $rayon_threads_flag" node_docker="--security-opt seccomp=unconfined blockblaz/zeam:devnet4 $zeam_global_flags node \ --custom-genesis /config \ @@ -114,7 +143,8 @@ node_docker="--security-opt seccomp=unconfined blockblaz/zeam:devnet4 $zeam_glob $aggregate_subnet_ids_flag \ $checkpoint_sync_flag \ $db_backend_flag \ - $chain_worker_flag" + $chain_worker_flag \ + $rayon_threads_flag" # choose either binary or docker node_setup="docker" From 3679c87d703786c9917298f8442b91ba7931bce2 Mon Sep 17 00:00:00 2001 From: ch4r10t33r Date: Thu, 21 May 2026 10:51:38 +0100 Subject: [PATCH 17/20] zeam-cmd: default --rayon-threads 12 for aggregators Apply twelve rayon workers whenever isAggregator is true unless ZEAM_RAYON_THREADS_AGGREGATOR overrides it. --- client-cmds/zeam-cmd.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/client-cmds/zeam-cmd.sh b/client-cmds/zeam-cmd.sh index ce30c38..c8b85b4 100644 --- a/client-cmds/zeam-cmd.sh +++ b/client-cmds/zeam-cmd.sh @@ -94,10 +94,11 @@ esac # Zig-workers) but underuses CPU on CPU-rich aggregators where the produce-path # FFI is the per-slot bottleneck. # -# Two knobs so non-aggregators stay on the default: -# - export ZEAM_RAYON_THREADS_AGGREGATOR=12 # aggregator-only override -# - export ZEAM_RAYON_THREADS=12 # uniform override for both roles -# The aggregator-specific value wins for aggregators when both are set. +# Aggregators default to 12 rayon threads; non-aggregators stay on zeam's +# auto-split unless overridden: +# - ZEAM_RAYON_THREADS_AGGREGATOR # aggregator override (default 12) +# - ZEAM_RAYON_THREADS # uniform override for both roles +# For aggregators, ZEAM_RAYON_THREADS_AGGREGATOR wins when set; otherwise 12. # # Sizing guidance for a 16-vCPU host: 12 is the recommended starting point # (cpu_count - 4 reserved system threads: libxev/libp2p/api/metrics). Do not @@ -108,8 +109,8 @@ esac # Older images do not recognise `--rayon-threads` and will fail to start. Leave # both env vars unset to suppress the flag entirely for pre-#903 images. rayon_threads_flag="" -if [ "$isAggregator" == "true" ] && [ -n "${ZEAM_RAYON_THREADS_AGGREGATOR:-}" ]; then - rayon_threads_flag="--rayon-threads $ZEAM_RAYON_THREADS_AGGREGATOR" +if [ "$isAggregator" == "true" ]; then + rayon_threads_flag="--rayon-threads ${ZEAM_RAYON_THREADS_AGGREGATOR:-12}" elif [ -n "${ZEAM_RAYON_THREADS:-}" ]; then rayon_threads_flag="--rayon-threads $ZEAM_RAYON_THREADS" fi From 29ecfe4f5dca2cfc039a158c5f9befcc0b8e6ab4 Mon Sep 17 00:00:00 2001 From: ch4r10t33r Date: Thu, 21 May 2026 13:28:06 +0100 Subject: [PATCH 18/20] ansible,spin-node: always pull client docker images before run Add an explicit docker pull step to every client Ansible role and use --pull=always on spin-node docker runs so registry tags are refreshed on each deploy. --- ansible/roles/ethlambda/tasks/main.yml | 5 + ansible/roles/gean/tasks/main.yml | 5 + ansible/roles/grandine/tasks/main.yml | 5 + ansible/roles/lantern/tasks/main.yml | 5 + ansible/roles/lighthouse/tasks/main.yml | 1 + ansible/roles/nlean/tasks/main.yml | 5 + ansible/roles/peam/tasks/main.yml | 5 + ansible/roles/qlean/tasks/main.yml | 5 + ansible/roles/ream/tasks/main.yml | 5 + ansible/roles/zeam/tasks/main.yml | 5 + spin-node.sh | 152 +++++++++++++++--------- 11 files changed, 143 insertions(+), 55 deletions(-) diff --git a/ansible/roles/ethlambda/tasks/main.yml b/ansible/roles/ethlambda/tasks/main.yml index e5aa7f5..b3fd678 100644 --- a/ansible/roles/ethlambda/tasks/main.yml +++ b/ansible/roles/ethlambda/tasks/main.yml @@ -110,6 +110,11 @@ - name: Deploy Ethlambda node using Docker block: + - name: Pull Ethlambda docker image + command: docker pull {{ ethlambda_docker_image }} + register: ethlambda_docker_pull + changed_when: '"Downloaded newer image" in ethlambda_docker_pull.stdout' + - name: Stop existing Ethlambda container (if any) command: docker rm -f {{ node_name }} register: ethlambda_stop diff --git a/ansible/roles/gean/tasks/main.yml b/ansible/roles/gean/tasks/main.yml index 4435d60..fe4f4e3 100644 --- a/ansible/roles/gean/tasks/main.yml +++ b/ansible/roles/gean/tasks/main.yml @@ -74,6 +74,11 @@ - name: Deploy Gean node using Docker block: + - name: Pull Gean docker image + command: docker pull {{ gean_docker_image }} + register: gean_docker_pull + changed_when: '"Downloaded newer image" in gean_docker_pull.stdout' + - name: Stop existing Gean container (if any) command: docker rm -f {{ node_name }} register: gean_stop diff --git a/ansible/roles/grandine/tasks/main.yml b/ansible/roles/grandine/tasks/main.yml index f0ba526..e52d257 100644 --- a/ansible/roles/grandine/tasks/main.yml +++ b/ansible/roles/grandine/tasks/main.yml @@ -80,6 +80,11 @@ - name: Deploy Grandine node using Docker block: + - name: Pull Grandine docker image + command: docker pull {{ grandine_docker_image }} + register: grandine_docker_pull + changed_when: '"Downloaded newer image" in grandine_docker_pull.stdout' + - name: Stop existing Grandine container (if any) command: docker rm -f {{ node_name }} register: grandine_stop diff --git a/ansible/roles/lantern/tasks/main.yml b/ansible/roles/lantern/tasks/main.yml index 90a03d5..8ff3a0b 100644 --- a/ansible/roles/lantern/tasks/main.yml +++ b/ansible/roles/lantern/tasks/main.yml @@ -65,6 +65,11 @@ - name: Deploy Lantern node using Docker block: + - name: Pull Lantern docker image + command: docker pull {{ lantern_docker_image }} + register: lantern_docker_pull + changed_when: '"Downloaded newer image" in lantern_docker_pull.stdout' + - name: Stop existing Lantern container (if any) command: docker rm -f {{ node_name }} register: lantern_stop diff --git a/ansible/roles/lighthouse/tasks/main.yml b/ansible/roles/lighthouse/tasks/main.yml index 1de0b91..9a99388 100644 --- a/ansible/roles/lighthouse/tasks/main.yml +++ b/ansible/roles/lighthouse/tasks/main.yml @@ -87,6 +87,7 @@ - name: Start Lighthouse container command: >- docker run -d + --pull=always --name {{ node_name }} --restart {{ docker_container_restart_policy }} {{ ('--memory ' + docker_memory_limits.lighthouse) if (lighthouse_is_aggregator | default('false')) != 'true' else '' }} diff --git a/ansible/roles/nlean/tasks/main.yml b/ansible/roles/nlean/tasks/main.yml index 9b4c70e..604c7ee 100644 --- a/ansible/roles/nlean/tasks/main.yml +++ b/ansible/roles/nlean/tasks/main.yml @@ -74,6 +74,11 @@ - name: Deploy Nlean node using Docker block: + - name: Pull Nlean docker image + command: docker pull {{ nlean_docker_image }} + register: nlean_docker_pull + changed_when: '"Downloaded newer image" in nlean_docker_pull.stdout' + - name: Stop existing Nlean container (if any) command: docker rm -f {{ node_name }} register: nlean_stop diff --git a/ansible/roles/peam/tasks/main.yml b/ansible/roles/peam/tasks/main.yml index 584983e..9f798a9 100644 --- a/ansible/roles/peam/tasks/main.yml +++ b/ansible/roles/peam/tasks/main.yml @@ -125,6 +125,11 @@ - name: Deploy Peam node using Docker block: + - name: Pull Peam docker image + command: docker pull {{ peam_docker_image }} + register: peam_docker_pull + changed_when: '"Downloaded newer image" in peam_docker_pull.stdout' + - name: Stop existing Peam container (if any) command: docker rm -f {{ node_name }} register: peam_stop diff --git a/ansible/roles/qlean/tasks/main.yml b/ansible/roles/qlean/tasks/main.yml index d02c639..5316eca 100644 --- a/ansible/roles/qlean/tasks/main.yml +++ b/ansible/roles/qlean/tasks/main.yml @@ -72,6 +72,11 @@ - name: Deploy Qlean node using Docker block: + - name: Pull Qlean docker image + command: docker pull {{ qlean_docker_image }} + register: qlean_docker_pull + changed_when: '"Downloaded newer image" in qlean_docker_pull.stdout' + - name: Stop existing Qlean container (if any) command: docker rm -f {{ node_name }} register: qlean_stop diff --git a/ansible/roles/ream/tasks/main.yml b/ansible/roles/ream/tasks/main.yml index 6eb61ea..4f0b0af 100644 --- a/ansible/roles/ream/tasks/main.yml +++ b/ansible/roles/ream/tasks/main.yml @@ -73,6 +73,11 @@ - name: Deploy Ream node using Docker block: + - name: Pull Ream docker image + command: docker pull {{ ream_docker_image }} + register: ream_docker_pull + changed_when: '"Downloaded newer image" in ream_docker_pull.stdout' + - name: Stop existing Ream container (if any) command: docker rm -f {{ node_name }} register: ream_stop diff --git a/ansible/roles/zeam/tasks/main.yml b/ansible/roles/zeam/tasks/main.yml index 6b7f48f..02f54d4 100644 --- a/ansible/roles/zeam/tasks/main.yml +++ b/ansible/roles/zeam/tasks/main.yml @@ -127,6 +127,11 @@ - name: Deploy Zeam node using Docker block: + - name: Pull Zeam docker image + command: docker pull {{ zeam_docker_image }} + register: zeam_docker_pull + changed_when: '"Downloaded newer image" in zeam_docker_pull.stdout' + - name: Stop existing Zeam container (if any) command: docker rm -f {{ node_name }} register: docker_stop_result diff --git a/spin-node.sh b/spin-node.sh index 08286ea..66badb9 100755 --- a/spin-node.sh +++ b/spin-node.sh @@ -287,7 +287,7 @@ if [ -n "$stopAllContainers" ] && [ "$stopAllContainers" == "true" ]; then exit 1 fi - [ "$dryRun" == "true" ] && echo "✅ Dry-run complete — no changes were made." || echo "✅ Stopped all non-observability containers on validator hosts." + [ "$dryRun" == "true" ] && echo "✅ Dry-run complete — no changes were made." || echo "✅ Stopped all non-observability containers on reachable validator hosts (unreachable hosts skipped)." exit 0 fi @@ -343,15 +343,13 @@ restart_with_checkpoint_sync=false # shows ops/inventory groups — not the per-validator committee subnet map. # Nodes without 'subnet:' and without a clear rule default to subnet 0. # -# When --aggregator is specified, that node is used as the aggregator for -# its own subnet; all other subnets still get a random selection (still -# excluding that node's client type from pools on other subnets). +# Default: keep isAggregator flags from validator-config.yaml (no reshuffle). +# Use --randomize to reset and pick one aggregator per subnet at random (unique +# by client type when possible). --aggregator sets one subnet's aggregator; +# with --randomize, other subnets are still filled randomly. # -# Default random mode (no --aggregator): aggregators are unique by CLIENT -# (prefix before the first '_', e.g. zeam from zeam_0). Example with 5 subnets: -# if zeam_* is chosen for subnet 0, no zeam_* node may be aggregator on -# subnets 1–4. If subnets outnumber distinct clients, the pool is exhausted -# and we fall back to unrestricted random with a warning. +# Ansible mode: scripts/assign-aggregator-ips.py aligns each subnet aggregator +# onto the matching Aggregator_servers IP from lean_ethereum_servers.txt. # Helper: get the subnet index for a node from the config. # If the node has an explicit 'subnet' field, use it. @@ -387,10 +385,45 @@ _client_prefix() { esac } +_aggregator_summary=() +_subnet_indices=() +for _node in "${nodes[@]}"; do + _subnet_indices+=("$(_node_subnet "$_node")") +done +_unique_subnets=($(printf '%s\n' "${_subnet_indices[@]}" | sort -un)) +echo "Detected ${#_unique_subnets[@]} subnet(s): ${_unique_subnets[*]}" + +_verify_aggregator_invariant() { + local _verify_failed=false + for _subnet_idx in "${_unique_subnets[@]}"; do + local _agg_count=0 + local _agg_name="" + for _node in "${nodes[@]}"; do + if [[ "$(_node_subnet "$_node")" == "$_subnet_idx" ]]; then + local _is_agg + _is_agg=$(yq eval ".validators[] | select(.name == \"$_node\") | .isAggregator" "$validator_config_file") + if [[ "$_is_agg" == "true" ]]; then + _agg_count=$((_agg_count + 1)) + _agg_name="$_node" + fi + fi + done + if [ "$_agg_count" -ne 1 ]; then + echo "Error: subnet $_subnet_idx has $_agg_count aggregator(s) — expected exactly 1" >&2 + _verify_failed=true + else + _aggregator_summary+=("subnet $_subnet_idx → $_agg_name") + fi + done + if [ "$_verify_failed" == "true" ]; then + echo "Aggregator invariant check failed. Aborting." >&2 + exit 1 + fi +} + if [ -n "$restartClient" ]; then echo "Note: skipping aggregator selection — --restart-client retains existing isAggregator assignments." - _aggregator_summary=() -else +elif [ "$randomizeAggregators" == "true" ]; then # If --aggregator was given, validate it exists before doing anything else. if [ -n "$aggregatorNode" ]; then @@ -401,45 +434,30 @@ else break fi done - if [[ "$aggregator_found" == false ]]; then + if [[ "$aggregator_found" == "false" ]]; then echo "Error: Specified aggregator '$aggregatorNode' not found in validator config" echo "Available nodes: ${nodes[@]}" exit 1 fi fi - # Collect unique subnet indices from the 'subnet' field (0 when absent). - _subnet_indices=() - for _node in "${nodes[@]}"; do - _subnet_indices+=("$(_node_subnet "$_node")") - done - _unique_subnets=($(printf '%s\n' "${_subnet_indices[@]}" | sort -un)) - - echo "Detected ${#_unique_subnets[@]} subnet(s): ${_unique_subnets[*]}" + _aggregator_summary=() - # Snapshot which nodes already have isAggregator: true before we reset anything. - # This lets us honour manual edits in the YAML when no --aggregator flag was passed. - # Uses dynamic variable names (_preset_agg_) for bash 3.2 compatibility - # (bash 3.2 ships with macOS and does not support declare -A). + # Snapshot presets before reset (used when --aggregator does not cover a subnet). for _node in "${nodes[@]}"; do _is_agg=$(yq eval ".validators[] | select(.name == \"$_node\") | .isAggregator" "$validator_config_file") if [[ "$_is_agg" == "true" ]]; then _sn="$(_node_subnet "$_node")" _varname="_preset_agg_${_sn}" - # Keep the first preset aggregator found per subnet. [[ -z "${!_varname:-}" ]] && printf -v "$_varname" '%s' "$_node" fi done - # Reset every node's isAggregator flag (skipped in dry-run). if [ "$dryRun" != "true" ]; then yq eval -i '.validators[].isAggregator = false' "$validator_config_file" fi - # Select one aggregator per subnet and set the flag. - # Priority: 1) --aggregator CLI flag 2) pre-existing isAggregator: true 3) random - # _used_agg_prefixes: client types already chosen (default random / preset / --aggregator). - _aggregator_summary=() + # Priority per subnet: 1) --aggregator 2) preset (if valid) 3) random _used_agg_prefixes=" " for _subnet_idx in "${_unique_subnets[@]}"; do _subnet_nodes=() @@ -450,19 +468,15 @@ else _selected_agg="" if [ -n "$aggregatorNode" ] && [[ "$(_node_subnet "$aggregatorNode")" == "$_subnet_idx" ]]; then - # 1. Explicit --aggregator flag. _selected_agg="$aggregatorNode" elif _pv="_preset_agg_${_subnet_idx}"; [ -n "${!_pv:-}" ]; then - # 2. A node had isAggregator: true in the config — respect the manual choice. _preset="${!_pv}" - # Validate the preset node is still in the active nodes list. _preset_valid=false for _n in "${_subnet_nodes[@]}"; do [[ "$_n" == "$_preset" ]] && _preset_valid=true && break done if [[ "$_preset_valid" == "true" ]]; then _selected_agg="$_preset" - # Default mode: one client type at most once across subnets — drop conflicting presets. if [ -z "$aggregatorNode" ]; then _pp="$(_client_prefix "$_selected_agg")" if [[ "$_used_agg_prefixes" == *" $_pp "* ]]; then @@ -471,13 +485,11 @@ else fi fi else - # Preset node no longer exists — fall back to random and warn. echo "Warning: preset aggregator '$_preset' for subnet $_subnet_idx is not in the active node list; selecting randomly." >&2 _selected_agg="" fi fi - # 3. Random (or preset fallback): prefer client types not yet used as aggregator. if [ -z "$_selected_agg" ]; then _eligible_aggs=() for _n in "${_subnet_nodes[@]}"; do @@ -505,31 +517,61 @@ else if [ "$dryRun" != "true" ]; then yq eval -i "(.validators[] | select(.name == \"$_selected_agg\") | .isAggregator) = true" "$validator_config_file" fi - _aggregator_summary+=("subnet $_subnet_idx → $_selected_agg") + _aggregator_summary+=("subnet $_subnet_idx → $_selected_agg (randomized)") done - # Verify the invariant: exactly 1 aggregator per subnet (skipped in dry-run). if [ "$dryRun" != "true" ]; then - _verify_failed=false - for _subnet_idx in "${_unique_subnets[@]}"; do - _agg_count=0 - for _node in "${nodes[@]}"; do - if [[ "$(_node_subnet "$_node")" == "$_subnet_idx" ]]; then - _is_agg=$(yq eval ".validators[] | select(.name == \"$_node\") | .isAggregator" "$validator_config_file") - [[ "$_is_agg" == "true" ]] && _agg_count=$((_agg_count + 1)) + _verify_aggregator_invariant + fi + +elif [ -n "$aggregatorNode" ]; then + aggregator_found=false + for available_node in "${nodes[@]}"; do + if [[ "$aggregatorNode" == "$available_node" ]]; then + aggregator_found=true + break + fi + done + if [[ "$aggregator_found" == "false" ]]; then + echo "Error: Specified aggregator '$aggregatorNode' not found in validator config" + echo "Available nodes: ${nodes[@]}" + exit 1 + fi + + _agg_subnet="$(_node_subnet "$aggregatorNode")" + echo "Setting aggregator for subnet $_agg_subnet to $aggregatorNode (other subnets unchanged)." + if [ "$dryRun" != "true" ]; then + for _node in "${nodes[@]}"; do + if [[ "$(_node_subnet "$_node")" == "$_agg_subnet" ]]; then + if [[ "$_node" == "$aggregatorNode" ]]; then + yq eval -i "(.validators[] | select(.name == \"$_node\") | .isAggregator) = true" "$validator_config_file" + else + yq eval -i "(.validators[] | select(.name == \"$_node\") | .isAggregator) = false" "$validator_config_file" fi - done - if [ "$_agg_count" -ne 1 ]; then - echo "Error: subnet $_subnet_idx has $_agg_count aggregator(s) — expected exactly 1" >&2 - _verify_failed=true fi done - if [ "$_verify_failed" == "true" ]; then - echo "Aggregator invariant check failed. Aborting." >&2 - exit 1 - fi + _verify_aggregator_invariant + else + _aggregator_summary+=("subnet $_agg_subnet → $aggregatorNode (--aggregator)") + fi + +else + echo "Keeping isAggregator assignments from validator-config.yaml (use --randomize to reshuffle)." + if [ "$dryRun" != "true" ]; then + _verify_aggregator_invariant fi +fi +# Ansible: align subnet aggregators onto dedicated Aggregator_servers IPs. +if [ -z "$restartClient" ] && [ "$deployment_mode" == "ansible" ]; then + _assign_agg_ip_args=("$scriptDir/scripts/assign-aggregator-ips.py" "$validator_config_file") + if [ "$dryRun" == "true" ]; then + _assign_agg_ip_args+=(--dry-run) + fi + if ! python3 "${_assign_agg_ip_args[@]}"; then + echo "Error: failed to assign aggregator server IPs." >&2 + exit 1 + fi fi # end: aggregator selection (skipped for --restart-client) # Print aggregator selection summary inline (quick confirmation during setup). @@ -993,13 +1035,13 @@ for item in "${spin_nodes[@]}"; do else # Extract image name from node_docker (find word containing ':' which is the image:tag) docker_image=$(echo "$node_docker" | grep -oE '[^ ]+:[^ ]+' | head -1) - # Pull image first + # Pull image before run; --pull=always re-checks the registry at start. if [ -n "$dockerWithSudo" ]; then sudo docker pull "$docker_image" || true else docker pull "$docker_image" || true fi - execCmd="docker run --rm --pull=never" + execCmd="docker run --rm --pull=always" if [ -n "$dockerWithSudo" ] then execCmd="sudo $execCmd" From eed3430d0d8d460cfa28bb7d92b4600254803404 Mon Sep 17 00:00:00 2001 From: ch4r10t33r Date: Mon, 25 May 2026 00:34:15 +0100 Subject: [PATCH 19/20] ansible, genesis: devnet4 deploy fixes and subnet 2 lantern swap Fix zeam chain-worker and rayon-threads CLI generation, set aggregator rayon to 12, replace ream subnet 2 with lantern, and harden stop-all-containers against unreachable hosts. --- README.md | 21 ++++++------- ansible-devnet/genesis/validator-config.yaml | 16 +++++----- ansible/inventory/group_vars/all.yml | 2 +- ansible/playbooks/stop-all-containers.yml | 16 +++++++++- ansible/roles/zeam/defaults/main.yml | 18 +++++------ ansible/roles/zeam/tasks/main.yml | 19 ++++++----- client-cmds/zeam-cmd.sh | 33 ++++++++------------ parse-env.sh | 7 ++++- run-ansible.sh | 3 +- 9 files changed, 73 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index ef591d1..741e93d 100644 --- a/README.md +++ b/README.md @@ -86,14 +86,14 @@ Grafana is started with the two pre-provisioned dashboards from [leanMetrics](ht ### Aggregator Selection ```sh -# Let the system randomly select an aggregator (default behavior) +# Default: keep isAggregator flags from validator-config.yaml NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis -# Manually specify which node should be the aggregator -NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis --aggregator zeam_0 +# Randomly select one aggregator per subnet (unique by client type when possible) +NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis --randomize -# The aggregator selection is applied automatically and the isAggregator flag -# is updated in validator-config.yaml before nodes are started +# Set one subnet's aggregator without changing other subnets +NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis --aggregator zeam_0 ``` ### Leanpoint deployment @@ -217,12 +217,11 @@ Every Ansible deployment automatically deploys an observability stack alongside - On Ctrl+C cleanup, the metrics stack is stopped automatically Note: Client metrics endpoints are always enabled regardless of this flag. -12. `--aggregator` specifies which node should act as the aggregator (1 aggregator per subnet). - - If not provided, one node will be randomly selected as the aggregator - - If provided, the specified node will be set as the aggregator - - The aggregator selection updates the `isAggregator` flag in `validator-config.yaml` - - Example: `--aggregator zeam_0` to make zeam_0 the aggregator - - Example: Without flag, a random node will be selected automatically +12. Aggregator flags (1 aggregator per subnet; `isAggregator` in `validator-config.yaml`): + - **Default:** existing `isAggregator` assignments are kept (not reshuffled on each run) + - `--randomize` resets and randomly selects one aggregator per subnet (unique by client type when possible) + - `--aggregator zeam_0` sets that node as aggregator for its subnet only; other subnets unchanged + - With `--randomize --aggregator zeam_0`, that subnet uses `zeam_0` and remaining subnets are filled at random 13. `--checkpoint-sync-url` specifies the URL to fetch finalized checkpoint state from for checkpoint sync. Default: `https://leanpoint.leanroadmap.org/lean/v0/states/finalized`. Only used when `--restart-client` is specified. 14. `--restart-client` comma-separated list of client node names (e.g., `zeam_0,ream_0`). When specified, those clients are stopped, their data cleared, and restarted using checkpoint sync. Genesis is skipped. Use with `--checkpoint-sync-url` to override the default URL. 15. `--prepare` verify and install the software required to run lean nodes on every remote server, and open + persist the necessary firewall ports. diff --git a/ansible-devnet/genesis/validator-config.yaml b/ansible-devnet/genesis/validator-config.yaml index 862cdb3..a2a37a9 100644 --- a/ansible-devnet/genesis/validator-config.yaml +++ b/ansible-devnet/genesis/validator-config.yaml @@ -23,7 +23,7 @@ validators: apiPort: 5055 isAggregator: true count: 1 -- name: ream_0 +- name: lantern_0 privkey: af27950128b49cda7e7bc9fcb7b0270f7a3945aa7543326f3bfdbd57d2a97a32 enrFields: ip: 89.167.114.168 @@ -95,7 +95,7 @@ validators: apiPort: 5055 isAggregator: false count: 1 -- name: ream_1 +- name: lantern_1 privkey: fc2f11f90dd90df33bfb5f3467c2cbc37cf93dbb41c7ac556f9eea117418e73d enrFields: ip: 37.27.220.14 @@ -167,7 +167,7 @@ validators: apiPort: 5055 isAggregator: false count: 1 -- name: ream_2 +- name: lantern_2 privkey: d723f0cc94645848bbacf8aea4e01944e2f4a6f9f2c69b17ed1868f492374a7a enrFields: ip: 178.104.149.208 @@ -239,7 +239,7 @@ validators: apiPort: 5055 isAggregator: false count: 1 -- name: ream_3 +- name: lantern_3 privkey: 4701dd38aa11a1affd69b5d95e535210f8c29e720222b906c46bd64cf018f9c7 enrFields: ip: 95.217.19.42 @@ -311,7 +311,7 @@ validators: apiPort: 5058 isAggregator: false count: 1 -- name: ream_4 +- name: lantern_4 privkey: 62e6c79311ce4ee1b95ff52c8d6b7cccea0bf0b927fbed595d454858b32c49d6 enrFields: ip: 204.168.190.188 @@ -383,7 +383,7 @@ validators: apiPort: 5056 isAggregator: false count: 1 -- name: ream_5 +- name: lantern_5 privkey: 4caa321fc7598a3da98623a03774ee177c0025fa783fc2fc9c082b297ed96433 enrFields: ip: 178.104.149.91 @@ -455,7 +455,7 @@ validators: apiPort: 5056 isAggregator: false count: 1 -- name: ream_6 +- name: lantern_6 privkey: 5401bb0acca1620ffddd776c98b5de53034b960eb84a1fe0ce381161789fd980 enrFields: ip: 95.216.173.151 @@ -527,7 +527,7 @@ validators: apiPort: 5057 isAggregator: false count: 1 -- name: ream_7 +- name: lantern_7 privkey: a8a9cb1f243380d915e4997b3bc5058295d94704870885d5ed6cb1dfea24d77e enrFields: ip: 135.181.82.109 diff --git a/ansible/inventory/group_vars/all.yml b/ansible/inventory/group_vars/all.yml index 451fcfa..3adc5b6 100644 --- a/ansible/inventory/group_vars/all.yml +++ b/ansible/inventory/group_vars/all.yml @@ -62,7 +62,7 @@ docker_memory_limits: # # REQUIRES: a zeam image built from PR #903 or later. Older images will # refuse `--rayon-threads` and fail to start; leave both empty in that case. -zeam_rayon_threads_aggregator: "" +zeam_rayon_threads_aggregator: "12" zeam_rayon_threads: "" # Node deployment mode: 'docker' or 'binary' diff --git a/ansible/playbooks/stop-all-containers.yml b/ansible/playbooks/stop-all-containers.yml index d9ab873..8de3b98 100644 --- a/ansible/playbooks/stop-all-containers.yml +++ b/ansible/playbooks/stop-all-containers.yml @@ -9,6 +9,7 @@ - name: Stop non-observability containers on all validator hosts hosts: prepare_hosts gather_facts: no + ignore_unreachable: true vars: observability_containers: - prometheus @@ -24,7 +25,7 @@ - name: Stop and remove every container except observability stack command: docker rm -f {{ item }} - loop: "{{ all_containers.stdout_lines | difference(observability_containers) | select('length') | list }}" + loop: "{{ all_containers.stdout_lines | default([]) | difference(observability_containers) | reject('equalto', '') | list }}" register: stop_result failed_when: false @@ -34,3 +35,16 @@ when: - stop_result.results is defined - stop_result.results | selectattr('rc', 'defined') | selectattr('rc', 'equalto', 0) | list | length > 0 + +- name: Summarize stop-all-containers run + hosts: localhost + connection: local + gather_facts: no + + tasks: + - name: Warn about unreachable validator hosts + debug: + msg: "Skipped (SSH unreachable): {{ hostvars[item].ansible_host | default(item) }}" + loop: "{{ groups['prepare_hosts'] | default([]) }}" + when: hostvars[item].unreachable | default(false) + run_once: true diff --git a/ansible/roles/zeam/defaults/main.yml b/ansible/roles/zeam/defaults/main.yml index 7916164..8b87474 100644 --- a/ansible/roles/zeam/defaults/main.yml +++ b/ansible/roles/zeam/defaults/main.yml @@ -15,20 +15,16 @@ zeam_global_flags: "" zeam_db_backend: "lmdb" # Chain-worker thread routing (zeam #803 slice c-2b/c-2c). -# Default "on": prod path post-c-2b. Matches the zeam compiled-in -# default (post-PR #830) and the client-cmds/zeam-cmd.sh script -# default (ZEAM_CHAIN_WORKER:-on). +# Default empty: omit --chain-worker on the CLI so zeam uses its +# compiled-in default (enabled post-PR #830). Do NOT pass "on" — zeam's +# bool flag does not take on/off values and "on" breaks parsing of +# subsequent flags such as --rayon-threads. # -# Override to "off" via -e zeam_chain_worker=off (or in inventory -# group_vars) to flip back to the legacy synchronous path — acts as -# the runtime kill-switch without needing a zeam rebuild/redeploy. -# -# Set to "" (empty string) to suppress the flag entirely, e.g. -# when running against a zeam build that does not recognise -# --chain-worker (v0.4.14 or earlier). +# Set to "off" or "false" to emit `--chain-worker false` (legacy sync +# kill-switch). Empty/"on" both mean enabled with no flag passed. # # REQUIRES: blockblaz/zeam:devnet4 >= v0.4.15. -zeam_chain_worker: "on" +zeam_chain_worker: "" # These should be passed from the playbook node_name: "" diff --git a/ansible/roles/zeam/tasks/main.yml b/ansible/roles/zeam/tasks/main.yml index 02f54d4..44a2e63 100644 --- a/ansible/roles/zeam/tasks/main.yml +++ b/ansible/roles/zeam/tasks/main.yml @@ -93,17 +93,20 @@ # zeam_rayon_threads applies to both roles, otherwise the flag is omitted # entirely so zeam picks its compiled-in auto-split (zeam #903 / #899). # -# Both vars are typically set in group_vars/all.yml. Default unset means no -# flag is passed — required for pre-#903 images, which would reject it. +# IMPORTANT: an empty Jinja branch must become "" not None — Ansible's +# default('') filter does NOT coalesce None (only undefined), and None +# stringifies to "None" in the docker command → zeam CLI InvalidCharacter. - name: Resolve effective rayon-threads override set_fact: - zeam_effective_rayon_threads: >- + zeam_effective_rayon_threads: "{{ _zeam_rayon_raw | default('', true) | string | trim }}" + vars: + _zeam_rayon_raw: >- {%- if (zeam_is_aggregator | default('false')) == 'true' - and (zeam_rayon_threads_aggregator | default('') | string | length) > 0 -%} + and (zeam_rayon_threads_aggregator | default('') | string | trim | length) > 0 -%} {{ zeam_rayon_threads_aggregator }} - {%- elif (zeam_rayon_threads | default('') | string | length) > 0 -%} + {%- elif (zeam_rayon_threads | default('') | string | trim | length) > 0 -%} {{ zeam_rayon_threads }} - {%- else -%}{%- endif -%} + {%- endif -%} - name: Ensure node key file exists stat: @@ -166,8 +169,8 @@ {{ ('--aggregate-subnet-ids ' + zeam_aggregate_subnet_ids) if (zeam_is_aggregator | default('false')) == 'true' and (zeam_aggregate_subnet_ids | default('') | length > 0) else '' }} {{ ('--checkpoint-sync-url ' + checkpoint_sync_url) if (checkpoint_sync_url is defined and checkpoint_sync_url | length > 0) else '' }} {{ ('--db-backend ' + zeam_db_backend) if (zeam_db_backend | default('') | length > 0) else '' }} - {{ ('--chain-worker ' + zeam_chain_worker) if (zeam_chain_worker | default('') | length > 0) else '' }} - {{ ('--rayon-threads ' + (zeam_effective_rayon_threads | trim)) if (zeam_effective_rayon_threads | default('') | trim | length > 0) else '' }} + {{ '--chain-worker false' if (zeam_chain_worker | default('') | lower in ['off', 'false']) else '' }} + {{ ('--rayon-threads ' + zeam_effective_rayon_threads) if (zeam_effective_rayon_threads | length > 0) else '' }} register: zeam_container_result changed_when: zeam_container_result.rc == 0 diff --git a/client-cmds/zeam-cmd.sh b/client-cmds/zeam-cmd.sh index c8b85b4..44b9389 100644 --- a/client-cmds/zeam-cmd.sh +++ b/client-cmds/zeam-cmd.sh @@ -55,34 +55,27 @@ db_backend_flag="--db-backend ${zeam_db_backend}" # (typical=1, never >16), and `lean_chain_queue_dropped_total` (should # stay 0 under nominal load). # -# Default `on`: matches the zeam compiled-in default (post-PR #830). -# Operators can override via `export ZEAM_CHAIN_WORKER=off` to flip -# back to the legacy synchronous path (kill-switch) without a -# rebuild/redeploy of zeam itself. +# Default empty/on: omit --chain-worker so zeam uses its compiled-in +# default (enabled post-PR #830). Do NOT pass `--chain-worker on` — +# zeam's bool flag does not take on/off values and "on" breaks parsing +# of subsequent flags such as --rayon-threads. +# +# Override via `export ZEAM_CHAIN_WORKER=off` to emit +# `--chain-worker false` (legacy synchronous kill-switch). # -# REQUIRES: a zeam build with chain-worker support, i.e. -# `blockblaz/zeam:devnet4` >= v0.4.15. Older images (v0.4.14 with the -# broken bool CLI shape, or v0.4.13 / pre-c-1) do not recognise -# `--chain-worker on` and will fail to start. If running against an -# older image set `export ZEAM_CHAIN_WORKER=` (empty) to suppress -# the flag entirely. # Note `${VAR-default}` (no colon) so an explicitly-empty -# `ZEAM_CHAIN_WORKER=` suppresses the flag entirely — the colon form -# would also overwrite the empty value with `on`, leaving no way to -# bypass for older zeam builds. +# `ZEAM_CHAIN_WORKER=` suppresses the flag entirely. zeam_chain_worker="${ZEAM_CHAIN_WORKER-on}" chain_worker_flag="" case "$zeam_chain_worker" in - on|off) - chain_worker_flag="--chain-worker $zeam_chain_worker" + on|"") + # Enabled (compiled default); omit flag. ;; - "") - # Explicitly empty — no flag, zeam takes its compiled-in - # default (`.on` post-PR #830). Use this against zeam - # builds that do not recognise `--chain-worker` at all. + off|false) + chain_worker_flag="--chain-worker false" ;; *) - echo "WARN(zeam-cmd): ZEAM_CHAIN_WORKER='$zeam_chain_worker' is not 'on' or 'off' or empty; ignoring (no --chain-worker flag passed)" >&2 + echo "WARN(zeam-cmd): ZEAM_CHAIN_WORKER='$zeam_chain_worker' is not 'on', 'off', 'false', or empty; ignoring (no --chain-worker flag passed)" >&2 ;; esac diff --git a/parse-env.sh b/parse-env.sh index 8a833d7..6d5608e 100755 --- a/parse-env.sh +++ b/parse-env.sh @@ -84,6 +84,10 @@ while [[ $# -gt 0 ]]; do shift # past argument shift # past value ;; + --randomize) + randomizeAggregators=true + shift + ;; --checkpoint-sync-url) checkpointSyncUrl="$2" shift @@ -190,7 +194,8 @@ echo "cleanData = $cleanData" echo "popupTerminal = $popupTerminal" echo "dockerTag = ${dockerTag:-latest}" echo "enableMetrics = $enableMetrics" -echo "aggregatorNode = ${aggregatorNode:-}" +echo "aggregatorNode = ${aggregatorNode:-}" +echo "randomizeAggregators = ${randomizeAggregators:-false}" echo "coreDumps = ${coreDumps:-disabled}" echo "checkpointSyncUrl = ${checkpointSyncUrl:-}" echo "restartClient = ${restartClient:-}" diff --git a/run-ansible.sh b/run-ansible.sh index 819ac79..e98d3de 100755 --- a/run-ansible.sh +++ b/run-ansible.sh @@ -266,7 +266,8 @@ if [ $EXIT_CODE -eq 0 ]; then elif [ "$action" == "observability" ]; then echo "✅ Observability stack deployed on all hosts!${_dry_tag}" elif [ "$action" == "stop-all-containers" ]; then - echo "✅ Stopped all non-observability containers on validator hosts!${_dry_tag}" + echo "✅ Stopped all non-observability containers on reachable validator hosts!${_dry_tag}" + echo " (Unreachable hosts are skipped; see playbook summary for SSH failures.)" else echo "✅ Ansible deployment completed successfully!${_dry_tag}" fi From fd5bc0e858e2b53aacbb175eff01fb0b7bd3bd24 Mon Sep 17 00:00:00 2001 From: ch4r10t33r Date: Mon, 25 May 2026 13:53:57 +0100 Subject: [PATCH 20/20] zeam: deploy local image and rayon defaults for devnet Use 0xpartha/zeam:local, set non-aggregator rayon to 6 on 8-vCPU hosts, and let 16-vCPU aggregators auto-tune (cpu_count - 4) when no override is set. --- ansible/inventory/group_vars/all.yml | 21 ++++++++------- ansible/roles/zeam/defaults/main.yml | 7 ++++- ansible/roles/zeam/tasks/main.yml | 11 +++++--- client-cmds/zeam-cmd.sh | 39 ++++++++++++++++------------ 4 files changed, 47 insertions(+), 31 deletions(-) diff --git a/ansible/inventory/group_vars/all.yml b/ansible/inventory/group_vars/all.yml index 3adc5b6..a763aec 100644 --- a/ansible/inventory/group_vars/all.yml +++ b/ansible/inventory/group_vars/all.yml @@ -48,21 +48,22 @@ docker_memory_limits: peam: "4g" # zeam --rayon-threads override for the multisig (XMSS) aggregate prover -# (zeam #903 / #899). Mirror of the ZEAM_RAYON_THREADS_AGGREGATOR / -# ZEAM_RAYON_THREADS env vars in client-cmds/zeam-cmd.sh. +# (zeam #903 / #899). Mirror of the ZEAM_RAYON_THREADS_* env vars in +# client-cmds/zeam-cmd.sh. # # zeam_rayon_threads_aggregator wins for aggregators; zeam_rayon_threads -# applies to both roles otherwise. Both empty (the default) means no flag -# is passed and zeam picks its compiled-in auto-split. +# applies to both roles when set; non-aggregators otherwise use +# zeam_rayon_threads_non_aggregator (default 6 on 8-vCPU devnet hosts). # -# 16-vCPU recommended starting point: 12 (= cpu_count - 4 reserved system -# threads). Do not exceed cpu_count - 4, or libxev/libp2p/api/metrics -# threads start to starve — watch -# `zeam_fork_choice_tick_interval_duration_seconds` p99. +# 16-vCPU recommended aggregator starting point: 12 (= cpu_count - 4 reserved +# system threads). 8-vCPU non-aggregators: 6 (same cpu_count - 4 rule). Do not +# exceed cpu_count - 4, or libxev/libp2p/api/metrics threads start to starve — +# watch `zeam_fork_choice_tick_interval_duration_seconds` p99. # # REQUIRES: a zeam image built from PR #903 or later. Older images will -# refuse `--rayon-threads` and fail to start; leave both empty in that case. -zeam_rayon_threads_aggregator: "12" +# refuse `--rayon-threads` and fail to start; leave all three empty in that case. +zeam_rayon_threads_aggregator: "" +zeam_rayon_threads_non_aggregator: "6" zeam_rayon_threads: "" # Node deployment mode: 'docker' or 'binary' diff --git a/ansible/roles/zeam/defaults/main.yml b/ansible/roles/zeam/defaults/main.yml index 8b87474..df83fdc 100644 --- a/ansible/roles/zeam/defaults/main.yml +++ b/ansible/roles/zeam/defaults/main.yml @@ -3,7 +3,7 @@ # Note: These are fallback defaults. Actual values are extracted from client-cmds/zeam-cmd.sh # in the tasks/main.yml file. These defaults are used if extraction fails. -zeam_docker_image: "blockblaz/zeam:devnet4" +zeam_docker_image: "0xpartha/zeam:local" zeam_binary_path: "{{ playbook_dir }}/../zig-out/bin/zeam" deployment_mode: docker # docker or binary @@ -26,6 +26,11 @@ zeam_db_backend: "lmdb" # REQUIRES: blockblaz/zeam:devnet4 >= v0.4.15. zeam_chain_worker: "" +# Rayon thread defaults (keep in sync with client-cmds/zeam-cmd.sh). +zeam_rayon_threads_aggregator: "" +zeam_rayon_threads_non_aggregator: "6" +zeam_rayon_threads: "" + # These should be passed from the playbook node_name: "" genesis_dir: "" diff --git a/ansible/roles/zeam/tasks/main.yml b/ansible/roles/zeam/tasks/main.yml index 44a2e63..7952f96 100644 --- a/ansible/roles/zeam/tasks/main.yml +++ b/ansible/roles/zeam/tasks/main.yml @@ -31,7 +31,7 @@ - name: Set docker image and deployment mode from client-cmd.sh set_fact: - zeam_docker_image: "{{ zeam_docker_image_raw.stdout | trim | default('blockblaz/zeam:devnet4') }}" + zeam_docker_image: "{{ zeam_docker_image_raw.stdout | trim | default('0xpartha/zeam:local') }}" deployment_mode: "{{ zeam_deployment_mode_raw.stdout | trim | default('docker') }}" delegate_to: localhost run_once: true @@ -89,9 +89,9 @@ when: (zeam_is_aggregator | default('false')) != 'true' # Resolve --rayon-threads override for this node. Mirrors the zeam-cmd.sh -# precedence: zeam_rayon_threads_aggregator wins for aggregators, otherwise -# zeam_rayon_threads applies to both roles, otherwise the flag is omitted -# entirely so zeam picks its compiled-in auto-split (zeam #903 / #899). +# precedence: zeam_rayon_threads_aggregator wins for aggregators; +# zeam_rayon_threads applies to both roles when set; non-aggregators otherwise +# use zeam_rayon_threads_non_aggregator (default 6). # # IMPORTANT: an empty Jinja branch must become "" not None — Ansible's # default('') filter does NOT coalesce None (only undefined), and None @@ -106,6 +106,9 @@ {{ zeam_rayon_threads_aggregator }} {%- elif (zeam_rayon_threads | default('') | string | trim | length) > 0 -%} {{ zeam_rayon_threads }} + {%- elif (zeam_is_aggregator | default('false')) != 'true' + and (zeam_rayon_threads_non_aggregator | default('') | string | trim | length) > 0 -%} + {{ zeam_rayon_threads_non_aggregator }} {%- endif -%} - name: Ensure node key file exists diff --git a/client-cmds/zeam-cmd.sh b/client-cmds/zeam-cmd.sh index 44b9389..40469b8 100644 --- a/client-cmds/zeam-cmd.sh +++ b/client-cmds/zeam-cmd.sh @@ -81,31 +81,38 @@ esac # Rayon worker count for the multisig (XMSS) aggregate prover (zeam #903 / #899). # -# Default unset → zeam picks an auto-split that gives roughly half of the -# post-system-thread CPU budget to rayon and half to its Zig worker pool. That -# split is fine for non-aggregators (which mostly verify, also via rayon-from- -# Zig-workers) but underuses CPU on CPU-rich aggregators where the produce-path -# FFI is the per-slot bottleneck. +# Aggregators use zeam auto-tune (cpu_count - 4 reserved system threads) when +# ZEAM_RAYON_THREADS_AGGREGATOR is unset. Non-aggregators default to 6 so +# block/attestation XMSS verification (onBlock verify_signatures) gets enough +# rayon workers on typical 8-vCPU devnet hosts. # -# Aggregators default to 12 rayon threads; non-aggregators stay on zeam's -# auto-split unless overridden: -# - ZEAM_RAYON_THREADS_AGGREGATOR # aggregator override (default 12) -# - ZEAM_RAYON_THREADS # uniform override for both roles -# For aggregators, ZEAM_RAYON_THREADS_AGGREGATOR wins when set; otherwise 12. +# Override env vars (client-cmds / ansible group_vars): +# - ZEAM_RAYON_THREADS_AGGREGATOR # optional aggregator override +# - ZEAM_RAYON_THREADS_NON_AGGREGATOR # non-aggregator default (6) +# - ZEAM_RAYON_THREADS # uniform override for both roles # -# Sizing guidance for a 16-vCPU host: 12 is the recommended starting point -# (cpu_count - 4 reserved system threads: libxev/libp2p/api/metrics). Do not -# exceed cpu_count - 4 or those reserved threads start to starve, surfacing as +# For aggregators, ZEAM_RAYON_THREADS_AGGREGATOR wins when set; otherwise zeam +# auto-tune. For non-aggregators, ZEAM_RAYON_THREADS wins when set; otherwise +# ZEAM_RAYON_THREADS_NON_AGGREGATOR (6). +# +# Sizing guidance for a 16-vCPU host: 12 is the recommended aggregator starting +# point (cpu_count - 4 reserved system threads: libxev/libp2p/api/metrics). For +# 8-vCPU non-aggregators: 6 (= cpu_count - 4, same budget rule). Do not exceed +# cpu_count - 4 or those reserved threads start to starve, surfacing as # `zeam_fork_choice_tick_interval_duration_seconds` p99 climbing. # # REQUIRES: a zeam build with PR #903 merged plus a docker image cut from it. # Older images do not recognise `--rayon-threads` and will fail to start. Leave -# both env vars unset to suppress the flag entirely for pre-#903 images. +# all three env vars unset to suppress the flag entirely for pre-#903 images. rayon_threads_flag="" if [ "$isAggregator" == "true" ]; then - rayon_threads_flag="--rayon-threads ${ZEAM_RAYON_THREADS_AGGREGATOR:-12}" + if [ -n "${ZEAM_RAYON_THREADS_AGGREGATOR:-}" ]; then + rayon_threads_flag="--rayon-threads ${ZEAM_RAYON_THREADS_AGGREGATOR}" + fi elif [ -n "${ZEAM_RAYON_THREADS:-}" ]; then rayon_threads_flag="--rayon-threads $ZEAM_RAYON_THREADS" +else + rayon_threads_flag="--rayon-threads ${ZEAM_RAYON_THREADS_NON_AGGREGATOR:-6}" fi node_binary="$scriptDir/../zig-out/bin/zeam $zeam_global_flags node \ @@ -124,7 +131,7 @@ node_binary="$scriptDir/../zig-out/bin/zeam $zeam_global_flags node \ $chain_worker_flag \ $rayon_threads_flag" -node_docker="--security-opt seccomp=unconfined blockblaz/zeam:devnet4 $zeam_global_flags node \ +node_docker="--security-opt seccomp=unconfined 0xpartha/zeam:local $zeam_global_flags node \ --custom-genesis /config \ --validator-config $validatorConfig \ --data-dir /data \