From 6d49e49fe14e5bdf69ebd9e3d56692b198d9222c Mon Sep 17 00:00:00 2001 From: Patrick_Audley Date: Mon, 2 Mar 2026 15:14:44 -0700 Subject: [PATCH 1/2] feat(rabbitmq): enable core plugins, production config, and Erlang VM tuning Enable eight tier-1 RabbitMQ plugins required by the cognitive architecture and production operations: consistent_hash_exchange, stream, stream_management, shovel, shovel_management, event_exchange, tracing, and top. Add rabbitmq.conf with production-tuned defaults: absolute memory watermark (512 MiB) with allocated calculation strategy for accurate container reporting, disk free limit, TCP keepalives with Nagle disabled, 128 channel max, 15 s stats emission interval, classic queue default, stream listener on port 5552, and guest loopback restriction. Configure Erlang VM scheduler optimizations via RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: disable speculative busy-waiting (+sbwt/+sbwtdcpu/+sbwtdio none) and bind scheduler threads to CPU topology (+stbt ts) to reduce context switching. Wire stream protocol port 5552 through docker-compose and .env.example, add container mem_limit/cpus knobs, mount rabbitmq.conf read-only, and register rabbitmq-plugins manage.sh command. --- .env.example | 11 +++++ docker-compose.yml | 4 ++ docker/rabbitmq/Dockerfile | 26 ++++++++++- rabbitmq/rabbitmq.conf | 92 ++++++++++++++++++++++++++++++++++++++ scripts/lib/maintenance.sh | 2 +- scripts/lib/rabbitmq.sh | 7 +++ scripts/manage.sh | 4 ++ 7 files changed, 144 insertions(+), 2 deletions(-) create mode 100644 rabbitmq/rabbitmq.conf diff --git a/.env.example b/.env.example index f9dec25..7841570 100644 --- a/.env.example +++ b/.env.example @@ -264,6 +264,8 @@ RABBITMQ_PORT=5672 RABBITMQ_HOST_PORT=5672 RABBITMQ_MANAGEMENT_PORT=15672 RABBITMQ_MANAGEMENT_HOST_PORT=15672 +RABBITMQ_STREAM_PORT=5552 +RABBITMQ_STREAM_HOST_PORT=5552 RABBITMQ_DEFAULT_USER=coredata RABBITMQ_DEFAULT_PASS_FILE=./secrets/rabbitmq_default_pass RABBITMQ_ERLANG_COOKIE_FILE=./secrets/rabbitmq_erlang_cookie @@ -271,6 +273,15 @@ RABBITMQ_DATA_MOUNT_PATH=/var/lib/rabbitmq # Pre-built RabbitMQ image uses UID 100, GID 101 (the rabbitmq user baked into the image). RABBITMQ_UID=100 RABBITMQ_GID=101 +# Container resource limits (0 = unlimited). +RABBITMQ_MEMORY_LIMIT=0 +RABBITMQ_CPU_LIMIT=0.0 +# Erlang VM tuning flags passed via RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS. +# +sbwt none — disable speculative scheduler busy-waiting (saves CPU) +# +sbwtdcpu none — disable dirty-CPU scheduler busy-waiting +# +sbwtdio none — disable dirty-IO scheduler busy-waiting +# +stbt ts — bind scheduler threads to topology (reduces context switches) +RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS=+sbwt none +sbwtdcpu none +sbwtdio none +stbt ts # Time zone for containers TZ=UTC diff --git a/docker-compose.yml b/docker-compose.yml index fc04f80..0e38732 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -356,8 +356,11 @@ services: TZ: ${TZ} entrypoint: ["/opt/core_data/bin/rabbitmq-entrypoint.sh"] command: ["rabbitmq-server"] + mem_limit: ${RABBITMQ_MEMORY_LIMIT:-0} + cpus: ${RABBITMQ_CPU_LIMIT:-0.0} volumes: - ./data/rabbitmq_data:${RABBITMQ_DATA_MOUNT_PATH:-/var/lib/rabbitmq} + - ./rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro - ./secrets/rabbitmq_default_pass:/run/secrets/rabbitmq_default_pass:ro - ./secrets/rabbitmq_erlang_cookie:/run/secrets/rabbitmq_erlang_cookie:ro networks: @@ -365,6 +368,7 @@ services: ports: - "${RABBITMQ_HOST_PORT}:${RABBITMQ_PORT}" - "${RABBITMQ_MANAGEMENT_HOST_PORT}:${RABBITMQ_MANAGEMENT_PORT}" + - "${RABBITMQ_STREAM_HOST_PORT:-5552}:${RABBITMQ_STREAM_PORT:-5552}" healthcheck: test: ["CMD-SHELL", "/opt/core_data/bin/rabbitmq-healthcheck.sh"] interval: 10s diff --git a/docker/rabbitmq/Dockerfile b/docker/rabbitmq/Dockerfile index a422cfa..54c605d 100644 --- a/docker/rabbitmq/Dockerfile +++ b/docker/rabbitmq/Dockerfile @@ -2,7 +2,31 @@ FROM rabbitmq:4.2-management-alpine LABEL org.opencontainers.image.source="https://github.com/paudley/core_data" \ - org.opencontainers.image.description="Core Data RabbitMQ with hardened entrypoint" + org.opencontainers.image.description="Core Data RabbitMQ with hardened entrypoint and core plugins" + +# Enable tier-1 core plugins required by the cognitive architecture and +# production operations tooling. The management image already enables +# rabbitmq_management, rabbitmq_management_agent, rabbitmq_web_dispatch, +# and rabbitmq_prometheus. +# +# Core requirements: +# consistent_hash_exchange – session affinity for LLM prefix-cache +# stream / stream_management – append-only Cognitive_Stream, Dreaming offsets +# +# Production operations: +# shovel / shovel_management – queue draining, dead-letter reprocessing +# event_exchange – broker events as AMQP messages (observability) +# tracing – message-level tracing via management UI +# top – per-process resource monitoring +RUN rabbitmq-plugins enable --offline \ + rabbitmq_consistent_hash_exchange \ + rabbitmq_stream \ + rabbitmq_stream_management \ + rabbitmq_shovel \ + rabbitmq_shovel_management \ + rabbitmq_event_exchange \ + rabbitmq_tracing \ + rabbitmq_top # Add secrets group and add rabbitmq user to it for reading shared secrets ARG SECRETS_GID=65532 diff --git a/rabbitmq/rabbitmq.conf b/rabbitmq/rabbitmq.conf new file mode 100644 index 0000000..09b1248 --- /dev/null +++ b/rabbitmq/rabbitmq.conf @@ -0,0 +1,92 @@ +# SPDX-FileCopyrightText: 2025 Blackcat Informatics® Inc. +# SPDX-License-Identifier: MIT +# +# Core Data — RabbitMQ Production Configuration +# +# This file is mounted read-only into the container at +# /etc/rabbitmq/rabbitmq.conf. Edit values here directly; +# RabbitMQ's sysctl-format config does not support environment +# variable substitution. +# +# Reference: https://www.rabbitmq.com/docs/configure +# Reference: https://www.rabbitmq.com/docs/production-checklist + +# ────────────────────────────────────────────── +# Memory & flow control +# ────────────────────────────────────────────── + +# In containerized environments use an absolute limit rather than a +# relative watermark so the Erlang VM does not mis-detect available +# memory. Adjust to match your container's mem_limit. +# 512 MiB is suitable for a lightweight single-node broker. +vm_memory_high_watermark.absolute = 512MiB + +# Begin paging messages to disk when memory reaches 50 % of the +# watermark. At 512 MiB this triggers at ~256 MiB. +vm_memory_high_watermark_paging_ratio = 0.5 + +# Use Erlang allocator stats for memory calculation — more accurate +# than the default RSS-based strategy inside containers. +vm_memory_calculation_strategy = allocated + +# ────────────────────────────────────────────── +# Disk free space +# ────────────────────────────────────────────── + +# Minimum free disk before the broker blocks publishers. Should +# roughly match the memory watermark to ensure safe page-out. +disk_free_limit.absolute = 512MiB + +# ────────────────────────────────────────────── +# Networking & connections +# ────────────────────────────────────────────── + +# Heartbeat — detect dead TCP connections. 60 s is a safe default +# that avoids false positives under transient load. +heartbeat = 60 + +# Maximum channels per connection. Prevents a single client from +# exhausting broker resources. +channel_max = 128 + +# TCP listen backlog — how many pending connections the kernel queues +# before refusing new ones. +tcp_listen_options.backlog = 256 + +# Enable TCP keepalives so the OS detects half-open connections. +tcp_listen_options.keepalive = true + +# Disable Nagle's algorithm for lower message latency. +tcp_listen_options.nodelay = true + +# ────────────────────────────────────────────── +# Queue & message defaults +# ────────────────────────────────────────────── + +# Default queue type for new declarations that do not specify one. +# Classic queues are the correct choice for a single-node deployment +# (quorum queues add Raft overhead with no replication benefit). +default_queue_type = classic + +# Consumer delivery acknowledgement timeout (ms). Consumers that +# hold messages longer than 30 minutes without ack are disconnected. +consumer_timeout = 1800000 + +# ────────────────────────────────────────────── +# Management & monitoring +# ────────────────────────────────────────────── + +# Increase the statistics emission interval from 5 s to 15 s. +# Reduces periodic overhead on connections, channels and queues +# while remaining sufficient for Prometheus scrape intervals. +collect_statistics_interval = 15000 + +# Disable guest user login from remote hosts (security hardening). +loopback_users.guest = true + +# ────────────────────────────────────────────── +# Streams +# ────────────────────────────────────────────── + +# Stream protocol listener port. +stream.listeners.tcp.1 = 5552 diff --git a/scripts/lib/maintenance.sh b/scripts/lib/maintenance.sh index 03ca4ee..7167b2b 100644 --- a/scripts/lib/maintenance.sh +++ b/scripts/lib/maintenance.sh @@ -56,7 +56,7 @@ cmd_pgbadger_report() { local cmd=(pgbadger --quiet --format csv --jobs "$jobs" --outfile "$output") [[ -n $since ]] && cmd+=(--begin "$since") cmd+=(/var/lib/postgresql/data/log/postgresql-*.csv) - compose_exec bash -lc "${cmd[@]}" + compose_exec bash -lc "${cmd[*]}" echo "pgBadger report written to ${output}" >&2 } diff --git a/scripts/lib/rabbitmq.sh b/scripts/lib/rabbitmq.sh index e90267f..a1c479e 100644 --- a/scripts/lib/rabbitmq.sh +++ b/scripts/lib/rabbitmq.sh @@ -8,6 +8,7 @@ RABBITMQ_SERVICE_NAME=${RABBITMQ_SERVICE_NAME:-rabbitmq} RABBITMQ_HOST=${RABBITMQ_HOST:-rabbitmq} RABBITMQ_PORT=${RABBITMQ_PORT:-5672} RABBITMQ_MANAGEMENT_PORT=${RABBITMQ_MANAGEMENT_PORT:-15672} +RABBITMQ_STREAM_PORT=${RABBITMQ_STREAM_PORT:-5552} ensure_rabbitmq_service() { if ! compose_has_service "${RABBITMQ_SERVICE_NAME}"; then @@ -86,6 +87,12 @@ USAGE echo "[rabbitmq] Definitions written to ${output_path}" >&2 } +cmd_rabbitmq_plugins() { + ensure_env + ensure_rabbitmq_service + rabbitmq_exec rabbitmq-plugins list "$@" +} + cmd_rabbitmq_overview() { ensure_env ensure_rabbitmq_service diff --git a/scripts/manage.sh b/scripts/manage.sh index 150b25e..c2963f6 100755 --- a/scripts/manage.sh +++ b/scripts/manage.sh @@ -241,6 +241,7 @@ Cache, messaging, pooling rabbitmq-ctl [args] Run rabbitmqctl inside the RabbitMQ container. rabbitmq-diagnostics [args] Run rabbitmq-diagnostics inside RabbitMQ. rabbitmq-export [--output PATH] Export RabbitMQ definitions to host (JSON). + rabbitmq-plugins [args] List or manage RabbitMQ plugins. rabbitmq-overview Show rabbitmq-diagnostics status summary. pgbouncer-stats SHOW STATS via PgBouncer admin console. pgbouncer-pools SHOW POOLS via PgBouncer admin console. @@ -1507,6 +1508,9 @@ rabbitmq-diagnostics) rabbitmq-export) cmd_rabbitmq_export "$@" ;; +rabbitmq-plugins) + cmd_rabbitmq_plugins "$@" + ;; rabbitmq-overview) cmd_rabbitmq_overview "$@" ;; From 368fc5445b43853c83a93433c4e8ddedd59d4b75 Mon Sep 17 00:00:00 2001 From: Patrick_Audley Date: Mon, 2 Mar 2026 15:30:39 -0700 Subject: [PATCH 2/2] fix(rabbitmq): quote Erlang VM args in .env.example for shell sourcing CI sources .env directly, so unquoted values with spaces like "+sbwt none" cause the shell to interpret "none" as a command. Wrap the RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS value in double quotes so both `source .env` and Docker Compose handle it correctly. --- .env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 7841570..bad0f70 100644 --- a/.env.example +++ b/.env.example @@ -281,7 +281,7 @@ RABBITMQ_CPU_LIMIT=0.0 # +sbwtdcpu none — disable dirty-CPU scheduler busy-waiting # +sbwtdio none — disable dirty-IO scheduler busy-waiting # +stbt ts — bind scheduler threads to topology (reduces context switches) -RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS=+sbwt none +sbwtdcpu none +sbwtdio none +stbt ts +RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS="+sbwt none +sbwtdcpu none +sbwtdio none +stbt ts" # Time zone for containers TZ=UTC