From dab1fc93d393b335f6c0714408266943514d602a Mon Sep 17 00:00:00 2001 From: Mina Abadir Date: Sat, 17 Oct 2026 10:22:31 -0400 Subject: [PATCH] fix(eks-addons): let Loki return query results larger than 4MiB In single-binary mode the querier hands each result back to the query frontend over gRPC, and Loki's server kept the 4MiB message default. A filter query over a busy workload's logs failed with "received message larger than max (31010467 vs. 4194304)". Raise the server's send and receive limits to 100MiB, the querier's own client limit. --- README.md | 2 +- compute/eks/addons/loki.tf | 8 ++++++++ compute/eks/addons/rvn-eks-addons-definition.yml | 4 ++-- compute/eks/addons/tests/logs.tftest.hcl | 11 +++++++++++ 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 104b4c6f..aeaadb51 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ sync by `node tools/ravion-modules/dist/src/cli.js readme` (enforced in CI, and | `rvn-ecs-web` | ECS Web Service | v2.3.0 | `compute/ecs_service/` | | `rvn-ecs-worker` | ECS Worker | v2.2.2 | `compute/ecs_service/` | | `rvn-efs` | EFS File System | v1.1.0 | `storage/efs/` | -| `rvn-eks-addons` | EKS Add-ons | v0.23.0 | `compute/eks/addons/` | +| `rvn-eks-addons` | EKS Add-ons | v0.23.2 | `compute/eks/addons/` | | `rvn-eks-chart` | EKS Helm Chart | v0.3.0 | `compute/eks_service/` | | `rvn-eks-cluster` | EKS Cluster | v0.9.0 | `compute/eks/` | | `rvn-eks-web` | EKS Web Service | v1.10.0 | `compute/eks_service/` | diff --git a/compute/eks/addons/loki.tf b/compute/eks/addons/loki.tf index 708204e3..ff9524d4 100644 --- a/compute/eks/addons/loki.tf +++ b/compute/eks/addons/loki.tf @@ -218,6 +218,14 @@ resource "helm_release" "loki" { server = { http_listen_port = 3100 + + # In single-binary mode the querier returns each result to the + # query frontend over gRPC, and a filter query over a busy + # workload's logs easily passes the 4MiB default, failing with + # "received message larger than max". 100MiB matches the querier's + # own client limit. + grpc_server_max_recv_msg_size = 104857600 + grpc_server_max_send_msg_size = 104857600 } # A schema is the one thing the chart refuses to guess, because diff --git a/compute/eks/addons/rvn-eks-addons-definition.yml b/compute/eks/addons/rvn-eks-addons-definition.yml index 513d6d17..44b918f6 100644 --- a/compute/eks/addons/rvn-eks-addons-definition.yml +++ b/compute/eks/addons/rvn-eks-addons-definition.yml @@ -4,8 +4,8 @@ definition: name: EKS Add-ons description: Selectable add-ons for an existing EKS cluster. release: - version: 0.23.0 - description: "Mimir ingesters drop series from pods that have gone, compacting early once they hold half their memory in series, so rollouts and scale-downs no longer fill them until they are killed for memory. Ingesters and store-gateways now refuse to share a node, as the three copies promise." + version: 0.23.2 + description: "Loki answers log searches whose results pass 4MiB. Its gRPC server kept the 4MiB message default, so a filter over a busy workload's logs failed with \"received message larger than max\"; it now accepts up to 100MiB, the querier's own limit." module: inputs: - id: cluster diff --git a/compute/eks/addons/tests/logs.tftest.hcl b/compute/eks/addons/tests/logs.tftest.hcl index 8ced4e90..8d31e772 100644 --- a/compute/eks/addons/tests/logs.tftest.hcl +++ b/compute/eks/addons/tests/logs.tftest.hcl @@ -298,6 +298,17 @@ run "loki_runs_as_a_single_binary_with_no_extras" { error_message = "The gateway, both memcached tiers, and the bundled MinIO must all be off" } + # In single-binary mode the querier hands each result back to the query + # frontend over gRPC. At the 4MiB default, a filter query over a busy + # workload's logs fails with "received message larger than max". + assert { + condition = alltrue([ + yamldecode(helm_release.loki[0].values[0]).loki.server.grpc_server_max_recv_msg_size == 104857600, + yamldecode(helm_release.loki[0].values[0]).loki.server.grpc_server_max_send_msg_size == 104857600, + ]) + error_message = "Loki's gRPC server must accept query results larger than the 4MiB default." + } + # Structured metadata is where `level` lives; without this the schema rejects # it and the whole label design collapses into index labels. assert {