From 86459c4c9c2a061070af0f283cbdcd82e690df9c Mon Sep 17 00:00:00 2001 From: Favourice01 Date: Tue, 20 Oct 2026 05:55:14 +0000 Subject: [PATCH] fix(infra): Redis TLS, ALB alarm dimensions, extra alarms, ECS autoscaling Closes #1773, #1774, #1776, #1777 --- .github/workflows/terraform.yml | 8 ++ infrastructure/README.md | 8 ++ .../terraform/environments/ci-validate.tfvars | 1 + .../terraform/environments/dev.tfvars | 1 + infrastructure/terraform/main.tf | 10 ++ infrastructure/terraform/modules/ecs/main.tf | 83 ++++++++++++ .../terraform/modules/monitoring/main.tf | 120 +++++++++++++++++- infrastructure/terraform/modules/rds/main.tf | 4 + .../terraform/modules/redis/main.tf | 6 +- .../terraform/tests/check_invariants.sh | 16 +++ infrastructure/terraform/variables.tf | 29 +++++ services/api/Cargo.toml | 2 +- 12 files changed, 284 insertions(+), 4 deletions(-) create mode 100755 infrastructure/terraform/tests/check_invariants.sh diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml index 418608fe..2dd0564a 100644 --- a/.github/workflows/terraform.yml +++ b/.github/workflows/terraform.yml @@ -53,6 +53,14 @@ jobs: if: steps.fmt.outcome == 'failure' run: exit 1 + terraform-invariants: + name: Terraform Invariant Checks + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - run: bash infrastructure/terraform/tests/check_invariants.sh + terraform-validate: name: Terraform Validate runs-on: ubuntu-latest diff --git a/infrastructure/README.md b/infrastructure/README.md index 82b5d49c..a52d5296 100644 --- a/infrastructure/README.md +++ b/infrastructure/README.md @@ -273,3 +273,11 @@ For infrastructure issues: - [Terraform Documentation](https://www.terraform.io/docs) - [AWS Provider Documentation](https://registry.terraform.io/providers/hashicorp/aws/latest/docs) - [PredictIQ Architecture](../docs/ARCHITECTURE.md) + +## Redis TLS, alarms and autoscaling + +- ElastiCache enforces in-transit encryption; the module emits a `rediss://` URL and the API's `redis` crate is built with `tokio-rustls-comp`. +- CloudWatch ALB alarms/dashboard use `aws_lb.main.arn_suffix` (passed into the monitoring module), not a wildcard. +- Alarms cover ECS CPU/memory/running tasks, ALB 5xx/latency/unhealthy hosts, RDS CPU/storage/connections and Redis memory/evictions/replication lag; each description links its runbook in `docs/runbooks/`. +- The ECS service autoscales on CPU and ALB request count per target between `api_min_count` and `api_max_count`. +- `infrastructure/terraform/tests/check_invariants.sh` (run in the Terraform CI workflow) guards these. diff --git a/infrastructure/terraform/environments/ci-validate.tfvars b/infrastructure/terraform/environments/ci-validate.tfvars index 65969cd5..6d07485a 100644 --- a/infrastructure/terraform/environments/ci-validate.tfvars +++ b/infrastructure/terraform/environments/ci-validate.tfvars @@ -17,5 +17,6 @@ redis_auth_token = "CiValidate!Redis#Placeholder2024XYZ" api_image_uri = "123456789012.dkr.ecr.us-east-1.amazonaws.com/predictiq:ci-validate" api_container_port = 8080 api_desired_count = 1 +api_min_count = 1 api_cpu = 256 api_memory = 512 diff --git a/infrastructure/terraform/environments/dev.tfvars b/infrastructure/terraform/environments/dev.tfvars index ab7f2368..17248872 100644 --- a/infrastructure/terraform/environments/dev.tfvars +++ b/infrastructure/terraform/environments/dev.tfvars @@ -7,5 +7,6 @@ backup_retention_days = 7 redis_node_type = "cache.t3.micro" redis_num_nodes = 1 api_desired_count = 1 +api_min_count = 1 api_cpu = 256 api_memory = 512 diff --git a/infrastructure/terraform/main.tf b/infrastructure/terraform/main.tf index 8a92a0e8..7119dbf7 100644 --- a/infrastructure/terraform/main.tf +++ b/infrastructure/terraform/main.tf @@ -116,6 +116,8 @@ module "ecs" { api_image_uri = var.api_image_uri api_container_port = var.api_container_port api_desired_count = var.api_desired_count + api_min_count = var.api_min_count + api_max_count = var.api_max_count api_cpu = var.api_cpu api_memory = var.api_memory db_host = module.rds.endpoint @@ -138,4 +140,12 @@ module "monitoring" { environment = var.environment ecs_cluster = module.ecs.cluster_name ecs_service = module.ecs.service_name + + alb_arn_suffix = module.ecs.alb_arn_suffix + target_group_arn_suffix = module.ecs.target_group_arn_suffix + rds_instance_id = module.rds.instance_id + redis_replication_group = module.redis.replication_group_id + api_desired_count = var.api_desired_count + rds_free_storage_bytes = var.rds_free_storage_alarm_bytes + alb_latency_seconds = var.alb_latency_alarm_seconds } diff --git a/infrastructure/terraform/modules/ecs/main.tf b/infrastructure/terraform/modules/ecs/main.tf index ed46a335..30f62866 100644 --- a/infrastructure/terraform/modules/ecs/main.tf +++ b/infrastructure/terraform/modules/ecs/main.tf @@ -26,6 +26,29 @@ variable "api_desired_count" { type = number } +variable "api_min_count" { + type = number + default = 2 + description = "Minimum number of API tasks for autoscaling" +} + +variable "api_max_count" { + type = number + default = 10 + description = "Maximum number of API tasks for autoscaling" +} + +variable "autoscaling_cpu_target" { + type = number + default = 60 +} + +variable "autoscaling_requests_per_target" { + type = number + default = 1000 + description = "Target ALB requests per target per minute" +} + variable "api_cpu" { type = number } @@ -359,6 +382,11 @@ resource "aws_ecs_service" "api" { desired_count = var.api_desired_count launch_type = "FARGATE" + # Desired count is owned by the autoscaler after the first apply. + lifecycle { + ignore_changes = [desired_count] + } + # Zero-downtime rolling deploy: always keep 100 % capacity during updates, # allow up to 200 % so new tasks start before old ones are drained. deployment_minimum_healthy_percent = 100 @@ -622,6 +650,53 @@ resource "aws_iam_role" "ecs_task_role" { }) } +# ── Autoscaling ──────────────────────────────────────────────────────────────── + +resource "aws_appautoscaling_target" "api" { + service_namespace = "ecs" + scalable_dimension = "ecs:service:DesiredCount" + resource_id = "service/${aws_ecs_cluster.main.name}/${aws_ecs_service.api.name}" + min_capacity = var.api_min_count + max_capacity = var.api_max_count +} + +resource "aws_appautoscaling_policy" "api_cpu" { + name = "predictiq-${var.environment}-api-cpu-target" + policy_type = "TargetTrackingScaling" + service_namespace = aws_appautoscaling_target.api.service_namespace + scalable_dimension = aws_appautoscaling_target.api.scalable_dimension + resource_id = aws_appautoscaling_target.api.resource_id + + target_tracking_scaling_policy_configuration { + target_value = var.autoscaling_cpu_target + scale_in_cooldown = 300 + scale_out_cooldown = 60 + + predefined_metric_specification { + predefined_metric_type = "ECSServiceAverageCPUUtilization" + } + } +} + +resource "aws_appautoscaling_policy" "api_requests" { + name = "predictiq-${var.environment}-api-requests-target" + policy_type = "TargetTrackingScaling" + service_namespace = aws_appautoscaling_target.api.service_namespace + scalable_dimension = aws_appautoscaling_target.api.scalable_dimension + resource_id = aws_appautoscaling_target.api.resource_id + + target_tracking_scaling_policy_configuration { + target_value = var.autoscaling_requests_per_target + scale_in_cooldown = 300 + scale_out_cooldown = 60 + + predefined_metric_specification { + predefined_metric_type = "ALBRequestCountPerTarget" + resource_label = "${aws_lb.main.arn_suffix}/${aws_lb_target_group.api.arn_suffix}" + } + } +} + # ── Data sources / Outputs ───────────────────────────────────────────────────── data "aws_region" "current" {} @@ -634,6 +709,14 @@ output "service_name" { value = aws_ecs_service.api.name } +output "alb_arn_suffix" { + value = aws_lb.main.arn_suffix +} + +output "target_group_arn_suffix" { + value = aws_lb_target_group.api.arn_suffix +} + output "alb_dns_name" { value = aws_lb.main.dns_name } diff --git a/infrastructure/terraform/modules/monitoring/main.tf b/infrastructure/terraform/modules/monitoring/main.tf index 474c9e04..74a0eab4 100644 --- a/infrastructure/terraform/modules/monitoring/main.tf +++ b/infrastructure/terraform/modules/monitoring/main.tf @@ -10,6 +10,38 @@ variable "ecs_service" { type = string } +variable "alb_arn_suffix" { + type = string + description = "aws_lb.arn_suffix (app//) used as the LoadBalancer dimension" +} + +variable "target_group_arn_suffix" { + type = string +} + +variable "rds_instance_id" { + type = string +} + +variable "redis_replication_group" { + type = string +} + +variable "api_desired_count" { + type = number + default = 2 +} + +variable "rds_free_storage_bytes" { + type = number + default = 5368709120 +} + +variable "alb_latency_seconds" { + type = number + default = 1 +} + locals { common_tags = { Project = "predictiq" @@ -41,7 +73,7 @@ resource "aws_cloudwatch_dashboard" "main" { type = "metric" properties = { metrics = [ - ["AWS/ApplicationELB", "TargetResponseTime", "LoadBalancer", "app/predictiq-${var.environment}-alb/*"], + ["AWS/ApplicationELB", "TargetResponseTime", "LoadBalancer", var.alb_arn_suffix], [".", "RequestCount", ".", "."], [".", "HTTPCode_Target_5XX_Count", ".", "."] ] @@ -129,7 +161,7 @@ resource "aws_cloudwatch_metric_alarm" "alb_5xx" { alarm_actions = [aws_sns_topic.alerts.arn] dimensions = { - LoadBalancer = "app/predictiq-${var.environment}-alb/*" + LoadBalancer = var.alb_arn_suffix } tags = merge( @@ -140,6 +172,90 @@ resource "aws_cloudwatch_metric_alarm" "alb_5xx" { ) } +locals { + runbook_base = "docs/runbooks" + alarms = { + rds_cpu = { + namespace = "AWS/RDS", metric = "CPUUtilization", stat = "Average", op = "GreaterThanThreshold" + threshold = 80, periods = 3, dims = { DBInstanceIdentifier = var.rds_instance_id } + runbook = "slow-database-queries.md", missing = "missing" + } + rds_free_storage = { + namespace = "AWS/RDS", metric = "FreeStorageSpace", stat = "Minimum", op = "LessThanThreshold" + threshold = var.rds_free_storage_bytes, periods = 1, dims = { DBInstanceIdentifier = var.rds_instance_id } + runbook = "critical-database-queries.md", missing = "missing" + } + rds_connections = { + namespace = "AWS/RDS", metric = "DatabaseConnections", stat = "Average", op = "GreaterThanThreshold" + threshold = 80, periods = 3, dims = { DBInstanceIdentifier = var.rds_instance_id } + runbook = "high-db-connection-pool-utilization.md", missing = "missing" + } + redis_memory = { + namespace = "AWS/ElastiCache", metric = "DatabaseMemoryUsagePercentage", stat = "Average", op = "GreaterThanThreshold" + threshold = 80, periods = 3, dims = { ReplicationGroupId = var.redis_replication_group } + runbook = "redis-failure.md", missing = "missing" + } + redis_evictions = { + namespace = "AWS/ElastiCache", metric = "Evictions", stat = "Sum", op = "GreaterThanThreshold" + threshold = 100, periods = 2, dims = { ReplicationGroupId = var.redis_replication_group } + runbook = "low-cache-hit-rate.md", missing = "notBreaching" + } + redis_replication_lag = { + namespace = "AWS/ElastiCache", metric = "ReplicationLag", stat = "Maximum", op = "GreaterThanThreshold" + threshold = 30, periods = 3, dims = { ReplicationGroupId = var.redis_replication_group } + runbook = "redis-failure.md", missing = "notBreaching" + } + alb_unhealthy_hosts = { + namespace = "AWS/ApplicationELB", metric = "UnHealthyHostCount", stat = "Maximum", op = "GreaterThanThreshold" + threshold = 0, periods = 2, dims = { LoadBalancer = var.alb_arn_suffix, TargetGroup = var.target_group_arn_suffix } + runbook = "service-down.md", missing = "missing" + } + ecs_running_tasks = { + namespace = "ECS/ContainerInsights", metric = "RunningTaskCount", stat = "Minimum", op = "LessThanThreshold" + threshold = var.api_desired_count, periods = 2, dims = { ClusterName = var.ecs_cluster, ServiceName = var.ecs_service } + runbook = "ecs-task-crash-loop.md", missing = "breaching" + } + } +} + +resource "aws_cloudwatch_metric_alarm" "extra" { + for_each = local.alarms + + alarm_name = "predictiq-${var.environment}-${replace(each.key, "_", "-")}" + comparison_operator = each.value.op + evaluation_periods = each.value.periods + metric_name = each.value.metric + namespace = each.value.namespace + period = 300 + statistic = each.value.stat + threshold = each.value.threshold + treat_missing_data = each.value.missing + alarm_description = "${each.key} breached. Runbook: ${local.runbook_base}/${each.value.runbook}" + alarm_actions = [aws_sns_topic.alerts.arn] + dimensions = each.value.dims + + tags = merge(local.common_tags, { Name = "predictiq-${var.environment}-${each.key}" }) +} + +resource "aws_cloudwatch_metric_alarm" "alb_latency" { + alarm_name = "predictiq-${var.environment}-alb-latency-high" + comparison_operator = "GreaterThanThreshold" + evaluation_periods = 3 + metric_name = "TargetResponseTime" + namespace = "AWS/ApplicationELB" + period = 300 + extended_statistic = "p95" + threshold = var.alb_latency_seconds + alarm_description = "ALB p95 latency high. Runbook: ${local.runbook_base}/high-api-response-time.md" + alarm_actions = [aws_sns_topic.alerts.arn] + + dimensions = { + LoadBalancer = var.alb_arn_suffix + } + + tags = merge(local.common_tags, { Name = "predictiq-${var.environment}-alb-latency-high" }) +} + data "aws_region" "current" {} output "sns_topic_arn" { diff --git a/infrastructure/terraform/modules/rds/main.tf b/infrastructure/terraform/modules/rds/main.tf index 5073f09b..c57dfe9d 100644 --- a/infrastructure/terraform/modules/rds/main.tf +++ b/infrastructure/terraform/modules/rds/main.tf @@ -136,3 +136,7 @@ output "database_url" { value = "postgresql://${var.db_username}:${var.db_password}@${aws_db_instance.main.endpoint}/${var.db_name}" sensitive = true } + +output "instance_id" { + value = aws_db_instance.main.identifier +} diff --git a/infrastructure/terraform/modules/redis/main.tf b/infrastructure/terraform/modules/redis/main.tf index 9336a3a0..8ad0fbf9 100644 --- a/infrastructure/terraform/modules/redis/main.tf +++ b/infrastructure/terraform/modules/redis/main.tf @@ -122,6 +122,10 @@ output "endpoint" { } output "redis_url" { - value = "redis://:${var.redis_auth_token}@${aws_elasticache_replication_group.main.primary_endpoint_address}:6379" + value = "rediss://:${var.redis_auth_token}@${aws_elasticache_replication_group.main.primary_endpoint_address}:6379" sensitive = true } + +output "replication_group_id" { + value = aws_elasticache_replication_group.main.id +} diff --git a/infrastructure/terraform/tests/check_invariants.sh b/infrastructure/terraform/tests/check_invariants.sh new file mode 100755 index 00000000..2ac4fb6c --- /dev/null +++ b/infrastructure/terraform/tests/check_invariants.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Static regression checks for redis TLS, ALB dimensions, alarms and autoscaling. +set -euo pipefail +cd "$(dirname "$0")/.." +fail() { echo "FAIL: $1"; exit 1; } +grep -q 'rediss://' modules/redis/main.tf || fail "redis_url must use rediss://" +! grep -q '"redis://' modules/redis/main.tf || fail "plaintext redis:// URL present" +grep -q 'tokio-rustls-comp' ../../services/api/Cargo.toml || fail "redis crate TLS feature missing" +! grep -q 'alb/\*' modules/monitoring/main.tf || fail "wildcard LoadBalancer dimension" +grep -q 'var.alb_arn_suffix' modules/monitoring/main.tf || fail "alarms must use alb_arn_suffix" +for m in FreeStorageSpace DatabaseMemoryUsagePercentage Evictions ReplicationLag UnHealthyHostCount TargetResponseTime RunningTaskCount; do + grep -q "$m" modules/monitoring/main.tf || fail "missing alarm for $m" +done +grep -q aws_appautoscaling_target modules/ecs/main.tf || fail "autoscaling target missing" +grep -q ALBRequestCountPerTarget modules/ecs/main.tf || fail "request-count scaling missing" +echo "OK" diff --git a/infrastructure/terraform/variables.tf b/infrastructure/terraform/variables.tf index 9fbb8876..8e53adf2 100644 --- a/infrastructure/terraform/variables.tf +++ b/infrastructure/terraform/variables.tf @@ -183,6 +183,35 @@ variable "api_desired_count" { } } +variable "api_min_count" { + description = "Minimum API tasks for autoscaling" + type = number + default = 2 + + validation { + condition = var.api_min_count >= 1 && var.api_min_count <= var.api_max_count + error_message = "api_min_count must be >= 1 and <= api_max_count." + } +} + +variable "api_max_count" { + description = "Maximum API tasks for autoscaling" + type = number + default = 10 +} + +variable "rds_free_storage_alarm_bytes" { + description = "Alarm when RDS free storage drops below this many bytes" + type = number + default = 5368709120 +} + +variable "alb_latency_alarm_seconds" { + description = "Alarm when ALB p95 TargetResponseTime exceeds this many seconds" + type = number + default = 1 +} + variable "api_cpu" { description = "API task CPU units" type = number diff --git a/services/api/Cargo.toml b/services/api/Cargo.toml index 73764721..7ca88154 100644 --- a/services/api/Cargo.toml +++ b/services/api/Cargo.toml @@ -31,7 +31,7 @@ chrono = { version = "0.4", features = ["serde"] } futures = "0.3" handlebars = "5.1" prometheus = "0.13" -redis = { version = "0.25", features = ["tokio-comp", "connection-manager", "streams"] } +redis = { version = "0.25", features = ["tokio-comp", "connection-manager", "streams", "tokio-rustls-comp", "tls-rustls-webpki-roots"] } deadpool-redis = { version = "0.15", features = ["rt_tokio_1"] } async-trait = "0.1" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }