fix(infra): Redis TLS, ALB alarm dimensions, extra alarms, ECS autoscaling

.github/workflows/terraform.yml

@@ -57,6 +57,14 @@ jobs:
         if: steps.fmt.outcome == 'failure'
         run: exit 1
 
+  terraform-invariants:
+    name: Terraform Invariant Checks
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v4
+      - run: bash infrastructure/terraform/tests/check_invariants.sh
+
   terraform-validate:
     name: Terraform Validate
     runs-on: ubuntu-latest

infrastructure/README.md

@@ -273,3 +273,11 @@ For infrastructure issues:
 - [Terraform Documentation](https://www.terraform.io/docs)
 - [AWS Provider Documentation](https://registry.terraform.io/providers/hashicorp/aws/latest/docs)
 - [PredictIQ Architecture](../docs/ARCHITECTURE.md)
+
+## Redis TLS, alarms and autoscaling
+
+- ElastiCache enforces in-transit encryption; the module emits a `rediss://` URL and the API's `redis` crate is built with `tokio-rustls-comp`.
+- CloudWatch ALB alarms/dashboard use `aws_lb.main.arn_suffix` (passed into the monitoring module), not a wildcard.
+- Alarms cover ECS CPU/memory/running tasks, ALB 5xx/latency/unhealthy hosts, RDS CPU/storage/connections and Redis memory/evictions/replication lag; each description links its runbook in `docs/runbooks/`.
+- The ECS service autoscales on CPU and ALB request count per target between `api_min_count` and `api_max_count`.
+- `infrastructure/terraform/tests/check_invariants.sh` (run in the Terraform CI workflow) guards these.

infrastructure/terraform/environments/ci-validate.tfvars

@@ -17,5 +17,6 @@ redis_auth_token      = "CiValidate!Redis#Placeholder2024XYZ"
 api_image_uri         = "123456789012.dkr.ecr.us-east-1.amazonaws.com/predictiq:ci-validate"
 api_container_port    = 8080
 api_desired_count     = 1
+api_min_count         = 1
 api_cpu               = 256
 api_memory            = 512

infrastructure/terraform/environments/dev.tfvars

@@ -7,5 +7,6 @@ backup_retention_days = 7
 redis_node_type     = "cache.t3.micro"
 redis_num_nodes     = 1
 api_desired_count   = 1
+api_min_count       = 1
 api_cpu             = 256
 api_memory          = 512

infrastructure/terraform/main.tf

@@ -116,6 +116,8 @@ module "ecs" {
   api_image_uri         = var.api_image_uri
   api_container_port    = var.api_container_port
   api_desired_count     = var.api_desired_count
+  api_min_count         = var.api_min_count
+  api_max_count         = var.api_max_count
   api_cpu               = var.api_cpu
   api_memory            = var.api_memory
   db_host               = module.rds.endpoint
@@ -141,6 +143,14 @@ module "monitoring" {
   ecs_cluster = module.ecs.cluster_name
   ecs_service = module.ecs.service_name
 
+  alb_arn_suffix          = module.ecs.alb_arn_suffix
+  target_group_arn_suffix = module.ecs.target_group_arn_suffix
+  rds_instance_id         = module.rds.instance_id
+  redis_replication_group = module.redis.replication_group_id
+  api_desired_count       = var.api_desired_count
+  rds_free_storage_bytes  = var.rds_free_storage_alarm_bytes
+  alb_latency_seconds     = var.alb_latency_alarm_seconds
+
   alert_email_endpoints = var.alert_email_endpoints
   alert_https_endpoints = var.alert_https_endpoints
 }

infrastructure/terraform/modules/ecs/main.tf

@@ -26,6 +26,29 @@ variable "api_desired_count" {
   type = number
 }
 
+variable "api_min_count" {
+  type        = number
+  default     = 2
+  description = "Minimum number of API tasks for autoscaling"
+}
+
+variable "api_max_count" {
+  type        = number
+  default     = 10
+  description = "Maximum number of API tasks for autoscaling"
+}
+
+variable "autoscaling_cpu_target" {
+  type    = number
+  default = 60
+}
+
+variable "autoscaling_requests_per_target" {
+  type        = number
+  default     = 1000
+  description = "Target ALB requests per target per minute"
+}
+
 variable "api_cpu" {
   type = number
 }
@@ -427,6 +450,11 @@ resource "aws_ecs_service" "api" {
   desired_count   = var.api_desired_count
   launch_type     = "FARGATE"
 
+  # Desired count is owned by the autoscaler after the first apply.
+  lifecycle {
+    ignore_changes = [desired_count]
+  }
+
   # Zero-downtime rolling deploy: always keep 100 % capacity during updates,
   # allow up to 200 % so new tasks start before old ones are drained.
   deployment_minimum_healthy_percent = 100
@@ -749,6 +777,53 @@ resource "aws_iam_role" "ecs_task_role" {
   })
 }
 
+# ── Autoscaling ────────────────────────────────────────────────────────────────
+
+resource "aws_appautoscaling_target" "api" {
+  service_namespace  = "ecs"
+  scalable_dimension = "ecs:service:DesiredCount"
+  resource_id        = "service/${aws_ecs_cluster.main.name}/${aws_ecs_service.api.name}"
+  min_capacity       = var.api_min_count
+  max_capacity       = var.api_max_count
+}
+
+resource "aws_appautoscaling_policy" "api_cpu" {
+  name               = "predictiq-${var.environment}-api-cpu-target"
+  policy_type        = "TargetTrackingScaling"
+  service_namespace  = aws_appautoscaling_target.api.service_namespace
+  scalable_dimension = aws_appautoscaling_target.api.scalable_dimension
+  resource_id        = aws_appautoscaling_target.api.resource_id
+
+  target_tracking_scaling_policy_configuration {
+    target_value       = var.autoscaling_cpu_target
+    scale_in_cooldown  = 300
+    scale_out_cooldown = 60
+
+    predefined_metric_specification {
+      predefined_metric_type = "ECSServiceAverageCPUUtilization"
+    }
+  }
+}
+
+resource "aws_appautoscaling_policy" "api_requests" {
+  name               = "predictiq-${var.environment}-api-requests-target"
+  policy_type        = "TargetTrackingScaling"
+  service_namespace  = aws_appautoscaling_target.api.service_namespace
+  scalable_dimension = aws_appautoscaling_target.api.scalable_dimension
+  resource_id        = aws_appautoscaling_target.api.resource_id
+
+  target_tracking_scaling_policy_configuration {
+    target_value       = var.autoscaling_requests_per_target
+    scale_in_cooldown  = 300
+    scale_out_cooldown = 60
+
+    predefined_metric_specification {
+      predefined_metric_type = "ALBRequestCountPerTarget"
+      resource_label         = "${aws_lb.main.arn_suffix}/${aws_lb_target_group.api.arn_suffix}"
+    }
+  }
+}
+
 # ── Data sources / Outputs ─────────────────────────────────────────────────────
 
 data "aws_region" "current" {}
@@ -767,6 +842,14 @@ output "service_name" {
   value = aws_ecs_service.api.name
 }
 
+output "alb_arn_suffix" {
+  value = aws_lb.main.arn_suffix
+}
+
+output "target_group_arn_suffix" {
+  value = aws_lb_target_group.api.arn_suffix
+}
+
 output "alb_dns_name" {
   value = aws_lb.main.dns_name
 }

infrastructure/terraform/modules/monitoring/main.tf

@@ -10,6 +10,38 @@ variable "ecs_service" {
   type = string
 }
 
+variable "alb_arn_suffix" {
+  type        = string
+  description = "aws_lb.arn_suffix (app/<name>/<id>) used as the LoadBalancer dimension"
+}
+
+variable "target_group_arn_suffix" {
+  type = string
+}
+
+variable "rds_instance_id" {
+  type = string
+}
+
+variable "redis_replication_group" {
+  type = string
+}
+
+variable "api_desired_count" {
+  type    = number
+  default = 2
+}
+
+variable "rds_free_storage_bytes" {
+  type    = number
+  default = 5368709120
+}
+
+variable "alb_latency_seconds" {
+  type    = number
+  default = 1
+}
+
 variable "alert_email_endpoints" {
   type        = list(string)
   default     = []
@@ -53,7 +85,7 @@ resource "aws_cloudwatch_dashboard" "main" {
         type = "metric"
         properties = {
           metrics = [
-            ["AWS/ApplicationELB", "TargetResponseTime", "LoadBalancer", "app/predictiq-${var.environment}-alb/*"],
+            ["AWS/ApplicationELB", "TargetResponseTime", "LoadBalancer", var.alb_arn_suffix],
             [".", "RequestCount", ".", "."],
             [".", "HTTPCode_Target_5XX_Count", ".", "."]
           ]
@@ -155,7 +187,7 @@ resource "aws_cloudwatch_metric_alarm" "alb_5xx" {
   alarm_actions       = [aws_sns_topic.alerts.arn]
 
   dimensions = {
-    LoadBalancer = "app/predictiq-${var.environment}-alb/*"
+    LoadBalancer = var.alb_arn_suffix
   }
 
   tags = merge(
@@ -166,6 +198,90 @@ resource "aws_cloudwatch_metric_alarm" "alb_5xx" {
   )
 }
 
+locals {
+  runbook_base = "docs/runbooks"
+  alarms = {
+    rds_cpu = {
+      namespace = "AWS/RDS", metric = "CPUUtilization", stat = "Average", op = "GreaterThanThreshold"
+      threshold = 80, periods = 3, dims = { DBInstanceIdentifier = var.rds_instance_id }
+      runbook = "slow-database-queries.md", missing = "missing"
+    }
+    rds_free_storage = {
+      namespace = "AWS/RDS", metric = "FreeStorageSpace", stat = "Minimum", op = "LessThanThreshold"
+      threshold = var.rds_free_storage_bytes, periods = 1, dims = { DBInstanceIdentifier = var.rds_instance_id }
+      runbook = "critical-database-queries.md", missing = "missing"
+    }
+    rds_connections = {
+      namespace = "AWS/RDS", metric = "DatabaseConnections", stat = "Average", op = "GreaterThanThreshold"
+      threshold = 80, periods = 3, dims = { DBInstanceIdentifier = var.rds_instance_id }
+      runbook = "high-db-connection-pool-utilization.md", missing = "missing"
+    }
+    redis_memory = {
+      namespace = "AWS/ElastiCache", metric = "DatabaseMemoryUsagePercentage", stat = "Average", op = "GreaterThanThreshold"
+      threshold = 80, periods = 3, dims = { ReplicationGroupId = var.redis_replication_group }
+      runbook = "redis-failure.md", missing = "missing"
+    }
+    redis_evictions = {
+      namespace = "AWS/ElastiCache", metric = "Evictions", stat = "Sum", op = "GreaterThanThreshold"
+      threshold = 100, periods = 2, dims = { ReplicationGroupId = var.redis_replication_group }
+      runbook = "low-cache-hit-rate.md", missing = "notBreaching"
+    }
+    redis_replication_lag = {
+      namespace = "AWS/ElastiCache", metric = "ReplicationLag", stat = "Maximum", op = "GreaterThanThreshold"
+      threshold = 30, periods = 3, dims = { ReplicationGroupId = var.redis_replication_group }
+      runbook = "redis-failure.md", missing = "notBreaching"
+    }
+    alb_unhealthy_hosts = {
+      namespace = "AWS/ApplicationELB", metric = "UnHealthyHostCount", stat = "Maximum", op = "GreaterThanThreshold"
+      threshold = 0, periods = 2, dims = { LoadBalancer = var.alb_arn_suffix, TargetGroup = var.target_group_arn_suffix }
+      runbook = "service-down.md", missing = "missing"
+    }
+    ecs_running_tasks = {
+      namespace = "ECS/ContainerInsights", metric = "RunningTaskCount", stat = "Minimum", op = "LessThanThreshold"
+      threshold = var.api_desired_count, periods = 2, dims = { ClusterName = var.ecs_cluster, ServiceName = var.ecs_service }
+      runbook = "ecs-task-crash-loop.md", missing = "breaching"
+    }
+  }
+}
+
+resource "aws_cloudwatch_metric_alarm" "extra" {
+  for_each = local.alarms
+
+  alarm_name          = "predictiq-${var.environment}-${replace(each.key, "_", "-")}"
+  comparison_operator = each.value.op
+  evaluation_periods  = each.value.periods
+  metric_name         = each.value.metric
+  namespace           = each.value.namespace
+  period              = 300
+  statistic           = each.value.stat
+  threshold           = each.value.threshold
+  treat_missing_data  = each.value.missing
+  alarm_description   = "${each.key} breached. Runbook: ${local.runbook_base}/${each.value.runbook}"
+  alarm_actions       = [aws_sns_topic.alerts.arn]
+  dimensions          = each.value.dims
+
+  tags = merge(local.common_tags, { Name = "predictiq-${var.environment}-${each.key}" })
+}
+
+resource "aws_cloudwatch_metric_alarm" "alb_latency" {
+  alarm_name          = "predictiq-${var.environment}-alb-latency-high"
+  comparison_operator = "GreaterThanThreshold"
+  evaluation_periods  = 3
+  metric_name         = "TargetResponseTime"
+  namespace           = "AWS/ApplicationELB"
+  period              = 300
+  extended_statistic  = "p95"
+  threshold           = var.alb_latency_seconds
+  alarm_description   = "ALB p95 latency high. Runbook: ${local.runbook_base}/high-api-response-time.md"
+  alarm_actions       = [aws_sns_topic.alerts.arn]
+
+  dimensions = {
+    LoadBalancer = var.alb_arn_suffix
+  }
+
+  tags = merge(local.common_tags, { Name = "predictiq-${var.environment}-alb-latency-high" })
+}
+
 data "aws_region" "current" {}
 
 output "sns_topic_arn" {

infrastructure/terraform/modules/rds/main.tf

@@ -149,3 +149,12 @@ output "endpoint" {
   value     = aws_db_instance.main.endpoint
   sensitive = true
 }
+
+output "database_url" {
+  value     = "postgresql://${var.db_username}:${var.db_password}@${aws_db_instance.main.endpoint}/${var.db_name}"
+  sensitive = true
+}
+
+output "instance_id" {
+  value = aws_db_instance.main.identifier
+}