diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml index bd3ebb22..f5762a8d 100644 --- a/.github/workflows/terraform.yml +++ b/.github/workflows/terraform.yml @@ -57,6 +57,14 @@ jobs: if: steps.fmt.outcome == 'failure' run: exit 1 + terraform-invariants: + name: Terraform Invariant Checks + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - run: bash infrastructure/terraform/tests/check_invariants.sh + terraform-validate: name: Terraform Validate runs-on: ubuntu-latest diff --git a/infrastructure/README.md b/infrastructure/README.md index 82b5d49c..a52d5296 100644 --- a/infrastructure/README.md +++ b/infrastructure/README.md @@ -273,3 +273,11 @@ For infrastructure issues: - [Terraform Documentation](https://www.terraform.io/docs) - [AWS Provider Documentation](https://registry.terraform.io/providers/hashicorp/aws/latest/docs) - [PredictIQ Architecture](../docs/ARCHITECTURE.md) + +## Redis TLS, alarms and autoscaling + +- ElastiCache enforces in-transit encryption; the module emits a `rediss://` URL and the API's `redis` crate is built with `tokio-rustls-comp`. +- CloudWatch ALB alarms/dashboard use `aws_lb.main.arn_suffix` (passed into the monitoring module), not a wildcard. +- Alarms cover ECS CPU/memory/running tasks, ALB 5xx/latency/unhealthy hosts, RDS CPU/storage/connections and Redis memory/evictions/replication lag; each description links its runbook in `docs/runbooks/`. +- The ECS service autoscales on CPU and ALB request count per target between `api_min_count` and `api_max_count`. +- `infrastructure/terraform/tests/check_invariants.sh` (run in the Terraform CI workflow) guards these. diff --git a/infrastructure/terraform/environments/ci-validate.tfvars b/infrastructure/terraform/environments/ci-validate.tfvars index 65969cd5..6d07485a 100644 --- a/infrastructure/terraform/environments/ci-validate.tfvars +++ b/infrastructure/terraform/environments/ci-validate.tfvars @@ -17,5 +17,6 @@ redis_auth_token = "CiValidate!Redis#Placeholder2024XYZ" api_image_uri = "123456789012.dkr.ecr.us-east-1.amazonaws.com/predictiq:ci-validate" api_container_port = 8080 api_desired_count = 1 +api_min_count = 1 api_cpu = 256 api_memory = 512 diff --git a/infrastructure/terraform/environments/dev.tfvars b/infrastructure/terraform/environments/dev.tfvars index ab7f2368..17248872 100644 --- a/infrastructure/terraform/environments/dev.tfvars +++ b/infrastructure/terraform/environments/dev.tfvars @@ -7,5 +7,6 @@ backup_retention_days = 7 redis_node_type = "cache.t3.micro" redis_num_nodes = 1 api_desired_count = 1 +api_min_count = 1 api_cpu = 256 api_memory = 512 diff --git a/infrastructure/terraform/main.tf b/infrastructure/terraform/main.tf index c1053304..2e379592 100644 --- a/infrastructure/terraform/main.tf +++ b/infrastructure/terraform/main.tf @@ -116,6 +116,8 @@ module "ecs" { api_image_uri = var.api_image_uri api_container_port = var.api_container_port api_desired_count = var.api_desired_count + api_min_count = var.api_min_count + api_max_count = var.api_max_count api_cpu = var.api_cpu api_memory = var.api_memory db_host = module.rds.endpoint @@ -141,6 +143,14 @@ module "monitoring" { ecs_cluster = module.ecs.cluster_name ecs_service = module.ecs.service_name + alb_arn_suffix = module.ecs.alb_arn_suffix + target_group_arn_suffix = module.ecs.target_group_arn_suffix + rds_instance_id = module.rds.instance_id + redis_replication_group = module.redis.replication_group_id + api_desired_count = var.api_desired_count + rds_free_storage_bytes = var.rds_free_storage_alarm_bytes + alb_latency_seconds = var.alb_latency_alarm_seconds + alert_email_endpoints = var.alert_email_endpoints alert_https_endpoints = var.alert_https_endpoints } diff --git a/infrastructure/terraform/modules/ecs/main.tf b/infrastructure/terraform/modules/ecs/main.tf index 1e11375d..2280747e 100644 --- a/infrastructure/terraform/modules/ecs/main.tf +++ b/infrastructure/terraform/modules/ecs/main.tf @@ -26,6 +26,29 @@ variable "api_desired_count" { type = number } +variable "api_min_count" { + type = number + default = 2 + description = "Minimum number of API tasks for autoscaling" +} + +variable "api_max_count" { + type = number + default = 10 + description = "Maximum number of API tasks for autoscaling" +} + +variable "autoscaling_cpu_target" { + type = number + default = 60 +} + +variable "autoscaling_requests_per_target" { + type = number + default = 1000 + description = "Target ALB requests per target per minute" +} + variable "api_cpu" { type = number } @@ -427,6 +450,11 @@ resource "aws_ecs_service" "api" { desired_count = var.api_desired_count launch_type = "FARGATE" + # Desired count is owned by the autoscaler after the first apply. + lifecycle { + ignore_changes = [desired_count] + } + # Zero-downtime rolling deploy: always keep 100 % capacity during updates, # allow up to 200 % so new tasks start before old ones are drained. deployment_minimum_healthy_percent = 100 @@ -749,6 +777,53 @@ resource "aws_iam_role" "ecs_task_role" { }) } +# ── Autoscaling ──────────────────────────────────────────────────────────────── + +resource "aws_appautoscaling_target" "api" { + service_namespace = "ecs" + scalable_dimension = "ecs:service:DesiredCount" + resource_id = "service/${aws_ecs_cluster.main.name}/${aws_ecs_service.api.name}" + min_capacity = var.api_min_count + max_capacity = var.api_max_count +} + +resource "aws_appautoscaling_policy" "api_cpu" { + name = "predictiq-${var.environment}-api-cpu-target" + policy_type = "TargetTrackingScaling" + service_namespace = aws_appautoscaling_target.api.service_namespace + scalable_dimension = aws_appautoscaling_target.api.scalable_dimension + resource_id = aws_appautoscaling_target.api.resource_id + + target_tracking_scaling_policy_configuration { + target_value = var.autoscaling_cpu_target + scale_in_cooldown = 300 + scale_out_cooldown = 60 + + predefined_metric_specification { + predefined_metric_type = "ECSServiceAverageCPUUtilization" + } + } +} + +resource "aws_appautoscaling_policy" "api_requests" { + name = "predictiq-${var.environment}-api-requests-target" + policy_type = "TargetTrackingScaling" + service_namespace = aws_appautoscaling_target.api.service_namespace + scalable_dimension = aws_appautoscaling_target.api.scalable_dimension + resource_id = aws_appautoscaling_target.api.resource_id + + target_tracking_scaling_policy_configuration { + target_value = var.autoscaling_requests_per_target + scale_in_cooldown = 300 + scale_out_cooldown = 60 + + predefined_metric_specification { + predefined_metric_type = "ALBRequestCountPerTarget" + resource_label = "${aws_lb.main.arn_suffix}/${aws_lb_target_group.api.arn_suffix}" + } + } +} + # ── Data sources / Outputs ───────────────────────────────────────────────────── data "aws_region" "current" {} @@ -767,6 +842,14 @@ output "service_name" { value = aws_ecs_service.api.name } +output "alb_arn_suffix" { + value = aws_lb.main.arn_suffix +} + +output "target_group_arn_suffix" { + value = aws_lb_target_group.api.arn_suffix +} + output "alb_dns_name" { value = aws_lb.main.dns_name } diff --git a/infrastructure/terraform/modules/monitoring/main.tf b/infrastructure/terraform/modules/monitoring/main.tf index 7c9302c9..fad4f37b 100644 --- a/infrastructure/terraform/modules/monitoring/main.tf +++ b/infrastructure/terraform/modules/monitoring/main.tf @@ -10,6 +10,38 @@ variable "ecs_service" { type = string } +variable "alb_arn_suffix" { + type = string + description = "aws_lb.arn_suffix (app//) used as the LoadBalancer dimension" +} + +variable "target_group_arn_suffix" { + type = string +} + +variable "rds_instance_id" { + type = string +} + +variable "redis_replication_group" { + type = string +} + +variable "api_desired_count" { + type = number + default = 2 +} + +variable "rds_free_storage_bytes" { + type = number + default = 5368709120 +} + +variable "alb_latency_seconds" { + type = number + default = 1 +} + variable "alert_email_endpoints" { type = list(string) default = [] @@ -53,7 +85,7 @@ resource "aws_cloudwatch_dashboard" "main" { type = "metric" properties = { metrics = [ - ["AWS/ApplicationELB", "TargetResponseTime", "LoadBalancer", "app/predictiq-${var.environment}-alb/*"], + ["AWS/ApplicationELB", "TargetResponseTime", "LoadBalancer", var.alb_arn_suffix], [".", "RequestCount", ".", "."], [".", "HTTPCode_Target_5XX_Count", ".", "."] ] @@ -155,7 +187,7 @@ resource "aws_cloudwatch_metric_alarm" "alb_5xx" { alarm_actions = [aws_sns_topic.alerts.arn] dimensions = { - LoadBalancer = "app/predictiq-${var.environment}-alb/*" + LoadBalancer = var.alb_arn_suffix } tags = merge( @@ -166,6 +198,90 @@ resource "aws_cloudwatch_metric_alarm" "alb_5xx" { ) } +locals { + runbook_base = "docs/runbooks" + alarms = { + rds_cpu = { + namespace = "AWS/RDS", metric = "CPUUtilization", stat = "Average", op = "GreaterThanThreshold" + threshold = 80, periods = 3, dims = { DBInstanceIdentifier = var.rds_instance_id } + runbook = "slow-database-queries.md", missing = "missing" + } + rds_free_storage = { + namespace = "AWS/RDS", metric = "FreeStorageSpace", stat = "Minimum", op = "LessThanThreshold" + threshold = var.rds_free_storage_bytes, periods = 1, dims = { DBInstanceIdentifier = var.rds_instance_id } + runbook = "critical-database-queries.md", missing = "missing" + } + rds_connections = { + namespace = "AWS/RDS", metric = "DatabaseConnections", stat = "Average", op = "GreaterThanThreshold" + threshold = 80, periods = 3, dims = { DBInstanceIdentifier = var.rds_instance_id } + runbook = "high-db-connection-pool-utilization.md", missing = "missing" + } + redis_memory = { + namespace = "AWS/ElastiCache", metric = "DatabaseMemoryUsagePercentage", stat = "Average", op = "GreaterThanThreshold" + threshold = 80, periods = 3, dims = { ReplicationGroupId = var.redis_replication_group } + runbook = "redis-failure.md", missing = "missing" + } + redis_evictions = { + namespace = "AWS/ElastiCache", metric = "Evictions", stat = "Sum", op = "GreaterThanThreshold" + threshold = 100, periods = 2, dims = { ReplicationGroupId = var.redis_replication_group } + runbook = "low-cache-hit-rate.md", missing = "notBreaching" + } + redis_replication_lag = { + namespace = "AWS/ElastiCache", metric = "ReplicationLag", stat = "Maximum", op = "GreaterThanThreshold" + threshold = 30, periods = 3, dims = { ReplicationGroupId = var.redis_replication_group } + runbook = "redis-failure.md", missing = "notBreaching" + } + alb_unhealthy_hosts = { + namespace = "AWS/ApplicationELB", metric = "UnHealthyHostCount", stat = "Maximum", op = "GreaterThanThreshold" + threshold = 0, periods = 2, dims = { LoadBalancer = var.alb_arn_suffix, TargetGroup = var.target_group_arn_suffix } + runbook = "service-down.md", missing = "missing" + } + ecs_running_tasks = { + namespace = "ECS/ContainerInsights", metric = "RunningTaskCount", stat = "Minimum", op = "LessThanThreshold" + threshold = var.api_desired_count, periods = 2, dims = { ClusterName = var.ecs_cluster, ServiceName = var.ecs_service } + runbook = "ecs-task-crash-loop.md", missing = "breaching" + } + } +} + +resource "aws_cloudwatch_metric_alarm" "extra" { + for_each = local.alarms + + alarm_name = "predictiq-${var.environment}-${replace(each.key, "_", "-")}" + comparison_operator = each.value.op + evaluation_periods = each.value.periods + metric_name = each.value.metric + namespace = each.value.namespace + period = 300 + statistic = each.value.stat + threshold = each.value.threshold + treat_missing_data = each.value.missing + alarm_description = "${each.key} breached. Runbook: ${local.runbook_base}/${each.value.runbook}" + alarm_actions = [aws_sns_topic.alerts.arn] + dimensions = each.value.dims + + tags = merge(local.common_tags, { Name = "predictiq-${var.environment}-${each.key}" }) +} + +resource "aws_cloudwatch_metric_alarm" "alb_latency" { + alarm_name = "predictiq-${var.environment}-alb-latency-high" + comparison_operator = "GreaterThanThreshold" + evaluation_periods = 3 + metric_name = "TargetResponseTime" + namespace = "AWS/ApplicationELB" + period = 300 + extended_statistic = "p95" + threshold = var.alb_latency_seconds + alarm_description = "ALB p95 latency high. Runbook: ${local.runbook_base}/high-api-response-time.md" + alarm_actions = [aws_sns_topic.alerts.arn] + + dimensions = { + LoadBalancer = var.alb_arn_suffix + } + + tags = merge(local.common_tags, { Name = "predictiq-${var.environment}-alb-latency-high" }) +} + data "aws_region" "current" {} output "sns_topic_arn" { diff --git a/infrastructure/terraform/modules/rds/main.tf b/infrastructure/terraform/modules/rds/main.tf index 1baccf10..4bb0815c 100644 --- a/infrastructure/terraform/modules/rds/main.tf +++ b/infrastructure/terraform/modules/rds/main.tf @@ -149,3 +149,12 @@ output "endpoint" { value = aws_db_instance.main.endpoint sensitive = true } + +output "database_url" { + value = "postgresql://${var.db_username}:${var.db_password}@${aws_db_instance.main.endpoint}/${var.db_name}" + sensitive = true +} + +output "instance_id" { + value = aws_db_instance.main.identifier +} diff --git a/infrastructure/terraform/modules/redis/main.tf b/infrastructure/terraform/modules/redis/main.tf index 3b96e061..857737e8 100644 --- a/infrastructure/terraform/modules/redis/main.tf +++ b/infrastructure/terraform/modules/redis/main.tf @@ -1,123 +1,5 @@ -variable "environment" { - type = string -} - -variable "vpc_id" { - type = string -} - -variable "subnet_ids" { - type = list(string) -} - -variable "node_type" { - type = string -} - -variable "num_cache_clusters" { - type = number - default = 2 -} - -variable "engine_version" { - type = string -} - -variable "redis_auth_token" { - type = string - sensitive = true - description = "AUTH token required to authenticate to the Redis replication group" -} - -variable "redis_multi_az_enabled" { - type = bool - default = true - description = "Enable Multi-AZ automatic failover for the Redis replication group." -} - -variable "ecs_tasks_sg_id" { - type = string - description = "Security group ID of the ECS tasks that are allowed to connect" -} - -locals { - common_tags = { - Project = "predictiq" - Environment = var.environment - Owner = "infrastructure-team" - ManagedBy = "terraform" - } -} - -resource "aws_elasticache_subnet_group" "main" { - name = "predictiq-${var.environment}-redis-subnet" - subnet_ids = var.subnet_ids - - tags = merge( - local.common_tags, - { - Name = "predictiq-${var.environment}-redis-subnet-group" - } - ) -} - -resource "aws_security_group" "redis" { - name = "predictiq-${var.environment}-redis-sg" - vpc_id = var.vpc_id - - # Inbound Redis from ECS tasks only - ingress { - from_port = 6379 - to_port = 6379 - protocol = "tcp" - security_groups = [var.ecs_tasks_sg_id] - } - - tags = merge( - local.common_tags, - { - Name = "predictiq-${var.environment}-redis-sg" - } - ) -} - -resource "aws_elasticache_replication_group" "main" { - replication_group_id = "predictiq-${var.environment}" - description = "PredictIQ Redis replication group (${var.environment})" - - engine = "redis" - engine_version = var.engine_version - node_type = var.node_type - num_cache_clusters = var.num_cache_clusters >= 2 ? var.num_cache_clusters : 2 - parameter_group_name = "default.redis7" - port = 6379 - - subnet_group_name = aws_elasticache_subnet_group.main.name - security_group_ids = [aws_security_group.redis.id] - - automatic_failover_enabled = var.redis_multi_az_enabled - multi_az_enabled = var.redis_multi_az_enabled - - at_rest_encryption_enabled = true - transit_encryption_enabled = true - auth_token = var.redis_auth_token - - maintenance_window = "mon:03:00-mon:04:00" - - tags = merge( - local.common_tags, - { - Name = "predictiq-${var.environment}-redis" - } - ) -} - -output "sg_id" { - value = aws_security_group.redis.id -} - -output "endpoint" { - value = aws_elasticache_replication_group.main.primary_endpoint_address +output "redis_url" { + value = "rediss://:${var.redis_auth_token}@${aws_elasticache_replication_group.main.primary_endpoint_address}:6379" sensitive = true } @@ -129,3 +11,7 @@ output "auth_token" { value = var.redis_auth_token sensitive = true } + +output "replication_group_id" { + value = aws_elasticache_replication_group.main.id +} diff --git a/infrastructure/terraform/tests/check_invariants.sh b/infrastructure/terraform/tests/check_invariants.sh new file mode 100755 index 00000000..2ac4fb6c --- /dev/null +++ b/infrastructure/terraform/tests/check_invariants.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Static regression checks for redis TLS, ALB dimensions, alarms and autoscaling. +set -euo pipefail +cd "$(dirname "$0")/.." +fail() { echo "FAIL: $1"; exit 1; } +grep -q 'rediss://' modules/redis/main.tf || fail "redis_url must use rediss://" +! grep -q '"redis://' modules/redis/main.tf || fail "plaintext redis:// URL present" +grep -q 'tokio-rustls-comp' ../../services/api/Cargo.toml || fail "redis crate TLS feature missing" +! grep -q 'alb/\*' modules/monitoring/main.tf || fail "wildcard LoadBalancer dimension" +grep -q 'var.alb_arn_suffix' modules/monitoring/main.tf || fail "alarms must use alb_arn_suffix" +for m in FreeStorageSpace DatabaseMemoryUsagePercentage Evictions ReplicationLag UnHealthyHostCount TargetResponseTime RunningTaskCount; do + grep -q "$m" modules/monitoring/main.tf || fail "missing alarm for $m" +done +grep -q aws_appautoscaling_target modules/ecs/main.tf || fail "autoscaling target missing" +grep -q ALBRequestCountPerTarget modules/ecs/main.tf || fail "request-count scaling missing" +echo "OK" diff --git a/infrastructure/terraform/variables.tf b/infrastructure/terraform/variables.tf index 2f2633af..a8db11ab 100644 --- a/infrastructure/terraform/variables.tf +++ b/infrastructure/terraform/variables.tf @@ -183,6 +183,35 @@ variable "api_desired_count" { } } +variable "api_min_count" { + description = "Minimum API tasks for autoscaling" + type = number + default = 2 + + validation { + condition = var.api_min_count >= 1 && var.api_min_count <= var.api_max_count + error_message = "api_min_count must be >= 1 and <= api_max_count." + } +} + +variable "api_max_count" { + description = "Maximum API tasks for autoscaling" + type = number + default = 10 +} + +variable "rds_free_storage_alarm_bytes" { + description = "Alarm when RDS free storage drops below this many bytes" + type = number + default = 5368709120 +} + +variable "alb_latency_alarm_seconds" { + description = "Alarm when ALB p95 TargetResponseTime exceeds this many seconds" + type = number + default = 1 +} + variable "api_cpu" { description = "API task CPU units" type = number diff --git a/services/api/Cargo.toml b/services/api/Cargo.toml index 73764721..7ca88154 100644 --- a/services/api/Cargo.toml +++ b/services/api/Cargo.toml @@ -31,7 +31,7 @@ chrono = { version = "0.4", features = ["serde"] } futures = "0.3" handlebars = "5.1" prometheus = "0.13" -redis = { version = "0.25", features = ["tokio-comp", "connection-manager", "streams"] } +redis = { version = "0.25", features = ["tokio-comp", "connection-manager", "streams", "tokio-rustls-comp", "tls-rustls-webpki-roots"] } deadpool-redis = { version = "0.15", features = ["rt_tokio_1"] } async-trait = "0.1" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }