Harden Terraform: alert subscriptions, RDS, no password URLs in state, KMS

.github/workflows/terraform.yml

@@ -31,6 +31,10 @@ jobs:
         with:
           terraform_version: ${{ env.TF_VERSION }}
 
+      - name: Security invariants (#1775 #1782 #1783 #1784)
+        run: bash tests/check_security_invariants.sh
+        working-directory: ${{ env.TF_WORKING_DIR }}
+
       - name: Terraform fmt
         id: fmt
         run: terraform fmt -check -recursive

docs/deployment.md

@@ -112,8 +112,7 @@ Note the following values — you will need them for environment variable config
 
 | Output | Usage |
 |---|---|
-| `database_url` | `DATABASE_URL` for the API service |
-| `redis_url` | `REDIS_URL` for the API service |
+| `endpoint` (rds/redis modules) | Host components injected into Secrets Manager; the API assembles connection URLs at runtime (no password-bearing URL in state) |
 | `alb_dns_name` | Configure DNS CNAME to this value |
 | `ecs_cluster_name` | Used for ECS deployment commands |
 | `ecs_service_name` | Used for ECS deployment commands |
@@ -218,7 +217,7 @@ The ECS task definition (`infrastructure/terraform/modules/ecs/main.tf`) injects
 | `DB_NAME` | `secrets` | `aws_secretsmanager_secret.db_name` |
 | `DB_USER` | `secrets` | `aws_secretsmanager_secret.db_user` |
 | `DB_PASSWORD` | `secrets` | `aws_secretsmanager_secret.db_password` |
-| `REDIS_URL` | `secrets` | `aws_secretsmanager_secret.redis_url` |
+| `REDIS_HOST` / `REDIS_PORT` / `REDIS_AUTH_TOKEN` | `secrets` | `aws_secretsmanager_secret.redis_host` / `redis_port` / `redis_auth_token` (API builds `rediss://` URL at runtime) |
 | `HMAC_KEY` | `secrets` | `aws_secretsmanager_secret.hmac_key` |
 | `SENDGRID_API_KEY` | `secrets` | `aws_secretsmanager_secret.sendgrid_api_key` |
 | `SENDGRID_KEY_ROTATED_AT` | `secrets` | `aws_secretsmanager_secret.sendgrid_key_rotated_at` |
@@ -552,4 +551,9 @@ All six checks should pass within 5 minutes of the deployment completing.
 >
 > | Date | Follower | Environment | Issues found | PR |
 > |---|---|---|---|---|
-> | *(not yet validated)* | | | | |
+> | *(not yet validated)* | | | | |
+### Alerting, encryption and RDS settings
+
+- Set `alert_email_endpoints` / `alert_https_endpoints` per environment to subscribe to the alerts SNS topic (email, PagerDuty, Slack webhook). Email subscribers must confirm the subscription.
+- Secrets Manager secrets and the ECS log group are encrypted with a per-environment CMK (key rotation on). Prod log retention is 365 days. Set `secret_rotation_lambda_arn` to enable 30-day rotation of the DB and Redis secrets.
+- RDS uses `engine_version = "15"` with automatic minor upgrades, a stable final snapshot name, Performance Insights and CloudWatch log exports.

infrastructure/terraform/main.tf

@@ -123,7 +123,9 @@ module "ecs" {
   db_name               = var.db_name
   db_user               = var.db_username
   db_password           = var.db_password
-  redis_url             = module.redis.redis_url
+  redis_host            = module.redis.endpoint
+  redis_port            = module.redis.port
+  redis_auth_token      = var.redis_auth_token
   acm_certificate_arn   = var.acm_certificate_arn
   hmac_key              = var.hmac_key
   sendgrid_api_key      = var.sendgrid_api_key
@@ -138,4 +140,7 @@ module "monitoring" {
   environment = var.environment
   ecs_cluster = module.ecs.cluster_name
   ecs_service = module.ecs.service_name
+
+  alert_email_endpoints = var.alert_email_endpoints
+  alert_https_endpoints = var.alert_https_endpoints
 }

infrastructure/terraform/modules/ecs/main.tf

@@ -34,11 +34,34 @@ variable "api_memory" {
   type = number
 }
 
-variable "redis_url" {
+# Redis connection components. The REDIS_URL is assembled by the application at
+# runtime, so the credential-bearing URL never appears in Terraform state.
+variable "redis_host" {
+  type = string
+}
+
+variable "redis_port" {
+  type    = number
+  default = 6379
+}
+
+variable "redis_auth_token" {
   type      = string
   sensitive = true
 }
 
+variable "secret_rotation_lambda_arn" {
+  type        = string
+  default     = null
+  description = "ARN of a rotation Lambda for DB/Redis secrets. When set, 30-day automatic rotation is enabled."
+}
+
+variable "log_retention_days" {
+  type        = number
+  default     = null
+  description = "CloudWatch log retention. Defaults to 365 in prod and 7 elsewhere."
+}
+
 # Individual database credential components.
 # Each is stored as its own Secrets Manager secret so they can be rotated
 # independently, and the assembled connection string (containing the password)
@@ -137,9 +160,46 @@ resource "aws_ecs_cluster" "main" {
   })
 }
 
+# Per-environment customer-managed key for secrets and logs.
+resource "aws_kms_key" "main" {
+  description             = "predictiq-${var.environment} secrets and log encryption"
+  enable_key_rotation     = true
+  deletion_window_in_days = 30
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Sid       = "EnableRootAccount"
+        Effect    = "Allow"
+        Principal = { AWS = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:root" }
+        Action    = "kms:*"
+        Resource  = "*"
+      },
+      {
+        Sid       = "AllowCloudWatchLogs"
+        Effect    = "Allow"
+        Principal = { Service = "logs.${data.aws_region.current.name}.amazonaws.com" }
+        Action    = ["kms:Encrypt*", "kms:Decrypt*", "kms:ReEncrypt*", "kms:GenerateDataKey*", "kms:Describe*"]
+        Resource  = "*"
+      }
+    ]
+  })
+
+  tags = merge(local.common_tags, {
+    Name = "predictiq-${var.environment}-kms"
+  })
+}
+
+resource "aws_kms_alias" "main" {
+  name          = "alias/predictiq-${var.environment}"
+  target_key_id = aws_kms_key.main.key_id
+}
+
 resource "aws_cloudwatch_log_group" "ecs" {
   name              = "/ecs/predictiq-${var.environment}"
-  retention_in_days = var.environment == "prod" ? 30 : 7
+  retention_in_days = coalesce(var.log_retention_days, var.environment == "prod" ? 365 : 7)
+  kms_key_id        = aws_kms_key.main.arn
 
   tags = merge(local.common_tags, {
     Name = "predictiq-${var.environment}-logs"
@@ -204,8 +264,16 @@ resource "aws_ecs_task_definition" "api" {
           valueFrom = aws_secretsmanager_secret.db_password.arn
         },
         {
-          name      = "REDIS_URL"
-          valueFrom = aws_secretsmanager_secret.redis_url.arn
+          name      = "REDIS_HOST"
+          valueFrom = aws_secretsmanager_secret.redis_host.arn
+        },
+        {
+          name      = "REDIS_PORT"
+          valueFrom = aws_secretsmanager_secret.redis_port.arn
+        },
+        {
+          name      = "REDIS_AUTH_TOKEN"
+          valueFrom = aws_secretsmanager_secret.redis_auth_token.arn
         },
         {
           name      = "HMAC_KEY"
@@ -394,20 +462,53 @@ resource "aws_ecs_service" "api" {
 # ECS task via the `secrets` block above.  No plaintext secrets are passed
 # through Terraform environment variables or ECS `environment` blocks.
 
-resource "aws_secretsmanager_secret" "redis_url" {
-  name        = "predictiq/${var.environment}/redis-url"
-  description = "Redis connection URL for the predictIQ API"
+resource "aws_secretsmanager_secret" "redis_host" {
+  name       = "predictiq/${var.environment}/redis-host"
+  kms_key_id = aws_kms_key.main.arn
+  tags       = merge(local.common_tags, { Name = "predictiq-${var.environment}-redis-host" })
+}
+
+resource "aws_secretsmanager_secret_version" "redis_host" {
+  secret_id     = aws_secretsmanager_secret.redis_host.id
+  secret_string = var.redis_host
+}
+
+resource "aws_secretsmanager_secret" "redis_port" {
+  name       = "predictiq/${var.environment}/redis-port"
+  kms_key_id = aws_kms_key.main.arn
+  tags       = merge(local.common_tags, { Name = "predictiq-${var.environment}-redis-port" })
+}
+
+resource "aws_secretsmanager_secret_version" "redis_port" {
+  secret_id     = aws_secretsmanager_secret.redis_port.id
+  secret_string = tostring(var.redis_port)
+}
+
+resource "aws_secretsmanager_secret" "redis_auth_token" {
+  name        = "predictiq/${var.environment}/redis-auth-token"
+  description = "Redis AUTH token. The application assembles REDIS_URL at runtime."
+  kms_key_id  = aws_kms_key.main.arn
 
   tags = merge(local.common_tags, {
-    Name            = "predictiq-${var.environment}-redis-url"
-    SecretType      = "connection-string"
-    RotationEnabled = "false"
+    Name            = "predictiq-${var.environment}-redis-auth-token"
+    SecretType      = "credential"
+    RotationEnabled = var.secret_rotation_lambda_arn != null ? "true" : "false"
   })
 }
 
-resource "aws_secretsmanager_secret_version" "redis_url" {
-  secret_id     = aws_secretsmanager_secret.redis_url.id
-  secret_string = var.redis_url
+resource "aws_secretsmanager_secret_version" "redis_auth_token" {
+  secret_id     = aws_secretsmanager_secret.redis_auth_token.id
+  secret_string = var.redis_auth_token
+}
+
+resource "aws_secretsmanager_secret_rotation" "redis_auth_token" {
+  count               = var.secret_rotation_lambda_arn != null ? 1 : 0
+  secret_id           = aws_secretsmanager_secret.redis_auth_token.id
+  rotation_lambda_arn = var.secret_rotation_lambda_arn
+
+  rotation_rules {
+    automatically_after_days = 30
+  }
 }
 
 # ── Database credentials (individual secrets) ────────────────────────────────
@@ -418,6 +519,7 @@ resource "aws_secretsmanager_secret_version" "redis_url" {
 # inside a secrecy::SecretString — see services/api/src/config.rs.
 
 resource "aws_secretsmanager_secret" "db_host" {
+  kms_key_id = aws_kms_key.main.arn
   name = "predictiq/${var.environment}/db-host"
   tags = merge(local.common_tags, { Name = "predictiq-${var.environment}-db-host" })
 }
@@ -428,6 +530,7 @@ resource "aws_secretsmanager_secret_version" "db_host" {
 }
 
 resource "aws_secretsmanager_secret" "db_port" {
+  kms_key_id = aws_kms_key.main.arn
   name = "predictiq/${var.environment}/db-port"
   tags = merge(local.common_tags, { Name = "predictiq-${var.environment}-db-port" })
 }
@@ -438,6 +541,7 @@ resource "aws_secretsmanager_secret_version" "db_port" {
 }
 
 resource "aws_secretsmanager_secret" "db_name" {
+  kms_key_id = aws_kms_key.main.arn
   name = "predictiq/${var.environment}/db-name"
   tags = merge(local.common_tags, { Name = "predictiq-${var.environment}-db-name" })
 }
@@ -448,6 +552,7 @@ resource "aws_secretsmanager_secret_version" "db_name" {
 }
 
 resource "aws_secretsmanager_secret" "db_user" {
+  kms_key_id = aws_kms_key.main.arn
   name = "predictiq/${var.environment}/db-user"
   tags = merge(local.common_tags, { Name = "predictiq-${var.environment}-db-user" })
 }
@@ -458,11 +563,22 @@ resource "aws_secretsmanager_secret_version" "db_user" {
 }
 
 resource "aws_secretsmanager_secret" "db_password" {
+  kms_key_id = aws_kms_key.main.arn
   name        = "predictiq/${var.environment}/db-password"
   description = "PostgreSQL password for the predictIQ application user. Never assembled into DATABASE_URL."
   tags = merge(local.common_tags, { Name = "predictiq-${var.environment}-db-password" })
 }
 
+resource "aws_secretsmanager_secret_rotation" "db_password" {
+  count               = var.secret_rotation_lambda_arn != null ? 1 : 0
+  secret_id           = aws_secretsmanager_secret.db_password.id
+  rotation_lambda_arn = var.secret_rotation_lambda_arn
+
+  rotation_rules {
+    automatically_after_days = 30
+  }
+}
+
 resource "aws_secretsmanager_secret_version" "db_password" {
   secret_id     = aws_secretsmanager_secret.db_password.id
   secret_string = var.db_password
@@ -471,6 +587,7 @@ resource "aws_secretsmanager_secret_version" "db_password" {
 # ── SendGrid ────────────────────────────────────────────────────────────────
 
 resource "aws_secretsmanager_secret" "hmac_key" {
+  kms_key_id = aws_kms_key.main.arn
   name        = "predictiq/${var.environment}/hmac-key"
   description = "HMAC secret key used to sign API payloads and webhook signatures"
 
@@ -488,6 +605,7 @@ resource "aws_secretsmanager_secret_version" "hmac_key" {
 }
 
 resource "aws_secretsmanager_secret" "sendgrid_api_key" {
+  kms_key_id = aws_kms_key.main.arn
   name        = "predictiq/${var.environment}/sendgrid-api-key"
   description = "SendGrid API key used by the predictIQ API for transactional email. Rotate every 90 days — see docs/runbooks/sendgrid-api-key-rotation.md."
 
@@ -507,6 +625,7 @@ resource "aws_secretsmanager_secret_version" "sendgrid_api_key" {
 # Stores only the rotation date (YYYY-MM-DD string) — not the key itself.
 # The application reads this at startup to warn if the key is older than 90 days.
 resource "aws_secretsmanager_secret" "sendgrid_key_rotated_at" {
+  kms_key_id = aws_kms_key.main.arn
   name        = "predictiq/${var.environment}/sendgrid-key-rotated-at"
   description = "ISO-8601 date (YYYY-MM-DD) when SENDGRID_API_KEY was last rotated. Update this alongside the key itself."
 
@@ -524,6 +643,7 @@ resource "aws_secretsmanager_secret_version" "sendgrid_key_rotated_at" {
 }
 
 resource "aws_secretsmanager_secret" "api_signing_key" {
+  kms_key_id = aws_kms_key.main.arn
   name        = "predictiq/${var.environment}/api-signing-key"
   description = "Private key used to sign API responses"
 
@@ -588,12 +708,19 @@ resource "aws_iam_role_policy" "ecs_task_execution_secrets" {
           aws_secretsmanager_secret.db_name.arn,
           aws_secretsmanager_secret.db_user.arn,
           aws_secretsmanager_secret.db_password.arn,
-          aws_secretsmanager_secret.redis_url.arn,
+          aws_secretsmanager_secret.redis_host.arn,
+          aws_secretsmanager_secret.redis_port.arn,
+          aws_secretsmanager_secret.redis_auth_token.arn,
           aws_secretsmanager_secret.hmac_key.arn,
           aws_secretsmanager_secret.sendgrid_api_key.arn,
           aws_secretsmanager_secret.sendgrid_key_rotated_at.arn,
           aws_secretsmanager_secret.api_signing_key.arn,
         ]
+      },
+      {
+        Effect   = "Allow"
+        Action   = ["kms:Decrypt"]
+        Resource = [aws_kms_key.main.arn]
       }
     ]
   })
@@ -626,6 +753,12 @@ resource "aws_iam_role" "ecs_task_role" {
 
 data "aws_region" "current" {}
 
+data "aws_caller_identity" "current" {}
+
+output "kms_key_arn" {
+  value = aws_kms_key.main.arn
+}
+
 output "cluster_name" {
   value = aws_ecs_cluster.main.name
 }
@@ -647,7 +780,9 @@ output "secret_arns" {
     db_name          = aws_secretsmanager_secret.db_name.arn
     db_user          = aws_secretsmanager_secret.db_user.arn
     db_password      = aws_secretsmanager_secret.db_password.arn
-    redis_url        = aws_secretsmanager_secret.redis_url.arn
+    redis_host       = aws_secretsmanager_secret.redis_host.arn
+    redis_port       = aws_secretsmanager_secret.redis_port.arn
+    redis_auth_token = aws_secretsmanager_secret.redis_auth_token.arn
     hmac_key         = aws_secretsmanager_secret.hmac_key.arn
     sendgrid_api_key = aws_secretsmanager_secret.sendgrid_api_key.arn
     api_signing_key  = aws_secretsmanager_secret.api_signing_key.arn

infrastructure/terraform/modules/monitoring/main.tf

@@ -10,6 +10,18 @@ variable "ecs_service" {
   type = string
 }
 
+variable "alert_email_endpoints" {
+  type        = list(string)
+  default     = []
+  description = "Email addresses subscribed to the alerts SNS topic."
+}
+
+variable "alert_https_endpoints" {
+  type        = list(string)
+  default     = []
+  description = "HTTPS endpoints (PagerDuty/Slack/webhook) subscribed to the alerts SNS topic."
+}
+
 locals {
   common_tags = {
     Project   = "predictiq"
@@ -66,6 +78,20 @@ resource "aws_sns_topic" "alerts" {
   )
 }
 
+resource "aws_sns_topic_subscription" "email" {
+  for_each  = toset(var.alert_email_endpoints)
+  topic_arn = aws_sns_topic.alerts.arn
+  protocol  = "email"
+  endpoint  = each.value
+}
+
+resource "aws_sns_topic_subscription" "https" {
+  for_each  = toset(var.alert_https_endpoints)
+  topic_arn = aws_sns_topic.alerts.arn
+  protocol  = "https"
+  endpoint  = each.value
+}
+
 resource "aws_cloudwatch_metric_alarm" "ecs_cpu" {
   alarm_name          = "predictiq-${var.environment}-ecs-cpu-high"
   comparison_operator = "GreaterThanThreshold"