diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml index f5762a8d..7d4e275f 100644 --- a/.github/workflows/terraform.yml +++ b/.github/workflows/terraform.yml @@ -57,6 +57,14 @@ jobs: if: steps.fmt.outcome == 'failure' run: exit 1 + terraform-hardening: + name: Terraform Hardening Check + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - run: python3 scripts/check_terraform_hardening.py + terraform-invariants: name: Terraform Invariant Checks runs-on: ubuntu-latest @@ -215,7 +223,7 @@ jobs: all-terraform-checks-passed: name: All Terraform Checks Passed - needs: [terraform-fmt, terraform-validate, terraform-plan] + needs: [terraform-fmt, terraform-hardening, terraform-invariants, terraform-validate, terraform-plan] runs-on: ubuntu-latest timeout-minutes: 20 steps: diff --git a/Dockerfile b/Dockerfile index a65e7290..4946b5a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,6 +37,7 @@ FROM debian:bookworm-slim@sha256:7c7b2c966bc9ee8cedfeef67e0e279108992c77681fa595 RUN apt-get update \ && apt-get install -y --no-install-recommends \ ca-certificates \ + curl \ libssl3 \ && rm -rf /var/lib/apt/lists/* diff --git a/infrastructure/README.md b/infrastructure/README.md index a52d5296..10d5728f 100644 --- a/infrastructure/README.md +++ b/infrastructure/README.md @@ -149,6 +149,8 @@ See `environments/README.md` for detailed environment management instructions. - Configurable CIDR blocks - Public and private subnets - NAT Gateway for private subnet egress +- Gateway endpoint (S3) and interface endpoints (ECR api/dkr, CloudWatch Logs, Secrets Manager) so AWS service traffic bypasses NAT +- VPC flow logs (all traffic) to CloudWatch Logs ### RDS Module - PostgreSQL database @@ -163,10 +165,13 @@ See `environments/README.md` for detailed environment management instructions. - **Failover RTO**: AWS promotes a read replica to primary in approximately **60–120 seconds**. During this window, write operations will fail; read-only operations served by replicas remain available. Applications should implement retry logic with exponential backoff (recommended: 3 retries, starting at 200 ms) to handle the failover window gracefully. - At-rest and in-transit encryption enabled - Subnet group for VPC placement +- Daily snapshots (`snapshot_retention_limit` = 7 days, window 01:00-02:00 UTC) and `auto_minor_version_upgrade` +- Parameter group family is derived from `redis_engine_version` (e.g. `7.0` -> `default.redis7`, `6.2` -> `default.redis6.x`) ### ECS Module - Fargate launch type -- Application Load Balancer +- Application Load Balancer with access logs (encrypted S3 bucket), `drop_invalid_header_fields`, deletion protection in prod, and an AWS managed-rules WAF (Common + Known Bad Inputs) +- Container health check on `/health/live` (15 s interval, 2 retries); target group probes every 15 s with `unhealthy_threshold = 2`; `stopTimeout` and `deregistration_delay` follow `shutdown_timeout_seconds` (default 30, matching `SHUTDOWN_TIMEOUT_SECS`) - Auto-scaling policies - CloudWatch logging diff --git a/infrastructure/terraform/modules/ecs/main.tf b/infrastructure/terraform/modules/ecs/main.tf index 7ace2d11..19ca9a34 100644 --- a/infrastructure/terraform/modules/ecs/main.tf +++ b/infrastructure/terraform/modules/ecs/main.tf @@ -214,6 +214,18 @@ variable "ecs_tasks_sg_id" { description = "Security group ID of the ECS tasks (managed at root level)" } +variable "shutdown_timeout_seconds" { + type = number + default = 30 + description = "App graceful-shutdown timeout (SHUTDOWN_TIMEOUT_SECS). Drives ECS stopTimeout and ALB deregistration delay." +} + +variable "alb_deletion_protection" { + type = bool + default = null + description = "Enable ALB deletion protection. Defaults to true in prod, false elsewhere." +} + locals { extra_secrets = { api_keys = { env = "API_KEYS", value = var.api_keys } @@ -307,6 +319,14 @@ resource "aws_ecs_task_definition" "api" { name = "api" image = var.api_image_uri essential = true + stopTimeout = var.shutdown_timeout_seconds + healthCheck = { + command = ["CMD-SHELL", "curl -fsS http://localhost:${var.api_container_port}/health/live || exit 1"] + interval = 15 + timeout = 5 + retries = 2 + startPeriod = 30 + } portMappings = [ { containerPort = var.api_container_port @@ -451,6 +471,150 @@ resource "aws_security_group_rule" "ecs_tasks_ingress_alb" { source_security_group_id = aws_security_group.alb.id } +# ── ALB access logs, WAF ─────────────────────────────────────────────────────── + +data "aws_elb_service_account" "main" {} + +data "aws_caller_identity" "current" {} + +resource "aws_s3_bucket" "alb_logs" { + bucket = "predictiq-${var.environment}-alb-logs-${data.aws_caller_identity.current.account_id}" + force_destroy = var.environment != "prod" + + tags = merge(local.common_tags, { + Name = "predictiq-${var.environment}-alb-logs" + }) +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "alb_logs" { + bucket = aws_s3_bucket.alb_logs.id + + # ALB access logs only support SSE-S3 (AES256); SSE-KMS is not supported. + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} + +resource "aws_s3_bucket_public_access_block" "alb_logs" { + bucket = aws_s3_bucket.alb_logs.id + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +resource "aws_s3_bucket_lifecycle_configuration" "alb_logs" { + bucket = aws_s3_bucket.alb_logs.id + + rule { + id = "expire-logs" + status = "Enabled" + + filter {} + + expiration { + days = var.environment == "prod" ? 90 : 14 + } + } +} + +resource "aws_s3_bucket_policy" "alb_logs" { + bucket = aws_s3_bucket.alb_logs.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "AllowALBLogDelivery" + Effect = "Allow" + Principal = { AWS = data.aws_elb_service_account.main.arn } + Action = "s3:PutObject" + Resource = "${aws_s3_bucket.alb_logs.arn}/alb/AWSLogs/${data.aws_caller_identity.current.account_id}/*" + }, + { + Sid = "DenyInsecureTransport" + Effect = "Deny" + Principal = "*" + Action = "s3:*" + Resource = [aws_s3_bucket.alb_logs.arn, "${aws_s3_bucket.alb_logs.arn}/*"] + Condition = { Bool = { "aws:SecureTransport" = "false" } } + } + ] + }) + + depends_on = [aws_s3_bucket_public_access_block.alb_logs] +} + +resource "aws_wafv2_web_acl" "alb" { + name = "predictiq-${var.environment}-alb-waf" + scope = "REGIONAL" + + default_action { + allow {} + } + + rule { + name = "AWSManagedRulesCommonRuleSet" + priority = 1 + + override_action { + none {} + } + + statement { + managed_rule_group_statement { + name = "AWSManagedRulesCommonRuleSet" + vendor_name = "AWS" + } + } + + visibility_config { + cloudwatch_metrics_enabled = true + metric_name = "common-rules" + sampled_requests_enabled = true + } + } + + rule { + name = "AWSManagedRulesKnownBadInputsRuleSet" + priority = 2 + + override_action { + none {} + } + + statement { + managed_rule_group_statement { + name = "AWSManagedRulesKnownBadInputsRuleSet" + vendor_name = "AWS" + } + } + + visibility_config { + cloudwatch_metrics_enabled = true + metric_name = "known-bad-inputs" + sampled_requests_enabled = true + } + } + + visibility_config { + cloudwatch_metrics_enabled = true + metric_name = "predictiq-${var.environment}-alb-waf" + sampled_requests_enabled = true + } + + tags = merge(local.common_tags, { + Name = "predictiq-${var.environment}-alb-waf" + }) +} + +resource "aws_wafv2_web_acl_association" "alb" { + resource_arn = aws_lb.main.arn + web_acl_arn = aws_wafv2_web_acl.alb.arn +} + resource "aws_lb" "main" { name = "predictiq-${var.environment}-alb" internal = false @@ -458,6 +622,17 @@ resource "aws_lb" "main" { security_groups = [aws_security_group.alb.id] subnets = var.public_subnet_ids + enable_deletion_protection = coalesce(var.alb_deletion_protection, var.environment == "prod") + drop_invalid_header_fields = true + + access_logs { + bucket = aws_s3_bucket.alb_logs.id + prefix = "alb" + enabled = true + } + + depends_on = [aws_s3_bucket_policy.alb_logs] + tags = merge(local.common_tags, { Name = "predictiq-${var.environment}-alb" }) @@ -470,11 +645,14 @@ resource "aws_lb_target_group" "api" { vpc_id = var.vpc_id target_type = "ip" + # Align with the app's graceful-shutdown timeout instead of the 300 s default. + deregistration_delay = var.shutdown_timeout_seconds + health_check { healthy_threshold = 2 - unhealthy_threshold = 3 + unhealthy_threshold = 2 timeout = 5 - interval = 60 + interval = 15 path = "/health/ready" matcher = "200" } diff --git a/infrastructure/terraform/modules/redis/main.tf b/infrastructure/terraform/modules/redis/main.tf index 857737e8..cc0cce6c 100644 --- a/infrastructure/terraform/modules/redis/main.tf +++ b/infrastructure/terraform/modules/redis/main.tf @@ -1,3 +1,139 @@ +variable "environment" { + type = string +} + +variable "vpc_id" { + type = string +} + +variable "subnet_ids" { + type = list(string) +} + +variable "node_type" { + type = string +} + +variable "num_cache_clusters" { + type = number + default = 2 +} + +variable "engine_version" { + type = string +} + +variable "redis_auth_token" { + type = string + sensitive = true + description = "AUTH token required to authenticate to the Redis replication group" +} + +variable "redis_multi_az_enabled" { + type = bool + default = true + description = "Enable Multi-AZ automatic failover for the Redis replication group." +} + +variable "snapshot_retention_limit" { + type = number + default = 7 + description = "Days to retain automatic Redis snapshots (0 disables backups)." +} + +variable "ecs_tasks_sg_id" { + type = string + description = "Security group ID of the ECS tasks that are allowed to connect" +} + +locals { + # e.g. "7.0" -> "redis7", "6.2" -> "redis6.x" + redis_major = tonumber(split(".", var.engine_version)[0]) + parameter_group_family = local.redis_major >= 7 ? "redis${local.redis_major}" : "redis${local.redis_major}.x" + + common_tags = { + Project = "predictiq" + Environment = var.environment + Owner = "infrastructure-team" + ManagedBy = "terraform" + } +} + +resource "aws_elasticache_subnet_group" "main" { + name = "predictiq-${var.environment}-redis-subnet" + subnet_ids = var.subnet_ids + + tags = merge( + local.common_tags, + { + Name = "predictiq-${var.environment}-redis-subnet-group" + } + ) +} + +resource "aws_security_group" "redis" { + name = "predictiq-${var.environment}-redis-sg" + vpc_id = var.vpc_id + + # Inbound Redis from ECS tasks only + ingress { + from_port = 6379 + to_port = 6379 + protocol = "tcp" + security_groups = [var.ecs_tasks_sg_id] + } + + tags = merge( + local.common_tags, + { + Name = "predictiq-${var.environment}-redis-sg" + } + ) +} + +resource "aws_elasticache_replication_group" "main" { + replication_group_id = "predictiq-${var.environment}" + description = "PredictIQ Redis replication group (${var.environment})" + + engine = "redis" + engine_version = var.engine_version + node_type = var.node_type + num_cache_clusters = var.num_cache_clusters >= 2 ? var.num_cache_clusters : 2 + parameter_group_name = "default.${local.parameter_group_family}" + port = 6379 + + subnet_group_name = aws_elasticache_subnet_group.main.name + security_group_ids = [aws_security_group.redis.id] + + automatic_failover_enabled = var.redis_multi_az_enabled + multi_az_enabled = var.redis_multi_az_enabled + + at_rest_encryption_enabled = true + transit_encryption_enabled = true + auth_token = var.redis_auth_token + + maintenance_window = "mon:03:00-mon:04:00" + snapshot_window = "01:00-02:00" + snapshot_retention_limit = var.snapshot_retention_limit + auto_minor_version_upgrade = true + + tags = merge( + local.common_tags, + { + Name = "predictiq-${var.environment}-redis" + } + ) +} + +output "sg_id" { + value = aws_security_group.redis.id +} + +output "endpoint" { + value = aws_elasticache_replication_group.main.primary_endpoint_address + sensitive = true +} + output "redis_url" { value = "rediss://:${var.redis_auth_token}@${aws_elasticache_replication_group.main.primary_endpoint_address}:6379" sensitive = true diff --git a/infrastructure/terraform/modules/vpc/main.tf b/infrastructure/terraform/modules/vpc/main.tf index 2940fc29..8f3fc14e 100644 --- a/infrastructure/terraform/modules/vpc/main.tf +++ b/infrastructure/terraform/modules/vpc/main.tf @@ -146,6 +146,107 @@ resource "aws_route_table_association" "private" { route_table_id = aws_route_table.private[count.index].id } +# ── VPC endpoints (keep AWS control-plane traffic off the NAT gateways) ──────── + +data "aws_region" "current" {} + +resource "aws_security_group" "vpc_endpoints" { + name = "predictiq-${var.environment}-vpce-sg" + vpc_id = aws_vpc.main.id + + ingress { + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = [var.cidr_block] + } + + tags = merge(local.common_tags, { + Name = "predictiq-${var.environment}-vpce-sg" + }) +} + +resource "aws_vpc_endpoint" "s3" { + vpc_id = aws_vpc.main.id + service_name = "com.amazonaws.${data.aws_region.current.name}.s3" + vpc_endpoint_type = "Gateway" + route_table_ids = aws_route_table.private[*].id + + tags = merge(local.common_tags, { + Name = "predictiq-${var.environment}-vpce-s3" + }) +} + +resource "aws_vpc_endpoint" "interface" { + for_each = toset(["ecr.api", "ecr.dkr", "logs", "secretsmanager"]) + + vpc_id = aws_vpc.main.id + service_name = "com.amazonaws.${data.aws_region.current.name}.${each.key}" + vpc_endpoint_type = "Interface" + subnet_ids = aws_subnet.private[*].id + security_group_ids = [aws_security_group.vpc_endpoints.id] + private_dns_enabled = true + + tags = merge(local.common_tags, { + Name = "predictiq-${var.environment}-vpce-${each.key}" + }) +} + +# ── VPC flow logs ────────────────────────────────────────────────────────────── + +resource "aws_cloudwatch_log_group" "flow_logs" { + name = "/vpc/predictiq-${var.environment}/flow-logs" + retention_in_days = var.environment == "prod" ? 90 : 14 + + tags = merge(local.common_tags, { + Name = "predictiq-${var.environment}-vpc-flow-logs" + }) +} + +resource "aws_iam_role" "flow_logs" { + name = "predictiq-${var.environment}-vpc-flow-logs-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { Service = "vpc-flow-logs.amazonaws.com" } + }] + }) + + tags = local.common_tags +} + +resource "aws_iam_role_policy" "flow_logs" { + name = "predictiq-${var.environment}-vpc-flow-logs-policy" + role = aws_iam_role.flow_logs.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Action = [ + "logs:CreateLogStream", + "logs:PutLogEvents", + "logs:DescribeLogStreams" + ] + Resource = "${aws_cloudwatch_log_group.flow_logs.arn}:*" + }] + }) +} + +resource "aws_flow_log" "main" { + vpc_id = aws_vpc.main.id + traffic_type = "ALL" + log_destination = aws_cloudwatch_log_group.flow_logs.arn + iam_role_arn = aws_iam_role.flow_logs.arn + + tags = merge(local.common_tags, { + Name = "predictiq-${var.environment}-vpc-flow-log" + }) +} + data "aws_availability_zones" "available" { state = "available" } diff --git a/scripts/check_terraform_hardening.py b/scripts/check_terraform_hardening.py new file mode 100644 index 00000000..805b849e --- /dev/null +++ b/scripts/check_terraform_hardening.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Regression check for ALB/ECS/VPC/Redis hardening (issues #1778-#1781).""" +import pathlib +import sys + +TF = pathlib.Path(__file__).resolve().parent.parent / "infrastructure" / "terraform" / "modules" + +REQUIRED = { + "ecs/main.tf": [ + "access_logs {", + "enable_deletion_protection", + "drop_invalid_header_fields = true", + "aws_wafv2_web_acl_association", + "AWSManagedRulesCommonRuleSet", + "healthCheck", + "/health/live", + "stopTimeout", + "deregistration_delay", + "unhealthy_threshold = 2", + "interval = 15", + ], + "vpc/main.tf": ["aws_vpc_endpoint", "aws_flow_log", "ecr.dkr", "secretsmanager", "logs"], + "redis/main.tf": ["snapshot_retention_limit", "auto_minor_version_upgrade = true", "parameter_group_family"], +} +FORBIDDEN = {"redis/main.tf": ['"default.redis7"']} + +errors = [] +for rel, needles in REQUIRED.items(): + text = (TF / rel).read_text() + errors += [f"{rel}: missing `{n}`" for n in needles if n not in text] +for rel, needles in FORBIDDEN.items(): + text = (TF / rel).read_text() + errors += [f"{rel}: must not contain {n}" for n in needles if n in text] + +if errors: + print("\n".join(errors)) + sys.exit(1) +print("terraform hardening checks passed")