Disaster Recovery Patterns¶

# dr-pilot-light.tf

# Primary Region Resources
provider "aws" {
  alias  = "primary"
  region = "us-east-1"
}

# DR Region Resources
provider "aws" {
  alias  = "dr"
  region = "us-west-2"
}

# Primary RDS with Multi-AZ
resource "aws_db_instance" "primary" {
  provider               = aws.primary
  identifier             = "${var.project_name}-primary-db"
  engine                 = "postgres"
  engine_version         = "15.4"
  instance_class         = "db.r6g.large"
  allocated_storage      = 100
  storage_encrypted      = true
  multi_az               = true

  backup_retention_period = 7
  backup_window           = "03:00-04:00"

  # Enable automated backups to DR region
  replicate_source_db = null

  tags = {
    Name        = "${var.project_name}-primary-db"
    Environment = "production"
    DR          = "primary"
  }
}

# Cross-Region Read Replica (Pilot Light)
resource "aws_db_instance" "dr_replica" {
  provider             = aws.dr
  identifier           = "${var.project_name}-dr-replica"
  replicate_source_db  = aws_db_instance.primary.arn
  instance_class       = "db.r6g.medium"  # Smaller for cost savings
  storage_encrypted    = true
  skip_final_snapshot  = true

  # Can be promoted to standalone in DR scenario
  tags = {
    Name        = "${var.project_name}-dr-replica"
    Environment = "dr"
    DR          = "replica"
  }
}

# S3 Cross-Region Replication
resource "aws_s3_bucket" "primary" {
  provider = aws.primary
  bucket   = "${var.project_name}-primary-assets"
}

resource "aws_s3_bucket" "dr" {
  provider = aws.dr
  bucket   = "${var.project_name}-dr-assets"
}

resource "aws_s3_bucket_replication_configuration" "replication" {
  provider = aws.primary
  bucket   = aws_s3_bucket.primary.id
  role     = aws_iam_role.replication.arn

  rule {
    id     = "replicate-all"
    status = "Enabled"

    destination {
      bucket        = aws_s3_bucket.dr.arn
      storage_class = "STANDARD"

      encryption_configuration {
        replica_kms_key_id = aws_kms_key.dr.arn
      }
    }

    source_selection_criteria {
      sse_kms_encrypted_objects {
        status = "Enabled"
      }
    }
  }
}

# Route 53 Health Check and Failover
resource "aws_route53_health_check" "primary" {
  fqdn              = "primary.${var.domain}"
  port              = 443
  type              = "HTTPS"
  resource_path     = "/health"
  failure_threshold = 3
  request_interval  = 30

  tags = {
    Name = "primary-health-check"
  }
}

resource "aws_route53_record" "failover_primary" {
  zone_id = var.zone_id
  name    = "app.${var.domain}"
  type    = "A"

  failover_routing_policy {
    type = "PRIMARY"
  }

  set_identifier = "primary"
  health_check_id = aws_route53_health_check.primary.id

  alias {
    name                   = aws_lb.primary.dns_name
    zone_id                = aws_lb.primary.zone_id
    evaluate_target_health = true
  }
}

resource "aws_route53_record" "failover_dr" {
  zone_id = var.zone_id
  name    = "app.${var.domain}"
  type    = "A"

  failover_routing_policy {
    type = "SECONDARY"
  }

  set_identifier = "dr"

  alias {
    name                   = aws_lb.dr.dns_name
    zone_id                = aws_lb.dr.zone_id
    evaluate_target_health = true
  }
}

# DR Application (Stopped by default)
resource "aws_autoscaling_group" "dr" {
  provider            = aws.dr
  name                = "${var.project_name}-dr-asg"
  desired_capacity    = 0  # Start with zero instances
  min_size            = 0
  max_size            = 10
  vpc_zone_identifier = var.dr_subnet_ids

  launch_template {
    id      = aws_launch_template.dr.id
    version = "$Latest"
  }

  tag {
    key                 = "Name"
    value               = "${var.project_name}-dr"
    propagate_at_launch = true
  }
}

# Lambda for automated failover
resource "aws_lambda_function" "failover" {
  provider      = aws.primary
  function_name = "${var.project_name}-failover"
  role          = aws_iam_role.failover_lambda.arn
  handler       = "failover.handler"
  runtime       = "python3.11"

  filename         = "failover.zip"
  source_code_hash = filebase64sha256("failover.zip")

  environment {
    variables = {
      DR_ASG_NAME     = aws_autoscaling_group.dr.name
      DR_DB_IDENTIFIER = aws_db_instance.dr_replica.identifier
      DR_REGION       = "us-west-2"
    }
  }
}

# CloudWatch Alarm to trigger failover
resource "aws_cloudwatch_metric_alarm" "primary_unhealthy" {
  provider            = aws.primary
  alarm_name          = "primary-region-unhealthy"
  comparison_operator = "LessThanThreshold"
  evaluation_periods  = 3
  metric_name         = "HealthyHostCount"
  namespace           = "AWS/ApplicationELB"
  period              = 60
  statistic           = "Average"
  threshold           = 1

  dimensions = {
    LoadBalancer = aws_lb.primary.arn_suffix
    TargetGroup  = aws_lb_target_group.primary.arn_suffix
  }

  alarm_actions = [aws_sns_topic.dr_alerts.arn]
}

# aurora-global.tf

resource "aws_rds_global_cluster" "main" {
  global_cluster_identifier = "${var.project_name}-global"
  engine                    = "aurora-postgresql"
  engine_version            = "15.4"
  database_name             = "app"
  storage_encrypted         = true
}

# Primary cluster
resource "aws_rds_cluster" "primary" {
  provider                  = aws.primary
  cluster_identifier        = "${var.project_name}-primary"
  engine                    = aws_rds_global_cluster.main.engine
  engine_version            = aws_rds_global_cluster.main.engine_version
  global_cluster_identifier = aws_rds_global_cluster.main.id
  master_username           = "admin"
  master_password           = var.db_password
  database_name             = "app"

  backup_retention_period = 7
  preferred_backup_window = "03:00-04:00"

  db_subnet_group_name   = aws_db_subnet_group.primary.name
  vpc_security_group_ids = [aws_security_group.db_primary.id]
}

resource "aws_rds_cluster_instance" "primary" {
  count              = 2
  provider           = aws.primary
  identifier         = "${var.project_name}-primary-${count.index}"
  cluster_identifier = aws_rds_cluster.primary.id
  instance_class     = "db.r6g.large"
  engine             = aws_rds_cluster.primary.engine
}

# Secondary cluster (DR region)
resource "aws_rds_cluster" "secondary" {
  provider                  = aws.dr
  cluster_identifier        = "${var.project_name}-secondary"
  engine                    = aws_rds_global_cluster.main.engine
  engine_version            = aws_rds_global_cluster.main.engine_version
  global_cluster_identifier = aws_rds_global_cluster.main.id

  # Secondary cluster - no master credentials needed
  skip_final_snapshot = true

  db_subnet_group_name   = aws_db_subnet_group.dr.name
  vpc_security_group_ids = [aws_security_group.db_dr.id]

  depends_on = [aws_rds_cluster_instance.primary]
}

resource "aws_rds_cluster_instance" "secondary" {
  count              = 1  # Fewer instances for cost savings
  provider           = aws.dr
  identifier         = "${var.project_name}-secondary-${count.index}"
  cluster_identifier = aws_rds_cluster.secondary.id
  instance_class     = "db.r6g.medium"
  engine             = aws_rds_cluster.secondary.engine
}

# dynamodb-global.tf

resource "aws_dynamodb_table" "main" {
  provider         = aws.primary
  name             = "${var.project_name}-data"
  billing_mode     = "PAY_PER_REQUEST"
  hash_key         = "pk"
  range_key        = "sk"
  stream_enabled   = true
  stream_view_type = "NEW_AND_OLD_IMAGES"

  attribute {
    name = "pk"
    type = "S"
  }

  attribute {
    name = "sk"
    type = "S"
  }

  # Global table replication
  replica {
    region_name = "us-west-2"
  }

  replica {
    region_name = "eu-west-1"
  }

  point_in_time_recovery {
    enabled = true
  }

  tags = {
    Name = "${var.project_name}-data"
  }
}

# aws-backup.tf

resource "aws_backup_vault" "primary" {
  provider = aws.primary
  name     = "${var.project_name}-backup-vault"
}

resource "aws_backup_vault" "dr" {
  provider = aws.dr
  name     = "${var.project_name}-backup-vault-dr"
}

resource "aws_backup_plan" "main" {
  provider = aws.primary
  name     = "${var.project_name}-backup-plan"

  rule {
    rule_name         = "daily-backup"
    target_vault_name = aws_backup_vault.primary.name
    schedule          = "cron(0 3 * * ? *)"

    lifecycle {
      delete_after = 30
    }

    # Cross-region copy
    copy_action {
      destination_vault_arn = aws_backup_vault.dr.arn
      lifecycle {
        delete_after = 30
      }
    }
  }

  rule {
    rule_name         = "weekly-backup"
    target_vault_name = aws_backup_vault.primary.name
    schedule          = "cron(0 4 ? * SUN *)"

    lifecycle {
      delete_after = 90
    }

    copy_action {
      destination_vault_arn = aws_backup_vault.dr.arn
      lifecycle {
        delete_after = 90
      }
    }
  }
}

resource "aws_backup_selection" "main" {
  provider     = aws.primary
  iam_role_arn = aws_iam_role.backup.arn
  name         = "${var.project_name}-backup-selection"
  plan_id      = aws_backup_plan.main.id

  selection_tag {
    type  = "STRINGEQUALS"
    key   = "Backup"
    value = "true"
  }
}

# cloudsql-dr.tf

# Primary instance
resource "google_sql_database_instance" "primary" {
  name             = "${var.project_name}-primary"
  database_version = "POSTGRES_15"
  region           = var.primary_region

  settings {
    tier              = "db-custom-4-16384"
    availability_type = "REGIONAL"

    backup_configuration {
      enabled                        = true
      start_time                     = "03:00"
      point_in_time_recovery_enabled = true
      transaction_log_retention_days = 7

      backup_retention_settings {
        retained_backups = 30
        retention_unit   = "COUNT"
      }
    }

    ip_configuration {
      ipv4_enabled    = false
      private_network = google_compute_network.vpc.id
    }

    insights_config {
      query_insights_enabled  = true
      record_application_tags = true
    }
  }

  deletion_protection = true
}

# Cross-region read replica
resource "google_sql_database_instance" "replica" {
  name                 = "${var.project_name}-replica"
  master_instance_name = google_sql_database_instance.primary.name
  region               = var.dr_region
  database_version     = "POSTGRES_15"

  replica_configuration {
    failover_target = true
  }

  settings {
    tier              = "db-custom-2-8192"  # Smaller for cost
    availability_type = "ZONAL"

    ip_configuration {
      ipv4_enabled    = false
      private_network = google_compute_network.vpc.id
    }
  }

  deletion_protection = true
}

# global-lb.tf

# Health check
resource "google_compute_health_check" "app" {
  name               = "${var.project_name}-health-check"
  check_interval_sec = 10
  timeout_sec        = 5
  healthy_threshold  = 2
  unhealthy_threshold = 3

  https_health_check {
    port         = 443
    request_path = "/health"
  }
}

# Primary backend
resource "google_compute_backend_service" "primary" {
  name                  = "${var.project_name}-backend-primary"
  protocol              = "HTTPS"
  port_name             = "https"
  load_balancing_scheme = "EXTERNAL_MANAGED"
  timeout_sec           = 30
  health_checks         = [google_compute_health_check.app.id]

  backend {
    group           = google_compute_region_instance_group_manager.primary.instance_group
    balancing_mode  = "UTILIZATION"
    capacity_scaler = 1.0
  }

  circuit_breakers {
    max_connections = 1000
  }

  outlier_detection {
    consecutive_errors = 5
    interval {
      seconds = 10
    }
    base_ejection_time {
      seconds = 30
    }
  }
}

# DR backend
resource "google_compute_backend_service" "dr" {
  name                  = "${var.project_name}-backend-dr"
  protocol              = "HTTPS"
  port_name             = "https"
  load_balancing_scheme = "EXTERNAL_MANAGED"
  timeout_sec           = 30
  health_checks         = [google_compute_health_check.app.id]

  backend {
    group           = google_compute_region_instance_group_manager.dr.instance_group
    balancing_mode  = "UTILIZATION"
    capacity_scaler = 0.5  # Lower capacity for warm standby
  }
}

# URL map with failover
resource "google_compute_url_map" "app" {
  name            = "${var.project_name}-url-map"
  default_service = google_compute_backend_service.primary.id

  host_rule {
    hosts        = ["app.example.com"]
    path_matcher = "app-paths"
  }

  path_matcher {
    name            = "app-paths"
    default_service = google_compute_backend_service.primary.id

    route_rules {
      priority = 1
      service  = google_compute_backend_service.primary.id

      route_action {
        retry_policy {
          num_retries = 3
          retry_conditions = [
            "5xx",
            "reset",
            "connect-failure"
          ]
        }

        # Failover to DR if primary fails
        fault_injection_policy {
          abort {
            http_status = 503
            percentage  = 0
          }
        }
      }
    }
  }
}

# Global forwarding rule
resource "google_compute_global_forwarding_rule" "app" {
  name                  = "${var.project_name}-forwarding-rule"
  ip_protocol           = "TCP"
  load_balancing_scheme = "EXTERNAL_MANAGED"
  port_range            = "443"
  target                = google_compute_target_https_proxy.app.id
  ip_address            = google_compute_global_address.app.id
}

# storage-replication.tf

# Dual-region bucket for automatic replication
resource "google_storage_bucket" "app_assets" {
  name     = "${var.project_name}-assets"
  location = "NAM4"  # Dual-region: us-central1 and us-east1

  storage_class = "STANDARD"
  force_destroy = false

  versioning {
    enabled = true
  }

  lifecycle_rule {
    condition {
      num_newer_versions = 5
    }
    action {
      type = "Delete"
    }
  }

  uniform_bucket_level_access = true
}

# Turbo replication for critical data
resource "google_storage_bucket" "critical_data" {
  name     = "${var.project_name}-critical"
  location = "US"  # Multi-region

  storage_class = "STANDARD"

  custom_placement_config {
    data_locations = ["US-CENTRAL1", "US-EAST1"]
  }

  versioning {
    enabled = true
  }

  # Turbo replication enabled
  rpo = "ASYNC_TURBO"
}

# AWS: KMS key replication for DR
resource "aws_kms_replica_key" "dr" {
  provider        = aws.dr
  primary_key_arn = aws_kms_key.primary.arn
  description     = "DR replica of primary KMS key"
}

# Azure: Customer-managed keys
resource "azurerm_key_vault_key" "dr_encryption" {
  name         = "dr-encryption-key"
  key_vault_id = azurerm_key_vault.dr.id
  key_type     = "RSA"
  key_size     = 2048
  key_opts     = ["decrypt", "encrypt", "sign", "unwrapKey", "wrapKey"]
}

# AWS: Use Global Accelerator for optimized routing
resource "aws_globalaccelerator_accelerator" "app" {
  name            = "${var.project_name}-accelerator"
  ip_address_type = "IPV4"
  enabled         = true

  attributes {
    flow_logs_enabled   = true
    flow_logs_s3_bucket = aws_s3_bucket.flow_logs.bucket
    flow_logs_s3_prefix = "globalaccelerator/"
  }
}

resource "aws_globalaccelerator_listener" "app" {
  accelerator_arn = aws_globalaccelerator_accelerator.app.id
  protocol        = "TCP"

  port_range {
    from_port = 443
    to_port   = 443
  }
}

resource "aws_globalaccelerator_endpoint_group" "primary" {
  listener_arn                  = aws_globalaccelerator_listener.app.id
  endpoint_group_region         = "us-east-1"
  health_check_interval_seconds = 10
  health_check_path             = "/health"
  health_check_protocol         = "HTTPS"
  threshold_count               = 3
  traffic_dial_percentage       = 100

  endpoint_configuration {
    endpoint_id                    = aws_lb.primary.arn
    weight                         = 100
    client_ip_preservation_enabled = true
  }
}

resource "aws_globalaccelerator_endpoint_group" "dr" {
  listener_arn          = aws_globalaccelerator_listener.app.id
  endpoint_group_region = "us-west-2"
  traffic_dial_percentage = 0  # DR standby

  endpoint_configuration {
    endpoint_id = aws_lb.dr.arn
    weight      = 100
  }
}

# Set low TTL for failover records
resource "aws_route53_record" "app" {
  ttl = 60  # 1 minute, not 3600!

  # Or use alias record for instant failover
  alias {
    name    = aws_lb.app.dns_name
    zone_id = aws_lb.app.zone_id
    evaluate_target_health = true
  }
}