Disaster Recovery & RTO/RPO Guide

# Tag EBS volumes with your environment name, then create a DLM policy:
# - Resource: EBS volumes
# - Target tag: Environment=<your-env-name>
# - Schedule: Daily at 02:00 UTC
# - Retention: 7 snapshots

# 1. Identify the snapshot to restore
aws ec2 describe-snapshots --filters "Name=tag:Environment,Values=<your-env-name>"
 
# 2. Create a volume from the snapshot
aws ec2 create-volume \
  --snapshot-id snap-xxxxxxxxxxxxxxxxx \
  --availability-zone <your-az> \
  --volume-type gp3
 
# 3. Update the PersistentVolume in Kubernetes to reference the new volume ID
kubectl patch pv <pv-name> -p '{"spec":{"awsElasticBlockStore":{"volumeID":"<new-vol-id>"}}}'

postgres:
  backup:
    enabled: true                    # toggle barman-cloud backups
    # barmanObjectStore points to S3 bucket or MinIO endpoint
    # IRSA / service account must have s3:PutObject, s3:GetObject, s3:ListBucket
    retentionPolicy: "30d"           # keep backups for 30 days
    compression: gzip

# List all backups for the cluster
kubectl get backup -n ha-supabase-db
 
# Check a specific backup
kubectl describe backup <backup-name> -n ha-supabase-db
 
# Trigger an on-demand backup
kubectl cnpg backup ha-supabase-db -n ha-supabase-db

# Create a new CNPG Cluster restoring from a backup
# (in a new namespace or after removing the original)
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
  name: ha-supabase-db-restored
  namespace: ha-supabase-db-restore
spec:
  instances: 3
  bootstrap:
    recovery:
      source: ha-supabase-db
      recoveryTarget:
        targetTime: "2026-03-19T03:00:00.000000+00:00"  # target point in time
  externalClusters:
    - name: ha-supabase-db
      barmanObjectStore:
        # same S3/MinIO config as the source cluster
        serverName: ha-supabase-db

affinity:
  podAntiAffinity:
    preferredDuringSchedulingIgnoredDuringExecution:
    - weight: 100
      podAffinityTerm:
        labelSelector:
          matchExpressions:
          - key: app
            operator: In
            values: ["web", "fastapi-backend", "celery-worker", "automator"]
        topologyKey: kubernetes.io/hostname

kubectl get nodes -o wide     # confirm nodes are in healthy AZs
kubectl get pods -A -o wide   # confirm pods are running
kubectl get ingress            # confirm ALB is routing correctly

# Monitor failover
kubectl get cluster ha-supabase-db -n ha-supabase-db -w
 
# Confirm new primary
kubectl cnpg status ha-supabase-db -n ha-supabase-db
 
# Check PgBouncer pooler is pointing to new primary
kubectl get svc -n ha-supabase-db | grep pooler

# 1. Find snapshots for the environment
aws ec2 describe-snapshots \
  --filters "Name=tag:Environment,Values=<your-env-name>" \
  --query 'Snapshots[*].[SnapshotId,StartTime,VolumeSize]' \
  --output table
 
# 2. Create a new volume from the chosen snapshot
aws ec2 create-volume \
  --snapshot-id snap-xxxxxxxxxxxxxxxxx \
  --availability-zone <target-az> \
  --volume-type gp3 \
  --tag-specifications 'ResourceType=volume,Tags=[{Key=Environment,Value=<your-env-name>}]'
 
# 3. Scale down the affected workload
kubectl scale deployment <workload> --replicas=0
 
# 4. Update the PV to reference the new EBS volume, then scale back up
kubectl patch pv <pv-name> -p '{"spec":{"csi":{"volumeHandle":"<new-volume-id>"}}}'
kubectl scale deployment <workload> --replicas=2

# 1. Bootstrap state bucket (if it doesn't exist)
cd terragrunt/environments/<your-env-name>/state
terragrunt apply
 
# 2. Recreate the EKS cluster and core infrastructure
cd terragrunt/environments/<your-env-name>
terragrunt apply
 
# 3. Deploy services in order (see Terragrunt Deployment Guide § Phase 6)
ENABLE_CNPG=true terragrunt apply --target='helm_release.local["cloudnative-pg"]'
ENABLE_HA_SUPABASE_DB=true terragrunt apply --target='helm_release.local["ha-supabase-db"]'
ENABLE_SUPABASE=true terragrunt apply --target='helm_release.local["supabase"]'

# List state versions in S3
aws s3api list-object-versions \
  --bucket ekb-terraform-state-<your-env-name> \
  --prefix <state-key>
 
# Restore a specific version
aws s3api get-object \
  --bucket ekb-terraform-state-<your-env-name> \
  --key <state-key> \
  --version-id <version-id> \
  terraform.tfstate
 
# Re-import or apply with the restored state