Locking #

# S3 Backend — needs DynamoDB for locking
terraform {
  backend "s3" {
    bucket         = "my-terraform-state"
    key            = "production/terraform.tfstate"
    region         = "ap-southeast-1"
    dynamodb_table = "terraform-state-lock"  # Required for locking
    encrypt        = true
  }
}

# Lock entry in DynamoDB while apply is running:
# LockID: "my-terraform-state/production/terraform.tfstate"
# Info:   {"ID":"abc123", "Operation":"OperationTypeApply",
#          "Who":"user@hostname", "Created":"2024-01-15T10:30:00Z"}

# GCS Backend — built-in locking, no extra resources needed
terraform {
  backend "gcs" {
    bucket = "my-terraform-state"
    prefix = "production"
    # Locking uses GCS object versioning — automatic
  }
}

# Terraform Cloud — built-in locking, managed by the platform
terraform {
  cloud {
    organization = "my-org"
    workspaces {
      name = "production"
    }
  }
}

# The error that appears when a lock is held:
$ terraform apply

╷
│ Error: Error acquiring the state lock
│
│ Error message: ConditionalCheckFailedException: ...
│ Lock Info:
│   ID:        abc123def456
│   Path:      my-terraform-state/production/terraform.tfstate
│   Operation: OperationTypeApply
│   Who:       [email protected]
│   Version:   1.6.3
│   Created:   2024-01-15 10:30:00.123456789 +0000 UTC
│   Info:
╵

# Important information from this error:
# - ID: used for force-unlock if needed
# - Who: who currently holds the lock
# - Created: when the lock was taken (if it's old, it might be stuck)

# Check whether there's an active lock
terraform plan
# If there's a lock, the error message will show the Lock ID

# Force unlock — use the Lock ID from the error message
terraform force-unlock abc123def456

# IMPORTANT: Make sure no other Terraform process is running
# before force-unlocking. Force unlocking an apply that's still
# running can corrupt state.

# Strategy 1: Serialized pipelines
# Make sure only one apply pipeline can run at a time
# (configuration in the CI/CD tool — GitHub Actions concurrency, GitLab resource groups)

# GitHub Actions: concurrency group
name: Terraform Apply
concurrency:
  group: terraform-production  # Only one workflow with this group may run
  cancel-in-progress: false    # Don't cancel the running one, queue it

jobs:
  apply:
    runs-on: ubuntu-latest
    steps:
      - name: Terraform Apply
        run: terraform apply -auto-approve tfplan

# Strategy 2: Reasonable timeouts
# If a pipeline is stuck and the lock isn't released after the timeout,
# the pipeline can be considered failed and an alert is sent

terraform apply -lock-timeout=5m  # Wait at most 5 minutes to acquire the lock
# If it doesn't succeed within 5 minutes, error and fail

# ANTI-PATTERN: Permanently disabling locking
terraform apply -lock=false  # ✗ Very dangerous in a team environment

# CORRECT: Disable locking only when truly forced
# and you're sure no other process is running
terraform plan -lock=false   # E.g. only for emergency inspection

# S3 + DynamoDB locking internals:
# 1. Terraform creates an item in DynamoDB with a LockID
# 2. Uses a DynamoDB conditional write:
#    PutItem with ConditionExpression = "attribute_not_exists(LockID)"
# 3. If the LockID already exists → ConditionalCheckFailedException
# 4. Only one process writes successfully → atomic

# DynamoDB lock item structure:
{
  "LockID": "my-bucket/production/terraform.tfstate",
  "Info": {
    "ID": "a3b4c5d6-e7f8-9012-abcd-ef1234567890",
    "Operation": "OperationTypeApply",
    "Who": "[email protected]",
    "Version": "1.6.3",
    "Created": "2024-01-15T10:30:00Z",
    "Path": "my-bucket/production/terraform.tfstate"
  }
}

# GCS locking internals:
# 1. Terraform creates a .tflock file in the bucket
# 2. Uses GCS preconditions:
#    ifGenerationMatch: 0 (only succeeds if the file doesn't exist)
# 3. Deletes the .tflock file when releasing
# 4. GCS guarantees atomicity through preconditions

# The default lock timeout is 0s — errors immediately if the lock is held
terraform apply
# Error: Error acquiring the state lock (fails immediately)

# Set a timeout to wait for the lock to be released
terraform apply -lock-timeout=5m   # Wait at most 5 minutes
terraform apply -lock-timeout=30s  # Wait at most 30 seconds
terraform apply -lock-timeout=1h   # Wait at most 1 hour

# SITUATION: Lock is stuck, you don't know who holds it
$ terraform plan

Error: Error acquiring the state lock
Lock Info:
  ID:        a3b4c5d6-e7f8-9012-abcd-ef1234567890
  Operation: OperationTypeApply
  Who:       deploy-agent@ci-runner-03
  Created:   2024-01-15 10:30:00 +0000 UTC
  Info:

# STEP 1: Check whether the process is still running
# In CI/CD — check whether the runner/pod is still active
kubectl get pods -l app=terraform-runner
# If the pod no longer exists → the lock is stuck, safe to force-unlock

# STEP 2: Check the lock's age
# A lock older than 30 minutes without progress is usually stuck
LOCK_CREATED="2024-01-15T10:30:00Z"
NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)
echo "Lock age: $(($(date -d "$NOW" +%s) - $(date -d "$LOCK_CREATED" +%s))) seconds"

# STEP 3: Check DynamoDB directly (for the S3 backend)
aws dynamodb get-item \
  --table-name terraform-state-lock \
  --key '{"LockID":{"S":"my-bucket/production/terraform.tfstate"}}'

# STEP 4: Force unlock if you're sure no process is active
terraform force-unlock a3b4c5d6-e7f8-9012-abcd-ef1234567890

# STEP 5: Verify the state is consistent after unlocking
terraform plan
# Should be "No changes" — if there are unexpected changes,
# the previous apply was probably cut off mid-way

# A lock table usable for many states (shared)
resource "aws_dynamodb_table" "terraform_lock" {
  name         = "terraform-state-lock"
  billing_mode = "PAY_PER_REQUEST"  # No provisioned capacity needed
  hash_key     = "LockID"

  attribute {
    name = "LockID"
    type = "S"  # String
  }

  # TTL for auto-cleaning stuck locks
  # (needs a TTL attribute named "ExpiresAt" on items)
  ttl {
    attribute_name = "ExpiresAt"
    enabled        = true
  }

  point_in_time_recovery {
    enabled = true  # Back up the lock table
  }
}

# PATTERNS:

# 1. Shared lock table for all states
#    One DynamoDB table, many different LockIDs
#    Best for: small-medium organizations (< 50 state files)
#    LockID: "bucket-name/path/to/terraform.tfstate"

# 2. Per-environment lock tables
#    Separate DynamoDB table per environment
#    terraform-lock-production, terraform-lock-staging
#    Best for: strict isolation between environments
#    Separates IAM permissions per table

# 3. Per-team lock tables
#    Each team has its own lock table
#    Infrastructure team: terraform-lock-infra
#    Application team: terraform-lock-apps
#    Best for: large organizations with many teams

# RECOMMENDATION: Start with pattern 1 (shared), split
# only if there's a specific IAM isolation need