Drift Detection Automation #

# .github/workflows/drift-detection.yml

name: Drift Detection

on:
  schedule:
    - cron: '0 8 * * 1-5'  # Every weekday at 8am
  workflow_dispatch:          # Can be triggered manually if needed

jobs:
  detect-drift:
    name: Detect Drift — ${{ matrix.environment }}
    runs-on: ubuntu-latest
    strategy:
      matrix:
        environment: [production, staging]
      fail-fast: false  # Continue even if one environment fails

    steps:
      - uses: actions/checkout@v4

      - uses: hashicorp/setup-terraform@v3
        with:
          terraform_version: "1.7.0"

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/TerraformReadRole
          aws-region: ap-southeast-1

      - name: Terraform Init
        run: terraform init
        working-directory: infrastructure/environments/${{ matrix.environment }}

      - name: Terraform Plan (Drift Check)
        id: plan
        run: |
          # -detailed-exitcode: exit 0 = no changes, exit 2 = changes exist
          terraform plan \
            -detailed-exitcode \
            -no-color \
            -refresh=true \
            2>&1 | tee plan_output.txt
          echo "exit_code=$?" >> $GITHUB_OUTPUT          
        working-directory: infrastructure/environments/${{ matrix.environment }}
        continue-on-error: true

      - name: Analyze Drift
        id: drift
        run: |
          EXIT_CODE="${{ steps.plan.outputs.exit_code }}"
          case $EXIT_CODE in
            0) echo "status=no_drift"   >> $GITHUB_OUTPUT
               echo "✅ No drift detected in ${{ matrix.environment }}" ;;
            1) echo "status=plan_error" >> $GITHUB_OUTPUT
               echo "❌ Terraform plan failed — check the logs" ;;
            2) echo "status=drift_detected" >> $GITHUB_OUTPUT
               echo "⚠️ Drift detected in ${{ matrix.environment }}!" ;;
          esac          

      - name: Extract Drift Summary
        if: steps.drift.outputs.status == 'drift_detected'
        run: |
          # Grab only the relevant parts of the plan output
          grep -E "^  [+~-]|will be|must be replaced|Plan:" plan_output.txt \
            | head -50 > drift_summary.txt
          cat drift_summary.txt          
        working-directory: infrastructure/environments/${{ matrix.environment }}

      - name: Notify Slack — Drift Detected
        if: steps.drift.outputs.status == 'drift_detected'
        uses: slackapi/slack-github-action@v1
        with:
          payload: |
            {
              "text": "⚠️ Infrastructure drift detected",
              "blocks": [
                {
                  "type": "section",
                  "text": {
                    "type": "mrkdwn",
                    "text": "*⚠️ Drift Detected: `${{ matrix.environment }}`*\n\nThe infrastructure doesn't match the Terraform state.\nSee details: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
                  }
                }
              ]
            }            
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_INFRA_WEBHOOK }}

# OPTION 1: Bring the infrastructure back to the Terraform configuration
# This is the default response and the most recommended
terraform apply

# OPTION 2: Update the state to reflect the actual condition
# Use this if the manual change was intentional and you want to keep it
# WARNING: this doesn't change the .tf configuration — only the state
terraform apply -refresh-only

# terraform refresh (deprecated — use apply -refresh-only)
# terraform refresh

# After a refresh-only apply, you need to update the .tf configuration
# to match the actual condition, then commit it to the repository

# Example: After drift is found on a security group
# Actual condition: extra rules that are truly wanted

# Choice A: Remove the manual rules (revert to Terraform)
# → terraform apply (without .tf configuration changes)

# Choice B: Add the rules to the configuration (accept the manual changes)
resource "aws_security_group_rule" "debug_access" {
  type              = "ingress"
  from_port         = 8080
  to_port           = 8080
  protocol          = "tcp"
  cidr_blocks       = ["10.0.0.0/8"]
  security_group_id = aws_security_group.web.id
  description       = "Internal debug access — added after drift review 2024-01-15"
}
# Commit this to the repository → drift resolved

# Add ignore_changes for attributes that change outside Terraform
resource "aws_instance" "web" {
  ami           = var.ami_id
  instance_type = var.instance_type

  lifecycle {
    ignore_changes = [
      # The AMI changes when there's an automatic patch
      ami,
      # Tags added by billing tools or security scanners
      tags["LastScannedBy"],
      tags["CostCenter"],
      # User data may be changed by external automation
      user_data_base64,
    ]
  }
}

# Filter the drift report — only show what isn't from ignore_changes
# (Terraform already automatically doesn't report attributes in ignore_changes)

# For custom filtering: parse the JSON plan output
terraform show -json tfplan | jq '
  .resource_changes[]
  | select(.change.actions != ["no-op"])
  | {
      resource: .address,
      actions: .change.actions,
      changes: (.change.before // {} | keys)
    }
'

DRIFT DETECTED → Notification → Review → Auto-fix or Manual Fix

Auto-fix flow:
1. Drift detection cron runs
2. Drift detected
3. Create a ticket/issue
4. Auto-apply if minor (tags, metadata)
5. Manual approval if major (instance type, security)

Manual fix flow:
1. Drift detected
2. Notify the team
3. An engineer reviews
4. Decide: fix the infrastructure or update the code
5. Apply the change
6. Close the ticket

# Drift remediation policy
# Minor drift: auto-remediate
# Major drift: require manual approval

# Sentinel policy
import "tfplan/v2" as tfplan
import "strings"

minor_drift_types = ["tags", "description", "metadata"]

is_minor_drift = rule {
  all tfplan.resource_changes as _, rc {
    rc.change.actions contains "update" implies
    all rc.change.before_unknown as field {
      strings.has_prefix(field, "tags") or field == "description"
    }
  }
}

main = rule {
  is_minor_drift or
  tfrun.workspace.name is not "production"
}

DRIFT METRICS:

Track drift occurrence over time:
├── Total resources with drift
├── Drift by resource type
├── Drift by environment
├── Time to remediation
└── Drift recurrence rate

Alerting thresholds:
- 0 drifts: healthy
- 1-5 drifts: warning
- 6+ drifts: critical
- Any security drift: critical

# CloudWatch Event Rule for drift detection
resource "aws_cloudwatch_event_rule" "drift" {
  name        = "terraform-drift-detected"
  description = "Trigger when Terraform detects drift"
  event_pattern = jsonencode({
    source      = ["custom.terraform"]
    detail-type = ["Drift Detected"]
  })
}

resource "aws_cloudwatch_event_target" "notify" {
  rule      = aws_cloudwatch_event_rule.drift.name
  target_id = "SendToSNS"
  arn       = aws_sns_topic.alerts.arn
}

resource "aws_sns_topic_subscription" "email" {
  topic_arn = aws_sns_topic.alerts.arn
  protocol  = "email"
  endpoint  = "[email protected]"
}