ELI-597: Automates NHS Number salt creation and rotation

.github/workflows/base-deploy.yml

@@ -202,6 +202,7 @@ jobs:
           TF_VAR_API_PRIVATE_KEY_CERT: ${{ secrets.API_PRIVATE_KEY_CERT }}
           TF_VAR_SPLUNK_HEC_TOKEN: ${{ secrets.SPLUNK_HEC_TOKEN }}
           TF_VAR_SPLUNK_HEC_ENDPOINT: ${{ secrets.SPLUNK_HEC_ENDPOINT }}
+          TF_VAR_OPERATOR_EMAILS: ${{ vars.SECRET_ROTATION_OPERATOR_EMAILS }}
 
         working-directory: ./infrastructure
         shell: bash

infrastructure/.gitignore

@@ -38,3 +38,8 @@ override.tf.json
 # Ignore CLI configuration files
 .terraformrc
 terraform.rc
+
+# Ignore secret rotation lambda zip files
+
+stacks/api-layer/scripts/create_pending_secret.zip
+stacks/api-layer/scripts/promote_to_current.zip

infrastructure/modules/secrets_manager/outputs.tf

@@ -6,3 +6,6 @@ output "aws_hashing_secret_name" {
   value = aws_secretsmanager_secret.hashing_secret.name
 }
 
+output "kms_key_arn" {
+  value = aws_kms_key.secrets_cmk.arn
+}

infrastructure/stacks/api-layer/cloudwatch.tf

@@ -34,3 +34,12 @@ resource "aws_cloudwatch_log_stream" "firehose_audit_stream" {
     aws_cloudwatch_log_group.firehose_audit
   ]
 }
+
+resource "aws_cloudwatch_log_group" "rotation_sfn_logs" {
+  name              = "/aws/stepfunctions/SecretRotationWorkflow"
+  kms_key_id        = aws_lambda_function.create_secret_lambda.kms_key_arn
+
+
+
+  retention_in_days = 365
+}

infrastructure/stacks/api-layer/iam_policies.tf

@@ -561,3 +561,79 @@ resource "aws_iam_role_policy" "external_secret_read_policy_attachment" {
   role   = aws_iam_role.write_access_role[count.index].id
   policy = data.aws_iam_policy_document.secrets_access_policy.json
 }
+
+# --- Rotation Logic Policies ---
+resource "aws_iam_policy" "rotation_secrets_policy" {
+  name        = "rotation_secrets_policy"
+  description = "Allow Lambda to read/write ONLY the hashing secret"
+  policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      {
+        Sid    = "ManageSecretBits",
+        Effect = "Allow",
+        Action = [
+          "secretsmanager:DescribeSecret",
+          "secretsmanager:PutSecretValue",
+          "secretsmanager:UpdateSecretVersionStage",
+          "secretsmanager:GetSecretValue"
+        ],
+        Resource = module.secrets_manager.aws_hashing_secret_arn
+      },
+      {
+        Sid    = "AllowKMSKeyUsage",
+        Effect = "Allow",
+        Action = [
+          "kms:Decrypt",
+          "kms:GenerateDataKey"
+        ],
+        Resource = module.secrets_manager.kms_key_arn
+      },
+      {
+        Sid    = "BasicLogging",
+        Effect = "Allow",
+        Action = [
+          "logs:CreateLogGroup",
+          "logs:CreateLogStream",
+          "logs:PutLogEvents"
+        ],
+        Resource = [
+          "arn:aws:logs:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/${aws_lambda_function.create_secret_lambda.function_name}:*",
+          "arn:aws:logs:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/${aws_lambda_function.promote_secret_lambda.function_name}:*"
+        ]
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "attach_rotation_secrets" {
+  role       = aws_iam_role.rotation_lambda_role.name
+  policy_arn = aws_iam_policy.rotation_secrets_policy.arn
+}
+
+resource "aws_iam_policy" "rotation_sfn_policy" {
+  name = "rotation_sfn_policy"
+  policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      {
+        Effect = "Allow",
+        Action = "lambda:InvokeFunction",
+        Resource = [
+          aws_lambda_function.create_secret_lambda.arn,
+          aws_lambda_function.promote_secret_lambda.arn
+        ]
+      },
+      {
+        Effect   = "Allow",
+        Action   = "sns:Publish",
+        Resource = aws_sns_topic.cli_login_topic.arn
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "attach_rotation_sfn" {
+  role       = aws_iam_role.rotation_sfn_role.name
+  policy_arn = aws_iam_policy.rotation_sfn_policy.arn
+}

infrastructure/stacks/api-layer/iam_roles.tf

@@ -74,3 +74,66 @@ resource "aws_iam_role" "eligibility_audit_firehose_role" {
   assume_role_policy   = data.aws_iam_policy_document.firehose_assume_role.json
   permissions_boundary = aws_iam_policy.assumed_role_permissions_boundary.arn
 }
+
+# --- Secret Rotation Roles ---
+resource "aws_iam_role" "rotation_lambda_role" {
+  name = "secret_rotation_lambda_role"
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [{
+      Action = "sts:AssumeRole",
+      Effect = "Allow",
+      Principal = { Service = "lambda.amazonaws.com" }
+    }]
+  })
+}
+
+resource "aws_iam_role" "rotation_sfn_role" {
+  name = "secret_rotation_workflow_role"
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [{
+      Action = "sts:AssumeRole",
+      Effect = "Allow",
+      Principal = { Service = "states.amazonaws.com" }
+    }]
+  })
+}
+
+# -----------------------------------------------------------------------------
+# IAM Role: Allow EventBridge to Start the Step Function
+# -----------------------------------------------------------------------------
+
+resource "aws_iam_role" "eventbridge_sfn_invoke_role" {
+  name = "eventbridge_invoke_sfn_role"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = "sts:AssumeRole"
+        Effect = "Allow"
+        Principal = { Service = "events.amazonaws.com" }
+      }
+    ]
+  })
+}
+
+resource "aws_iam_policy" "eventbridge_sfn_policy" {
+  name = "eventbridge_sfn_start_policy"
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect   = "Allow"
+        Action   = "states:StartExecution"
+        Resource = aws_sfn_state_machine.rotation_machine.arn
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "attach_eb_sfn" {
+  role       = aws_iam_role.eventbridge_sfn_invoke_role.name
+  policy_arn = aws_iam_policy.eventbridge_sfn_policy.arn
+}

infrastructure/stacks/api-layer/lambda.tf

@@ -33,3 +33,60 @@
   provisioned_concurrency_count     = 5
   api_domain_name                   = local.api_domain_name
 }
+
+# -----------------------------------------------------------------------------
+# Secret rotation lambdas
+# -----------------------------------------------------------------------------
+
+# 1. Generator Lambda
+data "archive_file" "create_zip" {
+  type        = "zip"
+  source_file = "${path.module}/scripts/create_pending_secret.py"
+  output_path = "${path.module}/scripts/create_pending_secret.zip"
+}
+
+resource "aws_lambda_function" "create_secret_lambda" {
+  #checkov:skip=CKV_AWS_116: No deadletter queue is required for this Lambda function
+  #checkov:skip=CKV_AWS_115: Not applicable as it will not be possible to trigger concurrently
+  #checkov:skip=CKV_AWS_272: Skipping code signing but flagged to create ticket to investigate on ELI-238
+  #checkov:skip=CKV_AWS_50: No x-ray needed for this function
+  #checkov:skip=CKV_AWS_173: No encryption needed for the secret name
+  #checkov:skip=CKV_AWS_117: Does not need to be in a VPC
+
+  filename      = data.archive_file.create_zip.output_path
+  function_name = "${terraform.workspace}-CreatePendingSecretFunction"
+  role          = aws_iam_role.rotation_lambda_role.arn
+  handler       = "create_pending_secret.lambda_handler"
+  runtime       = "python3.13"
+  timeout       = 30
+
+  environment {
+    variables = { SECRET_NAME = module.secrets_manager.aws_hashing_secret_name }
+  }
+}
+
+# 2. Promoter Lambda
+data "archive_file" "promote_zip" {
+  type        = "zip"
+  source_file = "${path.module}/scripts/promote_to_current.py"
+  output_path = "${path.module}/scripts/promote_to_current.zip"
+}
+
+resource "aws_lambda_function" "promote_secret_lambda" {
+  #checkov:skip=CKV_AWS_116: No deadletter queue is required for this Lambda function
+  #checkov:skip=CKV_AWS_115: Not applicable as it will not be possible to trigger concurrently
+  #checkov:skip=CKV_AWS_272: Skipping code signing but flagged to create ticket to investigate on ELI-238
+  #checkov:skip=CKV_AWS_50: No x-ray needed for this function
+  #checkov:skip=CKV_AWS_173: No encryption needed for the secret name
+  #checkov:skip=CKV_AWS_117: Does not need to be in a VPC
+  filename      = data.archive_file.promote_zip.output_path
+  function_name = "${terraform.workspace}-PromoteToCurrentFunction"
+  role          = aws_iam_role.rotation_lambda_role.arn
+  handler       = "promote_to_current.lambda_handler"
+  runtime       = "python3.13"
+  timeout       = 30
+
+  environment {
+    variables = { SECRET_NAME = module.secrets_manager.aws_hashing_secret_name }
+  }
+}

infrastructure/stacks/api-layer/scripts/create_pending_secret.py

@@ -0,0 +1,80 @@
+import json
+import logging
+import os
+import secrets
+import string
+
+import boto3
+from mangum.types import LambdaContext, LambdaEvent
+
+SECRET_NAME = os.environ.get("SECRET_NAME")
+REGION_NAME = os.environ.get("AWS_REGION")
+
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+
+
+class PendingVersionExistsError(Exception):
+    pass
+
+
+def generate_password(length: int = 32) -> str:
+    """Generates a secure random password."""
+    alphabet = string.ascii_letters + string.digits + "!@#$%^&*"
+    return "".join(secrets.choice(alphabet) for _ in range(length))
+
+
+def lambda_handler(
+    event: LambdaEvent,  # noqa: ARG001
+    context: LambdaContext,
+) -> dict:
+    sm_client = boto3.client("secretsmanager", region_name=REGION_NAME)
+
+    logger.info(
+        json.dumps(
+            {
+                "event": "rotation_started",
+                "request_id": context.aws_request_id,
+                "secret_name": SECRET_NAME,
+                "function": "create_pending_secret",
+            }
+        )
+    )
+
+    try:
+        metadata = sm_client.describe_secret(SecretId=SECRET_NAME)
+        # Check if any version currently has the 'AWSPENDING' label
+        for version_id, stages in metadata.get("VersionIdsToStages", {}).items():
+            if "AWSPENDING" in stages:
+                msg = f"Pending version already exists with version_id: {version_id}."
+
+                logger.warning(
+                    json.dumps(
+                        {
+                            "event": "rotation_aborted",
+                            "reason": "pending_version_exists",
+                            "pending_version_id": version_id,
+                        }
+                    )
+                )
+
+                raise PendingVersionExistsError(msg)
+    except sm_client.exceptions.ResourceNotFoundException:
+        logger.info("Secret not found. Proceeding to create (assuming it will be initialized).")
+
+    new_password = generate_password()
+
+    try:
+        resp = sm_client.put_secret_value(SecretId=SECRET_NAME, SecretString=new_password, VersionStages=["AWSPENDING"])
+
+        logger.info(
+            json.dumps({"event": "pending_version_created", "version_id": resp["VersionId"], "status": "success"})
+        )
+        return {"status": "success", "secret_name": SECRET_NAME, "version_id": resp["VersionId"]}
+
+    except sm_client.exceptions.ResourceNotFoundException as e:
+        exception_message = f"The secret '{SECRET_NAME}' was not found in region '{REGION_NAME}'."
+        raise sm_client.exceptions.ResourceNotFoundException(exception_message) from e
+    except Exception as e:
+        logger.exception(json.dumps({"event": "rotation_failed", "type": type(e).__name__}))
+        raise

infrastructure/stacks/api-layer/scripts/promote_to_current.py

@@ -0,0 +1,69 @@
+import json
+import logging
+import os
+
+import boto3
+from mangum.types import LambdaContext, LambdaEvent
+
+SECRET_NAME = os.environ.get("SECRET_NAME")
+REGION_NAME = os.environ.get("AWS_REGION")
+
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+
+
+def lambda_handler(
+    event: LambdaEvent,  # noqa: ARG001
+    context: LambdaContext,
+) -> dict:
+    sm_client = boto3.client("secretsmanager", region_name=REGION_NAME)
+    logger.info(
+        json.dumps(
+            {
+                "event": "promotion_started",
+                "request_id": context.aws_request_id,
+                "secret_name": SECRET_NAME,
+                "function": "promote_to_current",
+            }
+        )
+    )
+
+    try:
+        metadata = sm_client.describe_secret(SecretId=SECRET_NAME)
+        pending_version = None
+
+        for version_id, stages in metadata["VersionIdsToStages"].items():
+            if "AWSPENDING" in stages:
+                pending_version = version_id
+                break
+
+        if pending_version:
+            logger.info(
+                json.dumps(
+                    {"event": "promoting_version", "pending_version_id": pending_version, "action": "swap_AWSCURRENT"}
+                )
+            )
+
+            sm_client.update_secret_version_stage(
+                SecretId=SECRET_NAME, VersionStage="AWSCURRENT", MoveToVersionId=pending_version
+            )
+
+            sm_client.update_secret_version_stage(
+                SecretId=SECRET_NAME, VersionStage="AWSPENDING", RemoveFromVersionId=pending_version
+            )
+
+            logger.info(
+                json.dumps({"event": "promotion_complete", "new_current_version": pending_version, "status": "success"})
+            )
+
+            return {"status": "success", "action": "promoted_and_cleaned", "new_current_version": pending_version}
+
+    except Exception as e:
+        logger.exception(json.dumps({"event": "promotion_failed", "type": type(e).__name__}))
+        raise
+
+    else:
+        logger.warning(
+            json.dumps({"event": "promotion_skipped", "reason": "no_pending_version_found", "secret_name": SECRET_NAME})
+        )
+        return {"status": "skipped", "reason": "no_pending_version"}