From afc8f090dcad7b2b15976fc7ec54e805d46d084e Mon Sep 17 00:00:00 2001
From: Edd Almond <102675624+eddalmond1@users.noreply.github.com>
Date: Wed, 6 Aug 2025 16:24:46 +0100
Subject: [PATCH 01/22] eli-304 adding splunk firehose module

---
 .../modules/splunk_forwarder/cloudwatch.tf    |  8 ++++++
 .../modules/splunk_forwarder/eventbridge.tf   |  5 ++++
 .../modules/splunk_forwarder/firehose.tf      | 23 ++++++++++++++++
 .../modules/splunk_forwarder/iam.tf           | 26 +++++++++++++++++++
 .../modules/splunk_forwarder/variables.tf     | 19 ++++++++++++++
 5 files changed, 81 insertions(+)
 create mode 100644 infrastructure/modules/splunk_forwarder/cloudwatch.tf
 create mode 100644 infrastructure/modules/splunk_forwarder/eventbridge.tf
 create mode 100644 infrastructure/modules/splunk_forwarder/firehose.tf
 create mode 100644 infrastructure/modules/splunk_forwarder/iam.tf
 create mode 100644 infrastructure/modules/splunk_forwarder/variables.tf

diff --git a/infrastructure/modules/splunk_forwarder/cloudwatch.tf b/infrastructure/modules/splunk_forwarder/cloudwatch.tf
new file mode 100644
index 000000000..5f7cabbb1
--- /dev/null
+++ b/infrastructure/modules/splunk_forwarder/cloudwatch.tf
@@ -0,0 +1,8 @@
+resource "aws_cloudwatch_event_rule" "alarm_state_change" {
+  name        = "cloudwatch-alarm-state-change"
+  description = "Forward CloudWatch alarm state changes to Splunk via Firehose"
+  event_pattern = jsonencode({
+    "source": ["aws.cloudwatch"],
+    "detail-type": ["CloudWatch Alarm State Change"]
+  })
+}
diff --git a/infrastructure/modules/splunk_forwarder/eventbridge.tf b/infrastructure/modules/splunk_forwarder/eventbridge.tf
new file mode 100644
index 000000000..c02e43057
--- /dev/null
+++ b/infrastructure/modules/splunk_forwarder/eventbridge.tf
@@ -0,0 +1,5 @@
+resource "aws_cloudwatch_event_target" "alarm_to_splunk" {
+  rule      = aws_cloudwatch_event_rule.alarm_state_change.name
+  arn       = aws_kinesis_firehose_delivery_stream.splunk_delivery_stream.arn
+  role_arn  = aws_iam_role.eventbridge_to_firehose.arn
+}
diff --git a/infrastructure/modules/splunk_forwarder/firehose.tf b/infrastructure/modules/splunk_forwarder/firehose.tf
new file mode 100644
index 000000000..945d3c379
--- /dev/null
+++ b/infrastructure/modules/splunk_forwarder/firehose.tf
@@ -0,0 +1,23 @@
+resource "aws_kinesis_firehose_delivery_stream" "splunk_delivery_stream" {
+  name        = "splunk-alarm-events"
+  destination = "splunk"
+
+  # VPC configuration is only supported for HTTP endpoint destinations in Kinesis Firehose
+  # For Splunk destinations, the service runs in AWS-managed VPC but you can control network access
+  # via the subnets where EventBridge (the source) runs and IAM policies
+
+  splunk_configuration {
+    hec_endpoint      = var.splunk_hec_endpoint
+    hec_token         = var.splunk_hec_token
+    hec_endpoint_type = "Event"
+    s3_backup_mode    = "FailedEventsOnly"
+
+    s3_configuration {
+      role_arn           = var.splunk_firehose_s3_role_arn
+      bucket_arn         = var.splunk_firehose_s3_backup_arn
+      buffering_size     = 10
+      buffering_interval = 400
+      compression_format = "GZIP"
+    }
+  }
+}
diff --git a/infrastructure/modules/splunk_forwarder/iam.tf b/infrastructure/modules/splunk_forwarder/iam.tf
new file mode 100644
index 000000000..8fc8f9443
--- /dev/null
+++ b/infrastructure/modules/splunk_forwarder/iam.tf
@@ -0,0 +1,26 @@
+resource "aws_iam_role" "eventbridge_to_firehose" {
+  name = "eventbridge-to-firehose-role"
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [{
+      Effect = "Allow",
+      Principal = { Service = "events.amazonaws.com" },
+      Action = "sts:AssumeRole"
+    }]
+  })
+}
+
+resource "aws_iam_role_policy" "eventbridge_to_firehose_policy" {
+  role = aws_iam_role.eventbridge_to_firehose.id
+  policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [{
+      Effect = "Allow",
+      Action = [
+        "firehose:PutRecord",
+        "firehose:PutRecordBatch"
+      ],
+      Resource = aws_kinesis_firehose_delivery_stream.splunk_delivery_stream.arn
+    }]
+  })
+}
diff --git a/infrastructure/modules/splunk_forwarder/variables.tf b/infrastructure/modules/splunk_forwarder/variables.tf
new file mode 100644
index 000000000..3307fb1b5
--- /dev/null
+++ b/infrastructure/modules/splunk_forwarder/variables.tf
@@ -0,0 +1,19 @@
+variable "splunk_hec_endpoint" {
+  description = "Splunk HEC endpoint URL"
+  type        = string
+}
+
+variable "splunk_hec_token" {
+  description = "Splunk HEC token"
+  type        = string
+}
+
+variable "splunk_firehose_s3_backup_arn" {
+  description = "s3 bucket ARN for Firehose backups"
+  type        = string
+}
+
+variable "splunk_firehose_s3_role_arn" {
+  description = "IAM role ARN for Firehose to access S3"
+  type        = string
+}

From 26e6f448096793c5b122148f9a06b3763eb1a97a Mon Sep 17 00:00:00 2001
From: Edd Almond <102675624+eddalmond1@users.noreply.github.com>
Date: Wed, 6 Aug 2025 16:26:03 +0100
Subject: [PATCH 02/22] eli-304 adding splunk firehose to api-layer stack

---
 infrastructure/stacks/api-layer/data.tf       |  9 ++
 .../stacks/api-layer/iam_policies.tf          | 47 ++++++++++
 infrastructure/stacks/api-layer/iam_roles.tf  | 15 +++
 infrastructure/stacks/api-layer/s3_buckets.tf |  9 ++
 .../stacks/api-layer/splunk_forwarder.tf      |  8 ++
 infrastructure/stacks/api-layer/ssm.tf        | 94 +++++++++++++++++++
 6 files changed, 182 insertions(+)
 create mode 100644 infrastructure/stacks/api-layer/splunk_forwarder.tf
 create mode 100644 infrastructure/stacks/api-layer/ssm.tf

diff --git a/infrastructure/stacks/api-layer/data.tf b/infrastructure/stacks/api-layer/data.tf
index 6b159ad98..9318b2acc 100644
--- a/infrastructure/stacks/api-layer/data.tf
+++ b/infrastructure/stacks/api-layer/data.tf
@@ -28,3 +28,12 @@ data "aws_ssm_parameter" "mtls_api_ca_cert" {
   name            = "/${var.environment}/mtls/api_ca_cert"
   with_decryption = true
 }
+
+data "aws_ssm_parameter" "splunk_hec_token" {
+  name = "/splunk/hec/token"
+  with_decryption = true
+}
+data "aws_ssm_parameter" "splunk_hec_endpoint" {
+  name = "/splunk/hec/endpoint"
+  with_decryption = true
+}
diff --git a/infrastructure/stacks/api-layer/iam_policies.tf b/infrastructure/stacks/api-layer/iam_policies.tf
index 5f384895c..23fed6c83 100644
--- a/infrastructure/stacks/api-layer/iam_policies.tf
+++ b/infrastructure/stacks/api-layer/iam_policies.tf
@@ -284,6 +284,53 @@ resource "aws_kms_key_policy" "s3_rules_kms_key" {
   policy = data.aws_iam_policy_document.s3_rules_kms_key_policy.json
 }
 
+resource "aws_iam_role_policy" "splunk_firehose_policy" {
+  name = "splunk-firehose-policy"
+  role = aws_iam_role.splunk_firehose_assume_role.id
+
+  policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      # Allow Firehose to write to S3 backup bucket
+      {
+        Effect = "Allow",
+        Action = [
+          "s3:PutObject",
+          "s3:PutObjectAcl",
+          "s3:GetBucketLocation",
+          "s3:ListBucket"
+        ],
+        Resource = [
+          module.s3_firehose_backup_bucket.storage_bucket_arn,
+          "${module.s3_firehose_backup_bucket.storage_bucket_arn}/*"
+        ]
+      },
+      # Allow Firehose to use KMS key for S3 encryption
+      {
+        Effect = "Allow",
+        Action = [
+          "kms:Encrypt",
+          "kms:Decrypt",
+          "kms:GenerateDataKey",
+          "kms:DescribeKey"
+        ],
+        Resource = [module.s3_firehose_backup_bucket.storage_bucket_kms_key_arn]
+      },
+      # Allow Firehose to access Splunk endpoint (no explicit IAM action needed, but you can restrict VPC/network access separately)
+      # Allow logging to CloudWatch
+      {
+        Effect = "Allow",
+        Action = [
+          "logs:PutLogEvents",
+          "logs:CreateLogStream",
+          "logs:CreateLogGroup"
+        ],
+        Resource = "*"
+      }
+    ]
+  })
+}
+
 data "aws_iam_policy_document" "s3_audit_kms_key_policy" {
   #checkov:skip=CKV_AWS_111: Root user needs full KMS key management
   #checkov:skip=CKV_AWS_356: Root user needs full KMS key management
diff --git a/infrastructure/stacks/api-layer/iam_roles.tf b/infrastructure/stacks/api-layer/iam_roles.tf
index 2fe2618dc..22244bbe7 100644
--- a/infrastructure/stacks/api-layer/iam_roles.tf
+++ b/infrastructure/stacks/api-layer/iam_roles.tf
@@ -33,6 +33,21 @@ data "aws_iam_policy_document" "firehose_assume_role" {
   }
 }
 
+resource "aws_iam_role" "splunk_firehose_assume_role" {
+  name = "splunk-firehose-role"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [{
+      Effect = "Allow",
+      Principal = {
+        Service = "firehose.amazonaws.com"
+      },
+      Action = "sts:AssumeRole"
+    }]
+  })
+}
+
 # Roles
 
 resource "aws_iam_role" "eligibility_lambda_role" {
diff --git a/infrastructure/stacks/api-layer/s3_buckets.tf b/infrastructure/stacks/api-layer/s3_buckets.tf
index a1c554575..ab087b406 100644
--- a/infrastructure/stacks/api-layer/s3_buckets.tf
+++ b/infrastructure/stacks/api-layer/s3_buckets.tf
@@ -16,3 +16,12 @@ module "s3_audit_bucket" {
   stack_name             = local.stack_name
   workspace              = terraform.workspace
 }
+
+module "s3_firehose_backup_bucket" {
+  source       = "../../modules/s3"
+  bucket_name  = "eli-splunk-backup"
+  environment  = var.environment
+  project_name = var.project_name
+  stack_name   = local.stack_name
+  workspace    = terraform.workspace
+}
diff --git a/infrastructure/stacks/api-layer/splunk_forwarder.tf b/infrastructure/stacks/api-layer/splunk_forwarder.tf
new file mode 100644
index 000000000..24862e592
--- /dev/null
+++ b/infrastructure/stacks/api-layer/splunk_forwarder.tf
@@ -0,0 +1,8 @@
+module "splunk_forwarder" {
+  source = "../../modules/splunk_forwarder"
+
+  splunk_hec_endpoint           = data.aws_ssm_parameter.splunk_hec_endpoint.value
+  splunk_hec_token              = data.aws_ssm_parameter.splunk_hec_token.value
+  splunk_firehose_s3_role_arn   = aws_iam_role.splunk_firehose_assume_role.arn
+  splunk_firehose_s3_backup_arn = module.s3_firehose_backup_bucket.storage_bucket_arn
+}
diff --git a/infrastructure/stacks/api-layer/ssm.tf b/infrastructure/stacks/api-layer/ssm.tf
new file mode 100644
index 000000000..5667fba48
--- /dev/null
+++ b/infrastructure/stacks/api-layer/ssm.tf
@@ -0,0 +1,94 @@
+resource "aws_kms_key" "splunk_hec_kms" {
+  description             = "KMS key for encrypting Splunk HEC SSM parameters"
+  deletion_window_in_days = 7
+
+  tags = {
+    Name        = "splunk-hec-ssm-kms-key"
+    Environment = var.environment
+    Stack       = local.stack_name
+    Purpose     = "Splunk HEC SSM encryption"
+    ManagedBy   = "terraform"
+  }
+}
+
+resource "aws_kms_key_policy" "splunk_hec_kms_policy" {
+  key_id = aws_kms_key.splunk_hec_kms.id
+  policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      {
+        Sid       = "AllowRootAccountFullAccess"
+        Effect    = "Allow"
+        Principal = { AWS = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:root" }
+        Action    = "kms:*"
+        Resource  = "*"
+      },
+      {
+        Sid       = "AllowSSMServiceUseOfKey"
+        Effect    = "Allow"
+        Principal = { Service = "ssm.amazonaws.com" }
+        Action = [
+          "kms:Encrypt",
+          "kms:Decrypt",
+          "kms:ReEncrypt*",
+          "kms:GenerateDataKey*",
+          "kms:DescribeKey"
+        ]
+        Resource = "*"
+      },
+      {
+        Sid       = "AllowFirehoseServiceUseOfKey"
+        Effect    = "Allow"
+        Principal = { Service = "firehose.amazonaws.com" }
+        Action = [
+          "kms:Encrypt",
+          "kms:Decrypt",
+          "kms:ReEncrypt*",
+          "kms:GenerateDataKey*",
+          "kms:DescribeKey"
+        ]
+        Resource = "*"
+      },
+    ]
+  })
+}
+
+resource "aws_ssm_parameter" "splunk_hec_token" {
+  name        = "/splunk/hec/token"
+  description = "Splunk HEC token"
+  type        = "SecureString"
+  key_id      = aws_kms_key.splunk_hec_kms.id
+  value       = "REPLACE_ME" # Set a placeholder value
+  tier        = "Advanced"
+
+  tags = {
+    Environment = var.environment
+    Stack       = local.stack_name
+    Purpose     = "Splunk HEC token"
+    ManagedBy   = "terraform"
+  }
+
+  lifecycle {
+    ignore_changes = [value]
+  }
+}
+
+resource "aws_ssm_parameter" "splunk_hec_endpoint" {
+  name        = "/splunk/hec/endpoint"
+  description = "Splunk HEC endpoint"
+  type        = "SecureString"
+  key_id      = aws_kms_key.splunk_hec_kms.id
+  value       = "REPLACE_ME" # Set a placeholder value
+  tier        = "Advanced"
+
+  tags = {
+    Environment = var.environment
+    Stack       = local.stack_name
+    Purpose     = "Splunk HEC endpoint"
+    ManagedBy   = "terraform"
+  }
+
+  lifecycle {
+    ignore_changes = [value]
+  }
+}

From c99d0d8c5694f6363bc07a7418347c2adcc5d151 Mon Sep 17 00:00:00 2001
From: Edd Almond <102675624+eddalmond1@users.noreply.github.com>
Date: Thu, 7 Aug 2025 10:33:20 +0100
Subject: [PATCH 03/22] eli-304 adding kms encryption to firehose and ssm,
 refactoring so specific eventbridge feed is in api-layer, so that we can add
 additional splunk firehose feeds in future if needed (e.g. keep the module
 generic)

---
 .../modules/splunk_forwarder/cloudwatch.tf    |  8 --
 .../modules/splunk_forwarder/data.tf          |  1 +
 .../modules/splunk_forwarder/eventbridge.tf   |  5 --
 .../modules/splunk_forwarder/firehose.tf      | 28 +++++-
 .../modules/splunk_forwarder/iam.tf           | 72 ++++++++++-----
 .../modules/splunk_forwarder/outputs.tf       | 11 +++
 .../stacks/api-layer/eventbridge.tf           | 88 +++++++++++++++++++
 .../stacks/api-layer/iam_policies.tf          |  1 -
 infrastructure/stacks/api-layer/iam_roles.tf  |  7 +-
 9 files changed, 184 insertions(+), 37 deletions(-)
 delete mode 100644 infrastructure/modules/splunk_forwarder/cloudwatch.tf
 create mode 100644 infrastructure/modules/splunk_forwarder/data.tf
 delete mode 100644 infrastructure/modules/splunk_forwarder/eventbridge.tf
 create mode 100644 infrastructure/modules/splunk_forwarder/outputs.tf
 create mode 100644 infrastructure/stacks/api-layer/eventbridge.tf

diff --git a/infrastructure/modules/splunk_forwarder/cloudwatch.tf b/infrastructure/modules/splunk_forwarder/cloudwatch.tf
deleted file mode 100644
index 5f7cabbb1..000000000
--- a/infrastructure/modules/splunk_forwarder/cloudwatch.tf
+++ /dev/null
@@ -1,8 +0,0 @@
-resource "aws_cloudwatch_event_rule" "alarm_state_change" {
-  name        = "cloudwatch-alarm-state-change"
-  description = "Forward CloudWatch alarm state changes to Splunk via Firehose"
-  event_pattern = jsonencode({
-    "source": ["aws.cloudwatch"],
-    "detail-type": ["CloudWatch Alarm State Change"]
-  })
-}
diff --git a/infrastructure/modules/splunk_forwarder/data.tf b/infrastructure/modules/splunk_forwarder/data.tf
new file mode 100644
index 000000000..8fc4b38cc
--- /dev/null
+++ b/infrastructure/modules/splunk_forwarder/data.tf
@@ -0,0 +1 @@
+data "aws_caller_identity" "current" {}
diff --git a/infrastructure/modules/splunk_forwarder/eventbridge.tf b/infrastructure/modules/splunk_forwarder/eventbridge.tf
deleted file mode 100644
index c02e43057..000000000
--- a/infrastructure/modules/splunk_forwarder/eventbridge.tf
+++ /dev/null
@@ -1,5 +0,0 @@
-resource "aws_cloudwatch_event_target" "alarm_to_splunk" {
-  rule      = aws_cloudwatch_event_rule.alarm_state_change.name
-  arn       = aws_kinesis_firehose_delivery_stream.splunk_delivery_stream.arn
-  role_arn  = aws_iam_role.eventbridge_to_firehose.arn
-}
diff --git a/infrastructure/modules/splunk_forwarder/firehose.tf b/infrastructure/modules/splunk_forwarder/firehose.tf
index 945d3c379..535569e5b 100644
--- a/infrastructure/modules/splunk_forwarder/firehose.tf
+++ b/infrastructure/modules/splunk_forwarder/firehose.tf
@@ -1,7 +1,33 @@
+
+
+# KMS Key for Firehose encryption
+resource "aws_kms_key" "firehose_splunk_cmk" {
+  description             = "KMS key for encrypting Kinesis Firehose delivery stream data"
+  deletion_window_in_days = 7
+  enable_key_rotation     = true
+  tags = {
+    Name      = "firehose-splunk-cmk"
+    Purpose   = "Firehose encryption"
+    ManagedBy = "terraform"
+  }
+}
+
+# KMS Key Alias for easier identification
+resource "aws_kms_alias" "firehose_splunk_cmk_alias" {
+  name          = "alias/firehose-splunk-cmk"
+  target_key_id = aws_kms_key.firehose_splunk_cmk.key_id
+}
+
+# KMS Key Policy for Firehose
+
 resource "aws_kinesis_firehose_delivery_stream" "splunk_delivery_stream" {
   name        = "splunk-alarm-events"
   destination = "splunk"
-
+  server_side_encryption {
+    enabled  = true
+    key_type = "CUSTOMER_MANAGED_CMK"
+    key_arn  = aws_kms_key.firehose_splunk_cmk.arn
+  }
   # VPC configuration is only supported for HTTP endpoint destinations in Kinesis Firehose
   # For Splunk destinations, the service runs in AWS-managed VPC but you can control network access
   # via the subnets where EventBridge (the source) runs and IAM policies
diff --git a/infrastructure/modules/splunk_forwarder/iam.tf b/infrastructure/modules/splunk_forwarder/iam.tf
index 8fc8f9443..7b5c5946c 100644
--- a/infrastructure/modules/splunk_forwarder/iam.tf
+++ b/infrastructure/modules/splunk_forwarder/iam.tf
@@ -1,26 +1,56 @@
-resource "aws_iam_role" "eventbridge_to_firehose" {
-  name = "eventbridge-to-firehose-role"
-  assume_role_policy = jsonencode({
-    Version = "2012-10-17",
-    Statement = [{
-      Effect = "Allow",
-      Principal = { Service = "events.amazonaws.com" },
-      Action = "sts:AssumeRole"
-    }]
-  })
-}
+# EventBridge IAM roles now defined in api-layer stack for specific integration
 
-resource "aws_iam_role_policy" "eventbridge_to_firehose_policy" {
-  role = aws_iam_role.eventbridge_to_firehose.id
+resource "aws_kms_key_policy" "firehose_splunk_cmk_policy" {
+  key_id = aws_kms_key.firehose_splunk_cmk.id
   policy = jsonencode({
     Version = "2012-10-17",
-    Statement = [{
-      Effect = "Allow",
-      Action = [
-        "firehose:PutRecord",
-        "firehose:PutRecordBatch"
-      ],
-      Resource = aws_kinesis_firehose_delivery_stream.splunk_delivery_stream.arn
-    }]
+    Statement = [
+      {
+        Sid       = "AllowRootAccountFullAccess"
+        Effect    = "Allow"
+        Principal = { AWS = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:root" }
+        Action    = "kms:*"
+        Resource  = "*"
+      },
+      {
+        Sid       = "AllowFirehoseServiceUseOfKey"
+        Effect    = "Allow"
+        Principal = { Service = "firehose.amazonaws.com" }
+        Action = [
+          "kms:Encrypt",
+          "kms:Decrypt",
+          "kms:ReEncrypt*",
+          "kms:GenerateDataKey*",
+          "kms:DescribeKey"
+        ]
+        Resource = "*"
+      },
+      {
+        Sid       = "AllowEventBridgeUseOfKey"
+        Effect    = "Allow"
+        Principal = { Service = "events.amazonaws.com" }
+        Action = [
+          "kms:Encrypt",
+          "kms:Decrypt",
+          "kms:ReEncrypt*",
+          "kms:GenerateDataKey*",
+          "kms:DescribeKey"
+        ]
+        Resource = "*"
+      },
+      {
+        Sid       = "AllowCloudWatchUseOfKey"
+        Effect    = "Allow"
+        Principal = { Service = "cloudwatch.amazonaws.com" }
+        Action = [
+          "kms:Encrypt",
+          "kms:Decrypt",
+          "kms:ReEncrypt*",
+          "kms:GenerateDataKey*",
+          "kms:DescribeKey"
+        ]
+        Resource = "*"
+      }
+    ]
   })
 }
diff --git a/infrastructure/modules/splunk_forwarder/outputs.tf b/infrastructure/modules/splunk_forwarder/outputs.tf
new file mode 100644
index 000000000..03bcfe1a0
--- /dev/null
+++ b/infrastructure/modules/splunk_forwarder/outputs.tf
@@ -0,0 +1,11 @@
+# Output the Firehose delivery stream ARN for use by EventBridge
+output "firehose_delivery_stream_arn" {
+  description = "ARN of the Kinesis Firehose delivery stream for Splunk"
+  value       = aws_kinesis_firehose_delivery_stream.splunk_delivery_stream.arn
+}
+
+# Output the KMS key ARN for reference
+output "firehose_kms_key_arn" {
+  description = "ARN of the KMS key used for Firehose encryption"
+  value       = aws_kms_key.firehose_splunk_cmk.arn
+}
diff --git a/infrastructure/stacks/api-layer/eventbridge.tf b/infrastructure/stacks/api-layer/eventbridge.tf
new file mode 100644
index 000000000..e9d013a3e
--- /dev/null
+++ b/infrastructure/stacks/api-layer/eventbridge.tf
@@ -0,0 +1,88 @@
+# IAM role for EventBridge to write to Firehose
+resource "aws_iam_role" "eventbridge_firehose_role" {
+  name = "${var.environment}-eventbridge-to-firehose-role"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [{
+      Effect = "Allow"
+      Principal = {
+        Service = "events.amazonaws.com"
+      }
+      Action = "sts:AssumeRole"
+    }]
+  })
+
+  tags = {
+    Environment = var.environment
+    Purpose     = "splunk-forwarding"
+    ManagedBy   = "terraform"
+  }
+}
+
+# IAM policy for EventBridge to access Firehose
+resource "aws_iam_role_policy" "eventbridge_to_firehose_policy" {
+  name = "${var.environment}-eventbridge-to-firehose-policy"
+  role = aws_iam_role.eventbridge_firehose_role.id
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [{
+      Effect = "Allow"
+      Action = [
+        "firehose:PutRecord",
+        "firehose:PutRecordBatch"
+      ]
+      Resource = module.splunk_forwarder.firehose_delivery_stream_arn
+    }]
+  })
+}
+
+# EventBridge rule to capture CloudWatch alarm state changes
+resource "aws_cloudwatch_event_rule" "alarm_state_change" {
+  name        = "cloudwatch-alarm-state-change-to-splunk"
+  description = "Forward CloudWatch alarm state changes to Splunk via Firehose"
+
+  event_pattern = jsonencode({
+    source      = ["aws.cloudwatch"]
+    detail-type = ["CloudWatch Alarm State Change"]
+  })
+
+  tags = {
+    Environment = var.environment
+    Purpose     = "splunk-forwarding"
+    ManagedBy   = "terraform"
+  }
+}
+
+# EventBridge target to send events to Firehose
+resource "aws_cloudwatch_event_target" "firehose_target" {
+  rule     = aws_cloudwatch_event_rule.alarm_state_change.name
+  arn      = module.splunk_forwarder.firehose_delivery_stream_arn
+  role_arn = aws_iam_role.eventbridge_firehose_role.arn
+
+  # Transform the CloudWatch alarm event into a format suitable for Splunk
+  input_transformer {
+    input_paths = {
+      account    = "$.account"
+      region     = "$.region"
+      time       = "$.time"
+      alarm_name = "$.detail.alarmName"
+      new_state  = "$.detail.state.value"
+      old_state  = "$.detail.previousState.value"
+      reason     = "$.detail.state.reason"
+    }
+
+    input_template = jsonencode({
+      timestamp   = "<time>"
+      source      = "aws:cloudwatch:alarm"
+      account_id  = "<account>"
+      region      = "<region>"
+      alarm_name  = "<alarm_name>"
+      new_state   = "<new_state>"
+      old_state   = "<old_state>"
+      reason      = "<reason>"
+      severity    = "<new_state>" == "ALARM" ? "high" : "info"
+    })
+  }
+}
diff --git a/infrastructure/stacks/api-layer/iam_policies.tf b/infrastructure/stacks/api-layer/iam_policies.tf
index 23fed6c83..106326de1 100644
--- a/infrastructure/stacks/api-layer/iam_policies.tf
+++ b/infrastructure/stacks/api-layer/iam_policies.tf
@@ -316,7 +316,6 @@ resource "aws_iam_role_policy" "splunk_firehose_policy" {
         ],
         Resource = [module.s3_firehose_backup_bucket.storage_bucket_kms_key_arn]
       },
-      # Allow Firehose to access Splunk endpoint (no explicit IAM action needed, but you can restrict VPC/network access separately)
       # Allow logging to CloudWatch
       {
         Effect = "Allow",
diff --git a/infrastructure/stacks/api-layer/iam_roles.tf b/infrastructure/stacks/api-layer/iam_roles.tf
index 22244bbe7..0289bb8cf 100644
--- a/infrastructure/stacks/api-layer/iam_roles.tf
+++ b/infrastructure/stacks/api-layer/iam_roles.tf
@@ -46,9 +46,14 @@ resource "aws_iam_role" "splunk_firehose_assume_role" {
       Action = "sts:AssumeRole"
     }]
   })
+  tags = {
+    Environment = var.environment
+    Purpose     = "firehose-service-role"
+    ManagedBy   = "terraform"
+  }
 }
 
-# Roles
+# Note: EventBridge IAM roles are defined in eventbridge.tf for proper separation of concerns
 
 resource "aws_iam_role" "eligibility_lambda_role" {
   name                 = "eligibility_lambda-role${terraform.workspace == "default" ? "" : "-${terraform.workspace}"}"

From 8efdf122f3dc2d96ef115a4ec0161e6122082cc5 Mon Sep 17 00:00:00 2001
From: Edd Almond <102675624+eddalmond1@users.noreply.github.com>
Date: Thu, 7 Aug 2025 10:46:42 +0100
Subject: [PATCH 04/22] eli-304 adding kms key rotation for splunk_hec_kms

---
 infrastructure/stacks/api-layer/ssm.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/infrastructure/stacks/api-layer/ssm.tf b/infrastructure/stacks/api-layer/ssm.tf
index 5667fba48..f66ea5b44 100644
--- a/infrastructure/stacks/api-layer/ssm.tf
+++ b/infrastructure/stacks/api-layer/ssm.tf
@@ -1,7 +1,7 @@
 resource "aws_kms_key" "splunk_hec_kms" {
   description             = "KMS key for encrypting Splunk HEC SSM parameters"
   deletion_window_in_days = 7
-
+  enable_key_rotation     = true
   tags = {
     Name        = "splunk-hec-ssm-kms-key"
     Environment = var.environment

From 2f61b9e5bc75c9d5eab0b03938d0b6bfe85ea547 Mon Sep 17 00:00:00 2001
From: Edd Almond <102675624+eddalmond1@users.noreply.github.com>
Date: Thu, 7 Aug 2025 10:48:20 +0100
Subject: [PATCH 05/22] eli-304 removing blank lines

---
 infrastructure/modules/splunk_forwarder/firehose.tf | 2 --
 1 file changed, 2 deletions(-)

diff --git a/infrastructure/modules/splunk_forwarder/firehose.tf b/infrastructure/modules/splunk_forwarder/firehose.tf
index 535569e5b..83d35538a 100644
--- a/infrastructure/modules/splunk_forwarder/firehose.tf
+++ b/infrastructure/modules/splunk_forwarder/firehose.tf
@@ -1,5 +1,3 @@
-
-
 # KMS Key for Firehose encryption
 resource "aws_kms_key" "firehose_splunk_cmk" {
   description             = "KMS key for encrypting Kinesis Firehose delivery stream data"

From 5a983b95b226663a66eeca3a572b7d92c574e915 Mon Sep 17 00:00:00 2001
From: Edd Almond <102675624+eddalmond1@users.noreply.github.com>
Date: Thu, 7 Aug 2025 10:49:15 +0100
Subject: [PATCH 06/22] eli-304 removing redundant comment

---
 infrastructure/modules/splunk_forwarder/firehose.tf | 2 --
 1 file changed, 2 deletions(-)

diff --git a/infrastructure/modules/splunk_forwarder/firehose.tf b/infrastructure/modules/splunk_forwarder/firehose.tf
index 83d35538a..a8b429278 100644
--- a/infrastructure/modules/splunk_forwarder/firehose.tf
+++ b/infrastructure/modules/splunk_forwarder/firehose.tf
@@ -16,8 +16,6 @@ resource "aws_kms_alias" "firehose_splunk_cmk_alias" {
   target_key_id = aws_kms_key.firehose_splunk_cmk.key_id
 }
 
-# KMS Key Policy for Firehose
-
 resource "aws_kinesis_firehose_delivery_stream" "splunk_delivery_stream" {
   name        = "splunk-alarm-events"
   destination = "splunk"

From 47e17ec8373bf120fe51763931bac6bf0fb0d243 Mon Sep 17 00:00:00 2001
From: Edd Almond <102675624+eddalmond1@users.noreply.github.com>
Date: Wed, 13 Aug 2025 13:51:24 +0100
Subject: [PATCH 07/22] eli-304 adding overwrite to allow initial deploy

---
 infrastructure/stacks/api-layer/ssm.tf | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/infrastructure/stacks/api-layer/ssm.tf b/infrastructure/stacks/api-layer/ssm.tf
index f66ea5b44..afa4bff6d 100644
--- a/infrastructure/stacks/api-layer/ssm.tf
+++ b/infrastructure/stacks/api-layer/ssm.tf
@@ -60,6 +60,7 @@ resource "aws_ssm_parameter" "splunk_hec_token" {
   key_id      = aws_kms_key.splunk_hec_kms.id
   value       = "REPLACE_ME" # Set a placeholder value
   tier        = "Advanced"
+  overwrite   = true # Allow overwriting existing parameter
 
   tags = {
     Environment = var.environment
@@ -80,6 +81,7 @@ resource "aws_ssm_parameter" "splunk_hec_endpoint" {
   key_id      = aws_kms_key.splunk_hec_kms.id
   value       = "REPLACE_ME" # Set a placeholder value
   tier        = "Advanced"
+  overwrite   = true # Allow overwriting existing parameter
 
   tags = {
     Environment = var.environment

From 0879b69fbc04a9bc166824284ae97648e8680c1b Mon Sep 17 00:00:00 2001
From: Edd Almond <102675624+eddalmond1@users.noreply.github.com>
Date: Wed, 13 Aug 2025 14:22:45 +0100
Subject: [PATCH 08/22] eli-304 added deployment instructions for new ssm
 parameters

---
 .../stacks/api-layer/splunk_forwarder.tf      |  5 ++++
 infrastructure/stacks/api-layer/ssm.tf        | 25 ++++++++++++++-----
 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/infrastructure/stacks/api-layer/splunk_forwarder.tf b/infrastructure/stacks/api-layer/splunk_forwarder.tf
index 24862e592..1ad9f670a 100644
--- a/infrastructure/stacks/api-layer/splunk_forwarder.tf
+++ b/infrastructure/stacks/api-layer/splunk_forwarder.tf
@@ -5,4 +5,9 @@ module "splunk_forwarder" {
   splunk_hec_token              = data.aws_ssm_parameter.splunk_hec_token.value
   splunk_firehose_s3_role_arn   = aws_iam_role.splunk_firehose_assume_role.arn
   splunk_firehose_s3_backup_arn = module.s3_firehose_backup_bucket.storage_bucket_arn
+
+  depends_on = [
+    aws_ssm_parameter.splunk_hec_token,
+    aws_ssm_parameter.splunk_hec_endpoint
+  ]
 }
diff --git a/infrastructure/stacks/api-layer/ssm.tf b/infrastructure/stacks/api-layer/ssm.tf
index afa4bff6d..ebad3807b 100644
--- a/infrastructure/stacks/api-layer/ssm.tf
+++ b/infrastructure/stacks/api-layer/ssm.tf
@@ -1,3 +1,18 @@
+# In order to allow deployment, we need to:
+# 1. Manually deploy the SSM parameters using the AWS CLI or console
+#
+# aws ssm put-parameter \
+#   --name "/splunk/hec/token" \
+#   --value "ACTUAL_HEC_TOKEN" \
+#   --type "SecureString" \
+#   --tier "Advanced" \
+#   --description "Splunk HEC token"
+#
+# 2. Import the existing parameters into Terraform state:
+# terraform import aws_ssm_parameter.splunk_hec_token "/splunk/hec/token"
+# terraform import aws_ssm_parameter.splunk_hec_endpoint "/splunk/hec/endpoint"
+# 3. Deploy the Terraform
+
 resource "aws_kms_key" "splunk_hec_kms" {
   description             = "KMS key for encrypting Splunk HEC SSM parameters"
   deletion_window_in_days = 7
@@ -58,9 +73,8 @@ resource "aws_ssm_parameter" "splunk_hec_token" {
   description = "Splunk HEC token"
   type        = "SecureString"
   key_id      = aws_kms_key.splunk_hec_kms.id
-  value       = "REPLACE_ME" # Set a placeholder value
+  value       = "PLACEHOLDER" # This will be ignored due to lifecycle rule
   tier        = "Advanced"
-  overwrite   = true # Allow overwriting existing parameter
 
   tags = {
     Environment = var.environment
@@ -70,7 +84,7 @@ resource "aws_ssm_parameter" "splunk_hec_token" {
   }
 
   lifecycle {
-    ignore_changes = [value]
+    ignore_changes = [value, key_id] # Ignore value and key_id changes to preserve existing setup
   }
 }
 
@@ -79,9 +93,8 @@ resource "aws_ssm_parameter" "splunk_hec_endpoint" {
   description = "Splunk HEC endpoint"
   type        = "SecureString"
   key_id      = aws_kms_key.splunk_hec_kms.id
-  value       = "REPLACE_ME" # Set a placeholder value
+  value       = "PLACEHOLDER" # This will be ignored due to lifecycle rule
   tier        = "Advanced"
-  overwrite   = true # Allow overwriting existing parameter
 
   tags = {
     Environment = var.environment
@@ -91,6 +104,6 @@ resource "aws_ssm_parameter" "splunk_hec_endpoint" {
   }
 
   lifecycle {
-    ignore_changes = [value]
+    ignore_changes = [value, key_id] # Ignore value and key_id changes to preserve existing setup
   }
 }

From bdb1d59577b1a4355bd404339df825b03817a043 Mon Sep 17 00:00:00 2001
From: Edd Almond <102675624+eddalmond1@users.noreply.github.com>
Date: Wed, 13 Aug 2025 14:31:55 +0100
Subject: [PATCH 09/22] eli-304 added README.md for api-layer stack

---
 infrastructure/stacks/api-layer/README.md | 101 ++++++++++++++++++++++
 infrastructure/stacks/api-layer/ssm.tf    |  15 ----
 2 files changed, 101 insertions(+), 15 deletions(-)
 create mode 100644 infrastructure/stacks/api-layer/README.md

diff --git a/infrastructure/stacks/api-layer/README.md b/infrastructure/stacks/api-layer/README.md
new file mode 100644
index 000000000..a5b99bbb3
--- /dev/null
+++ b/infrastructure/stacks/api-layer/README.md
@@ -0,0 +1,101 @@
+# API Layer Stack
+
+## Overview
+
+The API Layer stack is the core infrastructure stack that provisions the main application components for the Eligibility Signposting API. This stack creates and manages the runtime environment, data storage, monitoring, and logging infrastructure required for the application to operate.
+
+## Components
+
+### Core Infrastructure
+
+- **Lambda Functions**: Serverless compute for the eligibility signposting API
+- **DynamoDB Table**: Primary data store for eligibility status information
+- **S3 Buckets**: Storage for rules, audit data, truststore, and backup files
+- **API Gateway**: HTTP API endpoint for external access
+- **VPC Configuration**: Networking and security groups for isolated execution
+
+### Data & Storage
+
+- **DynamoDB**: Encrypted table with KMS key for eligibility data storage
+- **S3 Buckets**:
+  - Rules bucket for configuration data
+  - Audit bucket for compliance logging
+  - Truststore bucket for certificates
+  - Firehose backup bucket for failed log delivery
+
+### Monitoring & Logging
+
+- **CloudWatch Alarms**: Monitoring for application health and performance
+- **Kinesis Firehose**: Log streaming to Splunk for centralized monitoring
+- **EventBridge**: Alarm forwarding to external monitoring systems
+- **X-Ray Tracing**: Distributed tracing for request tracking
+
+### Security
+
+- **KMS Keys**: Encryption for DynamoDB, S3, SNS, and SSM parameters
+- **IAM Roles**: Service-specific roles with least privilege access
+- **Permissions Boundaries**: Additional security controls for assumed roles
+- **SSM Parameters**: Secure storage for sensitive configuration
+
+## Dependencies
+
+This stack depends on:
+
+- **Bootstrap Stack**: Terraform state storage and base infrastructure
+- **Networking Stack**: VPC, subnets, and security groups
+- **IAMS Developer Roles Stack**: GitHub Actions deployment role
+
+## Pre-Deployment Requirements
+
+### Splunk Integration Setup
+
+Before deploying this stack, you must manually create the required SSM parameters for Splunk integration:
+
+#### 1. Create SSM Parameters Manually
+
+```bash
+# Create the HEC token parameter with your actual Splunk HEC token
+aws ssm put-parameter \
+  --name "/splunk/hec/token" \
+  --value "YOUR_ACTUAL_HEC_TOKEN" \
+  --type "SecureString" \
+  --tier "Advanced" \
+  --description "Splunk HEC token"
+
+# Create the HEC endpoint parameter with your actual Splunk endpoint
+aws ssm put-parameter \
+  --name "/splunk/hec/endpoint" \
+  --value "https://your-splunk-instance.com:8088/services/collector/event" \
+  --type "SecureString" \
+  --tier "Advanced" \
+  --description "Splunk HEC endpoint"
+```
+
+#### 2. Import Parameters into Terraform State
+
+```bash
+# Navigate to the api-layer stack directory
+cd infrastructure/stacks/api-layer
+
+# Import the existing parameters into Terraform state
+terraform import aws_ssm_parameter.splunk_hec_token "/splunk/hec/token"
+terraform import aws_ssm_parameter.splunk_hec_endpoint "/splunk/hec/endpoint"
+```
+
+#### 3. Deploy the Stack
+
+```bash
+# Plan the deployment to verify no changes to parameter values
+make terraform env=<env> workspace=default stack=api-layer tf-command=plan
+
+# Apply the changes
+make terraform env=<env> workspace=default stack=api-layer tf-command=apply
+```
+
+## Environment Variables
+
+The stack uses the following key variables:
+
+- `environment`: Deployment environment (dev, staging, prod)
+- `project_name`: Project identifier for resource naming
+- `default_aws_region`: AWS region for resource deployment
diff --git a/infrastructure/stacks/api-layer/ssm.tf b/infrastructure/stacks/api-layer/ssm.tf
index ebad3807b..fec6ed1fd 100644
--- a/infrastructure/stacks/api-layer/ssm.tf
+++ b/infrastructure/stacks/api-layer/ssm.tf
@@ -1,18 +1,3 @@
-# In order to allow deployment, we need to:
-# 1. Manually deploy the SSM parameters using the AWS CLI or console
-#
-# aws ssm put-parameter \
-#   --name "/splunk/hec/token" \
-#   --value "ACTUAL_HEC_TOKEN" \
-#   --type "SecureString" \
-#   --tier "Advanced" \
-#   --description "Splunk HEC token"
-#
-# 2. Import the existing parameters into Terraform state:
-# terraform import aws_ssm_parameter.splunk_hec_token "/splunk/hec/token"
-# terraform import aws_ssm_parameter.splunk_hec_endpoint "/splunk/hec/endpoint"
-# 3. Deploy the Terraform
-
 resource "aws_kms_key" "splunk_hec_kms" {
   description             = "KMS key for encrypting Splunk HEC SSM parameters"
   deletion_window_in_days = 7

From 9572585b73f530e21807fd0e99e74664ac01535e Mon Sep 17 00:00:00 2001
From: Edd Almond <102675624+eddalmond1@users.noreply.github.com>
Date: Wed, 13 Aug 2025 14:57:44 +0100
Subject: [PATCH 10/22] eli-304 updated deployment instructions

---
 infrastructure/stacks/api-layer/README.md | 27 ++++++++++++++++++-----
 infrastructure/stacks/api-layer/ssm.tf    | 12 +++++-----
 2 files changed, 28 insertions(+), 11 deletions(-)

diff --git a/infrastructure/stacks/api-layer/README.md b/infrastructure/stacks/api-layer/README.md
index a5b99bbb3..eefedcf90 100644
--- a/infrastructure/stacks/api-layer/README.md
+++ b/infrastructure/stacks/api-layer/README.md
@@ -49,12 +49,13 @@ This stack depends on:
 
 ### Splunk Integration Setup
 
-Before deploying this stack, you must manually create the required SSM parameters for Splunk integration:
+Before deploying this stack, you must manually create the required SSM parameters for Splunk integration. Create them initially with AWS managed encryption, then Terraform will automatically migrate them to use the customer-managed KMS key during deployment.
 
 #### 1. Create SSM Parameters Manually
 
 ```bash
 # Create the HEC token parameter with your actual Splunk HEC token
+# Use AWS managed encryption initially since customer KMS key doesn't exist yet
 aws ssm put-parameter \
   --name "/splunk/hec/token" \
   --value "YOUR_ACTUAL_HEC_TOKEN" \
@@ -85,13 +86,29 @@ terraform import aws_ssm_parameter.splunk_hec_endpoint "/splunk/hec/endpoint"
 #### 3. Deploy the Stack
 
 ```bash
-# Plan the deployment to verify no changes to parameter values
-make terraform env=<env> workspace=default stack=api-layer tf-command=plan
+# Plan the deployment - this will show the KMS key migration
+terraform plan
 
-# Apply the changes
-make terraform env=<env> workspace=default stack=api-layer tf-command=apply
+# Apply the changes - this will create the KMS key and migrate the SSM parameters
+terraform apply
 ```
 
+**Note**: During the `terraform apply`, the SSM parameters will be automatically updated to use the customer-managed KMS key while preserving their values.
+
+### Getting Splunk HEC Credentials
+
+To obtain the required Splunk HEC token and endpoint:
+
+1. **Access Splunk Instance**: Log into your Splunk deployment
+2. **Navigate to Data Inputs**: Go to Settings > Data Inputs > HTTP Event Collector
+3. **Create/Configure HEC Token**:
+   - Create a new token or use an existing one
+   - Ensure the token is enabled and has appropriate permissions
+   - Note the token value for the SSM parameter
+4. **Get HEC Endpoint**: The endpoint typically follows the format:
+   - `https://your-splunk-instance.com:8088/services/collector/event`
+   - Replace with your actual Splunk instance URL and port
+
 ## Environment Variables
 
 The stack uses the following key variables:
diff --git a/infrastructure/stacks/api-layer/ssm.tf b/infrastructure/stacks/api-layer/ssm.tf
index fec6ed1fd..7a078058f 100644
--- a/infrastructure/stacks/api-layer/ssm.tf
+++ b/infrastructure/stacks/api-layer/ssm.tf
@@ -57,8 +57,8 @@ resource "aws_ssm_parameter" "splunk_hec_token" {
   name        = "/splunk/hec/token"
   description = "Splunk HEC token"
   type        = "SecureString"
-  key_id      = aws_kms_key.splunk_hec_kms.id
-  value       = "PLACEHOLDER" # This will be ignored due to lifecycle rule
+  key_id      = aws_kms_key.splunk_hec_kms.id # Will migrate to customer key after initial creation
+  value       = "PLACEHOLDER"                 # This will be ignored due to lifecycle rule
   tier        = "Advanced"
 
   tags = {
@@ -69,7 +69,7 @@ resource "aws_ssm_parameter" "splunk_hec_token" {
   }
 
   lifecycle {
-    ignore_changes = [value, key_id] # Ignore value and key_id changes to preserve existing setup
+    ignore_changes = [value] # Only ignore value changes, allow key_id updates during migration
   }
 }
 
@@ -77,8 +77,8 @@ resource "aws_ssm_parameter" "splunk_hec_endpoint" {
   name        = "/splunk/hec/endpoint"
   description = "Splunk HEC endpoint"
   type        = "SecureString"
-  key_id      = aws_kms_key.splunk_hec_kms.id
-  value       = "PLACEHOLDER" # This will be ignored due to lifecycle rule
+  key_id      = aws_kms_key.splunk_hec_kms.id # Will migrate to customer key after initial creation
+  value       = "PLACEHOLDER"                 # This will be ignored due to lifecycle rule
   tier        = "Advanced"
 
   tags = {
@@ -89,6 +89,6 @@ resource "aws_ssm_parameter" "splunk_hec_endpoint" {
   }
 
   lifecycle {
-    ignore_changes = [value, key_id] # Ignore value and key_id changes to preserve existing setup
+    ignore_changes = [value] # Only ignore value changes, allow key_id updates during migration
   }
 }

From bdbba4c725c704df2572410322ddda554aaa0935 Mon Sep 17 00:00:00 2001
From: Edd Almond <102675624+eddalmond1@users.noreply.github.com>
Date: Wed, 13 Aug 2025 14:59:45 +0100
Subject: [PATCH 11/22] eli-304 vale corrections

---
 infrastructure/stacks/api-layer/README.md                       | 2 +-
 scripts/config/vale/styles/config/vocabularies/words/accept.txt | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/infrastructure/stacks/api-layer/README.md b/infrastructure/stacks/api-layer/README.md
index eefedcf90..2af1f54d1 100644
--- a/infrastructure/stacks/api-layer/README.md
+++ b/infrastructure/stacks/api-layer/README.md
@@ -113,6 +113,6 @@ To obtain the required Splunk HEC token and endpoint:
 
 The stack uses the following key variables:
 
-- `environment`: Deployment environment (dev, staging, prod)
+- `environment`: Deployment environment ("dev", "test", "preprod", "prod")
 - `project_name`: Project identifier for resource naming
 - `default_aws_region`: AWS region for resource deployment
diff --git a/scripts/config/vale/styles/config/vocabularies/words/accept.txt b/scripts/config/vale/styles/config/vocabularies/words/accept.txt
index 7ed71edb8..fb9904e36 100644
--- a/scripts/config/vale/styles/config/vocabularies/words/accept.txt
+++ b/scripts/config/vale/styles/config/vocabularies/words/accept.txt
@@ -17,6 +17,7 @@ README
 Redocly
 sed
 subnet
+Splunk
 Syft
 Terraform
 toolchain

From 23bab3fad2d8a7f31ebd6275fc9c4e32585e5353 Mon Sep 17 00:00:00 2001
From: Edd Almond <102675624+eddalmond1@users.noreply.github.com>
Date: Wed, 13 Aug 2025 15:02:17 +0100
Subject: [PATCH 12/22] eli-304 adding dev and preprod to word allowlist

---
 scripts/config/vale/styles/config/vocabularies/words/accept.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/config/vale/styles/config/vocabularies/words/accept.txt b/scripts/config/vale/styles/config/vocabularies/words/accept.txt
index fb9904e36..38ebac5bf 100644
--- a/scripts/config/vale/styles/config/vocabularies/words/accept.txt
+++ b/scripts/config/vale/styles/config/vocabularies/words/accept.txt
@@ -3,6 +3,7 @@ Bitwarden
 bot
 Cyber
 Dependabot
+dev
 Gitleaks
 Grype
 idempotence
@@ -11,6 +12,7 @@ OAuth
 Octokit
 onboarding
 Podman
+preprod
 Proxygen
 Python
 README

From 77d6f992aed9e11d50aef0d8aa7bd87093431b8b Mon Sep 17 00:00:00 2001
From: Edd Almond <102675624+eddalmond1@users.noreply.github.com>
Date: Tue, 19 Aug 2025 17:03:34 +0100
Subject: [PATCH 13/22] eli-304 shortening bucket name

---
 infrastructure/stacks/api-layer/s3_buckets.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/infrastructure/stacks/api-layer/s3_buckets.tf b/infrastructure/stacks/api-layer/s3_buckets.tf
index ab087b406..a2dc4af53 100644
--- a/infrastructure/stacks/api-layer/s3_buckets.tf
+++ b/infrastructure/stacks/api-layer/s3_buckets.tf
@@ -19,7 +19,7 @@ module "s3_audit_bucket" {
 
 module "s3_firehose_backup_bucket" {
   source       = "../../modules/s3"
-  bucket_name  = "eli-splunk-backup"
+  bucket_name  = "eli-splunk"
   environment  = var.environment
   project_name = var.project_name
   stack_name   = local.stack_name

From 3e887c6c46f63e791684b72d2fbff9977808e8e3 Mon Sep 17 00:00:00 2001
From: Edd Almond <102675624+eddalmond1@users.noreply.github.com>
Date: Wed, 20 Aug 2025 13:21:27 +0100
Subject: [PATCH 14/22] eli-304 vale issues, addressing in markdown

---
 infrastructure/stacks/api-layer/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/infrastructure/stacks/api-layer/README.md b/infrastructure/stacks/api-layer/README.md
index 2af1f54d1..2430c0aba 100644
--- a/infrastructure/stacks/api-layer/README.md
+++ b/infrastructure/stacks/api-layer/README.md
@@ -113,6 +113,6 @@ To obtain the required Splunk HEC token and endpoint:
 
 The stack uses the following key variables:
 
-- `environment`: Deployment environment ("dev", "test", "preprod", "prod")
+- `environment`: Deployment environment (`dev`, `test`, `preprod`, `prod`)
 - `project_name`: Project identifier for resource naming
 - `default_aws_region`: AWS region for resource deployment

From e73a67962dcc0acfaeb1c269cfff6c3ac890782a Mon Sep 17 00:00:00 2001
From: Edd Almond <102675624+eddalmond1@users.noreply.github.com>
Date: Wed, 20 Aug 2025 15:15:14 +0100
Subject: [PATCH 15/22] eli-304 dealing with ssm  + secrets

---
 .github/workflows/base-deploy.yml                   |  3 +++
 .github/workflows/cicd-2-publish.yaml               |  4 +++-
 .github/workflows/cicd-3-test.yaml                  |  3 ++-
 .github/workflows/manual-terraform-apply.yaml       |  3 ++-
 infrastructure/stacks/api-layer/data.tf             |  9 ---------
 infrastructure/stacks/api-layer/splunk_forwarder.tf |  4 ++--
 infrastructure/stacks/api-layer/ssm.tf              |  4 ++--
 infrastructure/stacks/api-layer/variables.tf        | 10 ++++++++++
 8 files changed, 24 insertions(+), 16 deletions(-)
 create mode 100644 infrastructure/stacks/api-layer/variables.tf

diff --git a/.github/workflows/base-deploy.yml b/.github/workflows/base-deploy.yml
index 1e83ff4f9..0529a22ac 100644
--- a/.github/workflows/base-deploy.yml
+++ b/.github/workflows/base-deploy.yml
@@ -131,6 +131,9 @@ jobs:
           TF_VAR_API_CA_CERT: ${{ secrets.API_CA_CERT }}
           TF_VAR_API_CLIENT_CERT: ${{ secrets.API_CLIENT_CERT }}
           TF_VAR_API_PRIVATE_KEY_CERT: ${{ secrets.API_PRIVATE_KEY_CERT }}
+          TF_VAR_SPLUNK_HEC_TOKEN: ${{ secrets.SPLUNK_HEC_TOKEN }}
+          TF_VAR_SPLUNK_HEC_ENDPOINT: ${{ secrets.SPLUNK_HEC_ENDPOINT }}
+
         working-directory: ./infrastructure
         shell: bash
         run: |
diff --git a/.github/workflows/cicd-2-publish.yaml b/.github/workflows/cicd-2-publish.yaml
index 0c0977b14..2dc06cb37 100644
--- a/.github/workflows/cicd-2-publish.yaml
+++ b/.github/workflows/cicd-2-publish.yaml
@@ -93,13 +93,15 @@ jobs:
           role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/service-roles/github-actions-api-deployment-role
           aws-region: eu-west-2
 
-      - name: "Terraform Plan Stacks"
+      - name: "Terraform Apply"
         env:
           ENVIRONMENT: dev
           WORKSPACE: "default"
           TF_VAR_API_CA_CERT: ${{ secrets.API_CA_CERT }}
           TF_VAR_API_CLIENT_CERT: ${{ secrets.API_CLIENT_CERT }}
           TF_VAR_API_PRIVATE_KEY_CERT: ${{ secrets.API_PRIVATE_KEY_CERT }}
+          TF_VAR_SPLUNK_HEC_TOKEN: ${{ secrets.SPLUNK_HEC_TOKEN }}
+          TF_VAR_SPLUNK_HEC_ENDPOINT: ${{ secrets.SPLUNK_HEC_ENDPOINT }}
 
         # just planning for now for safety and until review
         run: |
diff --git a/.github/workflows/cicd-3-test.yaml b/.github/workflows/cicd-3-test.yaml
index d9ef5af5e..bcd0609de 100644
--- a/.github/workflows/cicd-3-test.yaml
+++ b/.github/workflows/cicd-3-test.yaml
@@ -119,7 +119,8 @@ jobs:
           TF_VAR_API_CA_CERT: ${{ secrets.API_CA_CERT }}
           TF_VAR_API_CLIENT_CERT: ${{ secrets.API_CLIENT_CERT }}
           TF_VAR_API_PRIVATE_KEY_CERT: ${{ secrets.API_PRIVATE_KEY_CERT }}
-
+          TF_VAR_SPLUNK_HEC_TOKEN: ${{ secrets.SPLUNK_HEC_TOKEN }}
+          TF_VAR_SPLUNK_HEC_ENDPOINT: ${{ secrets.SPLUNK_HEC_ENDPOINT }}
         run: |
           mkdir -p ./build
           echo "Running: make terraform env=$ENVIRONMENT workspace=$WORKSPACE stack=networking tf-command=apply"
diff --git a/.github/workflows/manual-terraform-apply.yaml b/.github/workflows/manual-terraform-apply.yaml
index dd34483fa..8d29e4230 100644
--- a/.github/workflows/manual-terraform-apply.yaml
+++ b/.github/workflows/manual-terraform-apply.yaml
@@ -63,7 +63,8 @@ jobs:
           TF_VAR_API_CA_CERT: ${{ secrets.API_CA_CERT }}
           TF_VAR_API_CLIENT_CERT: ${{ secrets.API_CLIENT_CERT }}
           TF_VAR_API_PRIVATE_KEY_CERT: ${{ secrets.API_PRIVATE_KEY_CERT }}
-
+          TF_VAR_SPLUNK_HEC_TOKEN: ${{ secrets.SPLUNK_HEC_TOKEN }}
+          TF_VAR_SPLUNK_HEC_ENDPOINT: ${{ secrets.SPLUNK_HEC_ENDPOINT }}
         run: |
           mkdir -p ./build
           echo "Running: make terraform env=$ENVIRONMENT workspace=$WORKSPACE stack=networking tf-command=plan args=\"-auto-approve\""
diff --git a/infrastructure/stacks/api-layer/data.tf b/infrastructure/stacks/api-layer/data.tf
index 9318b2acc..6b159ad98 100644
--- a/infrastructure/stacks/api-layer/data.tf
+++ b/infrastructure/stacks/api-layer/data.tf
@@ -28,12 +28,3 @@ data "aws_ssm_parameter" "mtls_api_ca_cert" {
   name            = "/${var.environment}/mtls/api_ca_cert"
   with_decryption = true
 }
-
-data "aws_ssm_parameter" "splunk_hec_token" {
-  name = "/splunk/hec/token"
-  with_decryption = true
-}
-data "aws_ssm_parameter" "splunk_hec_endpoint" {
-  name = "/splunk/hec/endpoint"
-  with_decryption = true
-}
diff --git a/infrastructure/stacks/api-layer/splunk_forwarder.tf b/infrastructure/stacks/api-layer/splunk_forwarder.tf
index 1ad9f670a..d794496d1 100644
--- a/infrastructure/stacks/api-layer/splunk_forwarder.tf
+++ b/infrastructure/stacks/api-layer/splunk_forwarder.tf
@@ -1,8 +1,8 @@
 module "splunk_forwarder" {
   source = "../../modules/splunk_forwarder"
 
-  splunk_hec_endpoint           = data.aws_ssm_parameter.splunk_hec_endpoint.value
-  splunk_hec_token              = data.aws_ssm_parameter.splunk_hec_token.value
+  splunk_hec_endpoint           = aws_ssm_parameter.splunk_hec_endpoint.value
+  splunk_hec_token              = aws_ssm_parameter.splunk_hec_token.value
   splunk_firehose_s3_role_arn   = aws_iam_role.splunk_firehose_assume_role.arn
   splunk_firehose_s3_backup_arn = module.s3_firehose_backup_bucket.storage_bucket_arn
 
diff --git a/infrastructure/stacks/api-layer/ssm.tf b/infrastructure/stacks/api-layer/ssm.tf
index 7a078058f..efd6e9a93 100644
--- a/infrastructure/stacks/api-layer/ssm.tf
+++ b/infrastructure/stacks/api-layer/ssm.tf
@@ -58,7 +58,7 @@ resource "aws_ssm_parameter" "splunk_hec_token" {
   description = "Splunk HEC token"
   type        = "SecureString"
   key_id      = aws_kms_key.splunk_hec_kms.id # Will migrate to customer key after initial creation
-  value       = "PLACEHOLDER"                 # This will be ignored due to lifecycle rule
+  value       = var.splunk_hec_token
   tier        = "Advanced"
 
   tags = {
@@ -78,7 +78,7 @@ resource "aws_ssm_parameter" "splunk_hec_endpoint" {
   description = "Splunk HEC endpoint"
   type        = "SecureString"
   key_id      = aws_kms_key.splunk_hec_kms.id # Will migrate to customer key after initial creation
-  value       = "PLACEHOLDER"                 # This will be ignored due to lifecycle rule
+  value       = var.splunk_hec_endpoint
   tier        = "Advanced"
 
   tags = {
diff --git a/infrastructure/stacks/api-layer/variables.tf b/infrastructure/stacks/api-layer/variables.tf
new file mode 100644
index 000000000..b88139b12
--- /dev/null
+++ b/infrastructure/stacks/api-layer/variables.tf
@@ -0,0 +1,10 @@
+variable "splunk_hec_token" {
+  type        = string
+  description = "The HEC token for ITOC splunk"
+  sensitive   = true
+}
+variable "splunk_hec_endpoint" {
+  type        = string
+  description = "The HEC endpoint url for ITOC splunk"
+  sensitive   = true
+}

From 1ca53524a4d6baa058695eca9e2fdbeedc69438a Mon Sep 17 00:00:00 2001
From: Edd Almond <102675624+eddalmond1@users.noreply.github.com>
Date: Wed, 20 Aug 2025 15:47:39 +0100
Subject: [PATCH 16/22] eli-304 changing formatting of message for Splunk

---
 .../stacks/api-layer/eventbridge.tf           | 22 +++++++++++--------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/infrastructure/stacks/api-layer/eventbridge.tf b/infrastructure/stacks/api-layer/eventbridge.tf
index e9d013a3e..c8badff17 100644
--- a/infrastructure/stacks/api-layer/eventbridge.tf
+++ b/infrastructure/stacks/api-layer/eventbridge.tf
@@ -74,15 +74,19 @@ resource "aws_cloudwatch_event_target" "firehose_target" {
     }
 
     input_template = jsonencode({
-      timestamp   = "<time>"
-      source      = "aws:cloudwatch:alarm"
-      account_id  = "<account>"
-      region      = "<region>"
-      alarm_name  = "<alarm_name>"
-      new_state   = "<new_state>"
-      old_state   = "<old_state>"
-      reason      = "<reason>"
-      severity    = "<new_state>" == "ALARM" ? "high" : "info"
+      time = "<time>"
+      host = "aws"
+      source = "aws:cloudwatch:alarm"
+      sourcetype = "aws:cloudwatch:alarm"
+      event = {
+        account_id  = "<account>"
+        region      = "<region>"
+        alarm_name  = "<alarm_name>"
+        new_state   = "<new_state>"
+        old_state   = "<old_state>"
+        reason      = "<reason>"
+        severity    = "<new_state>" == "ALARM" ? "high" : "info"
+      }
     })
   }
 }

From fcfc4f1bb2362b46f7388a7683a8658acc798fa6 Mon Sep 17 00:00:00 2001
From: Edd Almond <102675624+eddalmond1@users.noreply.github.com>
Date: Thu, 21 Aug 2025 13:24:03 +0100
Subject: [PATCH 17/22] eli-304 amending firehose to use 'raw' endpoint

---
 infrastructure/modules/splunk_forwarder/firehose.tf | 2 +-
 infrastructure/stacks/api-layer/eventbridge.tf      | 6 ++----
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/infrastructure/modules/splunk_forwarder/firehose.tf b/infrastructure/modules/splunk_forwarder/firehose.tf
index a8b429278..f7ba21549 100644
--- a/infrastructure/modules/splunk_forwarder/firehose.tf
+++ b/infrastructure/modules/splunk_forwarder/firehose.tf
@@ -31,7 +31,7 @@ resource "aws_kinesis_firehose_delivery_stream" "splunk_delivery_stream" {
   splunk_configuration {
     hec_endpoint      = var.splunk_hec_endpoint
     hec_token         = var.splunk_hec_token
-    hec_endpoint_type = "Event"
+    hec_endpoint_type = "Raw"
     s3_backup_mode    = "FailedEventsOnly"
 
     s3_configuration {
diff --git a/infrastructure/stacks/api-layer/eventbridge.tf b/infrastructure/stacks/api-layer/eventbridge.tf
index c8badff17..8fb22c625 100644
--- a/infrastructure/stacks/api-layer/eventbridge.tf
+++ b/infrastructure/stacks/api-layer/eventbridge.tf
@@ -75,17 +75,15 @@ resource "aws_cloudwatch_event_target" "firehose_target" {
 
     input_template = jsonencode({
       time = "<time>"
-      host = "aws"
-      source = "aws:cloudwatch:alarm"
+      source = "elid-${var.environment}:cloudwatch:alarm"
       sourcetype = "aws:cloudwatch:alarm"
       event = {
-        account_id  = "<account>"
-        region      = "<region>"
         alarm_name  = "<alarm_name>"
         new_state   = "<new_state>"
         old_state   = "<old_state>"
         reason      = "<reason>"
         severity    = "<new_state>" == "ALARM" ? "high" : "info"
+        region      = "<region>"
       }
     })
   }

From ce694f802ef97904509022f3b111a24c26c992d2 Mon Sep 17 00:00:00 2001
From: Edd Almond <102675624+eddalmond1@users.noreply.github.com>
Date: Thu, 21 Aug 2025 13:32:07 +0100
Subject: [PATCH 18/22] eli-304 removing severity as we can do this logic in
 Splunk

---
 infrastructure/stacks/api-layer/eventbridge.tf | 1 -
 1 file changed, 1 deletion(-)

diff --git a/infrastructure/stacks/api-layer/eventbridge.tf b/infrastructure/stacks/api-layer/eventbridge.tf
index 8fb22c625..99c97f90d 100644
--- a/infrastructure/stacks/api-layer/eventbridge.tf
+++ b/infrastructure/stacks/api-layer/eventbridge.tf
@@ -82,7 +82,6 @@ resource "aws_cloudwatch_event_target" "firehose_target" {
         new_state   = "<new_state>"
         old_state   = "<old_state>"
         reason      = "<reason>"
-        severity    = "<new_state>" == "ALARM" ? "high" : "info"
         region      = "<region>"
       }
     })

From ac6457ea5ab5167096ed2e6f3892e82a88b07a92 Mon Sep 17 00:00:00 2001
From: Edd Almond <102675624+eddalmond1@users.noreply.github.com>
Date: Fri, 22 Aug 2025 11:58:18 +0100
Subject: [PATCH 19/22] eli-304 removing readme, as worked out a better way to
 deal with SSM

---
 infrastructure/stacks/api-layer/README.md | 118 ----------------------
 1 file changed, 118 deletions(-)
 delete mode 100644 infrastructure/stacks/api-layer/README.md

diff --git a/infrastructure/stacks/api-layer/README.md b/infrastructure/stacks/api-layer/README.md
deleted file mode 100644
index 2430c0aba..000000000
--- a/infrastructure/stacks/api-layer/README.md
+++ /dev/null
@@ -1,118 +0,0 @@
-# API Layer Stack
-
-## Overview
-
-The API Layer stack is the core infrastructure stack that provisions the main application components for the Eligibility Signposting API. This stack creates and manages the runtime environment, data storage, monitoring, and logging infrastructure required for the application to operate.
-
-## Components
-
-### Core Infrastructure
-
-- **Lambda Functions**: Serverless compute for the eligibility signposting API
-- **DynamoDB Table**: Primary data store for eligibility status information
-- **S3 Buckets**: Storage for rules, audit data, truststore, and backup files
-- **API Gateway**: HTTP API endpoint for external access
-- **VPC Configuration**: Networking and security groups for isolated execution
-
-### Data & Storage
-
-- **DynamoDB**: Encrypted table with KMS key for eligibility data storage
-- **S3 Buckets**:
-  - Rules bucket for configuration data
-  - Audit bucket for compliance logging
-  - Truststore bucket for certificates
-  - Firehose backup bucket for failed log delivery
-
-### Monitoring & Logging
-
-- **CloudWatch Alarms**: Monitoring for application health and performance
-- **Kinesis Firehose**: Log streaming to Splunk for centralized monitoring
-- **EventBridge**: Alarm forwarding to external monitoring systems
-- **X-Ray Tracing**: Distributed tracing for request tracking
-
-### Security
-
-- **KMS Keys**: Encryption for DynamoDB, S3, SNS, and SSM parameters
-- **IAM Roles**: Service-specific roles with least privilege access
-- **Permissions Boundaries**: Additional security controls for assumed roles
-- **SSM Parameters**: Secure storage for sensitive configuration
-
-## Dependencies
-
-This stack depends on:
-
-- **Bootstrap Stack**: Terraform state storage and base infrastructure
-- **Networking Stack**: VPC, subnets, and security groups
-- **IAMS Developer Roles Stack**: GitHub Actions deployment role
-
-## Pre-Deployment Requirements
-
-### Splunk Integration Setup
-
-Before deploying this stack, you must manually create the required SSM parameters for Splunk integration. Create them initially with AWS managed encryption, then Terraform will automatically migrate them to use the customer-managed KMS key during deployment.
-
-#### 1. Create SSM Parameters Manually
-
-```bash
-# Create the HEC token parameter with your actual Splunk HEC token
-# Use AWS managed encryption initially since customer KMS key doesn't exist yet
-aws ssm put-parameter \
-  --name "/splunk/hec/token" \
-  --value "YOUR_ACTUAL_HEC_TOKEN" \
-  --type "SecureString" \
-  --tier "Advanced" \
-  --description "Splunk HEC token"
-
-# Create the HEC endpoint parameter with your actual Splunk endpoint
-aws ssm put-parameter \
-  --name "/splunk/hec/endpoint" \
-  --value "https://your-splunk-instance.com:8088/services/collector/event" \
-  --type "SecureString" \
-  --tier "Advanced" \
-  --description "Splunk HEC endpoint"
-```
-
-#### 2. Import Parameters into Terraform State
-
-```bash
-# Navigate to the api-layer stack directory
-cd infrastructure/stacks/api-layer
-
-# Import the existing parameters into Terraform state
-terraform import aws_ssm_parameter.splunk_hec_token "/splunk/hec/token"
-terraform import aws_ssm_parameter.splunk_hec_endpoint "/splunk/hec/endpoint"
-```
-
-#### 3. Deploy the Stack
-
-```bash
-# Plan the deployment - this will show the KMS key migration
-terraform plan
-
-# Apply the changes - this will create the KMS key and migrate the SSM parameters
-terraform apply
-```
-
-**Note**: During the `terraform apply`, the SSM parameters will be automatically updated to use the customer-managed KMS key while preserving their values.
-
-### Getting Splunk HEC Credentials
-
-To obtain the required Splunk HEC token and endpoint:
-
-1. **Access Splunk Instance**: Log into your Splunk deployment
-2. **Navigate to Data Inputs**: Go to Settings > Data Inputs > HTTP Event Collector
-3. **Create/Configure HEC Token**:
-   - Create a new token or use an existing one
-   - Ensure the token is enabled and has appropriate permissions
-   - Note the token value for the SSM parameter
-4. **Get HEC Endpoint**: The endpoint typically follows the format:
-   - `https://your-splunk-instance.com:8088/services/collector/event`
-   - Replace with your actual Splunk instance URL and port
-
-## Environment Variables
-
-The stack uses the following key variables:
-
-- `environment`: Deployment environment (`dev`, `test`, `preprod`, `prod`)
-- `project_name`: Project identifier for resource naming
-- `default_aws_region`: AWS region for resource deployment

From dbbbe3f80fc9438b50bc17356f8b5ba2c8fccf49 Mon Sep 17 00:00:00 2001
From: Edd Almond <102675624+eddalmond1@users.noreply.github.com>
Date: Fri, 22 Aug 2025 12:00:39 +0100
Subject: [PATCH 20/22] eli-304 removing / changing code comments

---
 infrastructure/stacks/api-layer/ssm.tf | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/infrastructure/stacks/api-layer/ssm.tf b/infrastructure/stacks/api-layer/ssm.tf
index efd6e9a93..6b17bf939 100644
--- a/infrastructure/stacks/api-layer/ssm.tf
+++ b/infrastructure/stacks/api-layer/ssm.tf
@@ -69,7 +69,7 @@ resource "aws_ssm_parameter" "splunk_hec_token" {
   }
 
   lifecycle {
-    ignore_changes = [value] # Only ignore value changes, allow key_id updates during migration
+    ignore_changes = [value] # Needs manual changes in future
   }
 }
 
@@ -77,7 +77,7 @@ resource "aws_ssm_parameter" "splunk_hec_endpoint" {
   name        = "/splunk/hec/endpoint"
   description = "Splunk HEC endpoint"
   type        = "SecureString"
-  key_id      = aws_kms_key.splunk_hec_kms.id # Will migrate to customer key after initial creation
+  key_id      = aws_kms_key.splunk_hec_kms.id
   value       = var.splunk_hec_endpoint
   tier        = "Advanced"
 
@@ -89,6 +89,6 @@ resource "aws_ssm_parameter" "splunk_hec_endpoint" {
   }
 
   lifecycle {
-    ignore_changes = [value] # Only ignore value changes, allow key_id updates during migration
+    ignore_changes = [value]
   }
 }

From 65c13cf2ed2d29003fd43a5baeec416e1d3ddeb2 Mon Sep 17 00:00:00 2001
From: Karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com>
Date: Thu, 21 Aug 2025 23:26:06 +0100
Subject: [PATCH 21/22] Eli 387 lambda hardening (#306)

* provisioned concurrency

* enable dead letter queue

* enhanced monitoring

* lambda function versioning

* provison concurrancy - alias version fix

* removed checkov as we implemented dead letter queue

* prod conditions and github roles

* github roles

* fix for corrupt kms policy

* create queue

* get the latest function for concurrant provisioning
---
 infrastructure/modules/lambda/data.tf         |  5 ++
 infrastructure/modules/lambda/kms.tf          |  2 +-
 infrastructure/modules/lambda/lambda.tf       | 33 ++++++-
 infrastructure/modules/lambda/sqs.tf          | 23 +++++
 infrastructure/modules/lambda/variables.tf    | 23 ++++-
 .../assumed_role_permissions_boundary.tf      |  5 +-
 .../stacks/api-layer/iam_policies.tf          |  6 ++
 infrastructure/stacks/api-layer/lambda.tf     | 34 +++----
 .../github_actions_policies.tf                | 89 +++++++++++--------
 .../iams_permissions_boundary.tf              | 12 ++-
 10 files changed, 174 insertions(+), 58 deletions(-)
 create mode 100644 infrastructure/modules/lambda/sqs.tf

diff --git a/infrastructure/modules/lambda/data.tf b/infrastructure/modules/lambda/data.tf
index 8fc4b38cc..9331249e2 100644
--- a/infrastructure/modules/lambda/data.tf
+++ b/infrastructure/modules/lambda/data.tf
@@ -1 +1,6 @@
 data "aws_caller_identity" "current" {}
+
+data "aws_lambda_function" "existing" {
+  function_name = var.lambda_func_name
+  qualifier     = "$LATEST"
+}
diff --git a/infrastructure/modules/lambda/kms.tf b/infrastructure/modules/lambda/kms.tf
index 55c5133f0..738cb32b2 100644
--- a/infrastructure/modules/lambda/kms.tf
+++ b/infrastructure/modules/lambda/kms.tf
@@ -7,7 +7,7 @@ resource "aws_kms_key" "lambda_cmk" {
 }
 
 resource "aws_kms_alias" "lambda_cmk" {
-  name          = "alias/${terraform.workspace == "default" ? "" : "${terraform.workspace}-"}${var.lambda_func_name}-cmk"
+  name          = "alias/${terraform.workspace == "default" ? "" : "${terraform.workspace}-"}${var.lambda_func_name}-key"
   target_key_id = aws_kms_key.lambda_cmk.key_id
 }
 
diff --git a/infrastructure/modules/lambda/lambda.tf b/infrastructure/modules/lambda/lambda.tf
index f31a6e762..47466b3ba 100644
--- a/infrastructure/modules/lambda/lambda.tf
+++ b/infrastructure/modules/lambda/lambda.tf
@@ -1,5 +1,4 @@
 resource "aws_lambda_function" "eligibility_signposting_lambda" {
-  #checkov:skip=CKV_AWS_116: No deadletter queue is configured for this Lambda function, yet
   #checkov:skip=CKV_AWS_115: Concurrent execution limit will be set at APIM level, not at Lambda level
   #checkov:skip=CKV_AWS_272: Skipping code signing but flagged to create ticket to investigate on ELI-238
   # If the file is not in the current working directory you will need to include a
@@ -11,7 +10,7 @@ resource "aws_lambda_function" "eligibility_signposting_lambda" {
 
   source_code_hash = filebase64sha256(var.file_name)
 
-  runtime     = "python3.13"
+  runtime     = var.runtime
   timeout     = 30
   memory_size = 2048
 
@@ -33,7 +32,37 @@ resource "aws_lambda_function" "eligibility_signposting_lambda" {
     security_group_ids = var.security_group_ids
   }
 
+  dead_letter_config {
+    target_arn = aws_sqs_queue.lambda_dlq.arn
+  }
+
+  layers = compact([
+  var.environment == "prod" ? "arn:aws:lambda:${var.region}:580247275435:layer:LambdaInsightsExtension:${var.lambda_insights_extension_version}" : null
+  ])
+
   tracing_config {
     mode = "Active"
   }
 }
+
+# lambda alias required for provisioning concurrency
+resource "aws_lambda_alias" "campaign_alias" {
+  count            = var.environment == "prod" ? 1 : 0
+  name             = "live"
+  function_name    = coalesce(
+    aws_lambda_function.eligibility_signposting_lambda.function_name,
+    data.aws_lambda_function.existing.function_name
+  )
+  function_version = coalesce(
+    aws_lambda_function.eligibility_signposting_lambda.version,
+    data.aws_lambda_function.existing.version
+  )
+}
+
+# provisioned concurrency - number of pre-warmed lambda containers
+resource "aws_lambda_provisioned_concurrency_config" "campaign_pc" {
+  count                             = var.environment == "prod" ? 1 : 0
+  function_name                     = var.lambda_func_name
+  qualifier                         = aws_lambda_alias.campaign_alias[0].name
+  provisioned_concurrent_executions = var.provisioned_concurrency_count
+}
diff --git a/infrastructure/modules/lambda/sqs.tf b/infrastructure/modules/lambda/sqs.tf
new file mode 100644
index 000000000..f538b8383
--- /dev/null
+++ b/infrastructure/modules/lambda/sqs.tf
@@ -0,0 +1,23 @@
+resource "aws_sqs_queue" "lambda_dlq" {
+  name = "${var.lambda_func_name}_dead_letter_queue"
+  kms_master_key_id = aws_kms_key.lambda_cmk.id
+  tags = var.tags
+}
+
+# sql policy attachment
+resource "aws_iam_role_policy" "lambda_sqs_send_inline" {
+  name = "LambdaSQSMessageSendPolicy"
+  role = var.eligibility_lambda_role_name
+
+  policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      {
+        Sid      = "AllowSQSSendMessage",
+        Effect   = "Allow",
+        Action   = ["sqs:SendMessage"],
+        Resource = aws_sqs_queue.lambda_dlq.arn
+      }
+    ]
+  })
+}
diff --git a/infrastructure/modules/lambda/variables.tf b/infrastructure/modules/lambda/variables.tf
index 229c1fbb4..92d9d081a 100644
--- a/infrastructure/modules/lambda/variables.tf
+++ b/infrastructure/modules/lambda/variables.tf
@@ -1,5 +1,10 @@
 variable "eligibility_lambda_role_arn" {
-  description = "lambda read role arn for dynamodb"
+  description = "lambda role arn"
+  type        = string
+}
+
+variable "eligibility_lambda_role_name" {
+  description = "lambda role name"
   type        = string
 }
 
@@ -8,6 +13,12 @@ variable "lambda_func_name" {
   type        = string
 }
 
+variable "runtime" {
+  description = "runtime of the Lambda function"
+  type        = string
+}
+
+
 variable "vpc_intra_subnets" {
   description = "vpc private subnets for lambda"
   type        = list(string)
@@ -52,3 +63,13 @@ variable "enable_xray_patching"{
   description = "flag to enable xray tracing, which puts an entry for dynamodb, s3 and firehose in trace map"
   type        = string
 }
+
+variable "provisioned_concurrency_count" {
+  description = "Number of prewarmed Lambda instances"
+  type        = number
+}
+
+variable "lambda_insights_extension_version" {
+  description = "version number of LambdaInsightsExtension"
+  type        = number
+}
diff --git a/infrastructure/stacks/api-layer/assumed_role_permissions_boundary.tf b/infrastructure/stacks/api-layer/assumed_role_permissions_boundary.tf
index efc3168b2..7a9e28f94 100644
--- a/infrastructure/stacks/api-layer/assumed_role_permissions_boundary.tf
+++ b/infrastructure/stacks/api-layer/assumed_role_permissions_boundary.tf
@@ -52,7 +52,10 @@ data "aws_iam_policy_document" "assumed_role_permissions_boundary" {
 
       # X-Ray - Lambda tracing
       "xray:PutTraceSegments",
-      "xray:PutTelemetryRecords"
+      "xray:PutTelemetryRecords",
+
+      #SQS - message management
+      "sqs:SendMessage"
     ]
 
     resources = ["*"]
diff --git a/infrastructure/stacks/api-layer/iam_policies.tf b/infrastructure/stacks/api-layer/iam_policies.tf
index 106326de1..7e4ed0b31 100644
--- a/infrastructure/stacks/api-layer/iam_policies.tf
+++ b/infrastructure/stacks/api-layer/iam_policies.tf
@@ -189,6 +189,12 @@ resource "aws_iam_role_policy_attachment" "lambda_logs_policy_attachment" {
   policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
 }
 
+#Attach CloudWatchLambdaInsightsExecutionRolePolicy to lambda for enhanced monitoring
+resource "aws_iam_role_policy_attachment" "lambda_insights_policy" {
+  role       = aws_iam_role.eligibility_lambda_role.name
+  policy_arn = "arn:aws:iam::aws:policy/CloudWatchLambdaInsightsExecutionRolePolicy"
+}
+
 # Policy doc for S3 Audit bucket
 data "aws_iam_policy_document" "s3_audit_bucket_policy" {
   statement {
diff --git a/infrastructure/stacks/api-layer/lambda.tf b/infrastructure/stacks/api-layer/lambda.tf
index 68885b6d7..5c71a2787 100644
--- a/infrastructure/stacks/api-layer/lambda.tf
+++ b/infrastructure/stacks/api-layer/lambda.tf
@@ -11,19 +11,23 @@ data "aws_subnet" "private_subnets" {
 }
 
 module "eligibility_signposting_lambda_function" {
-  source                          = "../../modules/lambda"
-  eligibility_lambda_role_arn     = aws_iam_role.eligibility_lambda_role.arn
-  workspace                       = local.workspace
-  environment                     = var.environment
-  lambda_func_name                = "${terraform.workspace == "default" ? "" : "${terraform.workspace}-"}eligibility_signposting_api"
-  security_group_ids              = [data.aws_security_group.main_sg.id]
-  vpc_intra_subnets               = [for v in data.aws_subnet.private_subnets : v.id]
-  file_name                       = "../../../dist/lambda.zip"
-  handler                         = "eligibility_signposting_api.app.lambda_handler"
-  eligibility_rules_bucket_name   = module.s3_rules_bucket.storage_bucket_name
-  eligibility_status_table_name   = module.eligibility_status_table.table_name
-  kinesis_audit_stream_to_s3_name = module.eligibility_audit_firehose_delivery_stream.firehose_stream_name
-  log_level                       = "INFO"
-  enable_xray_patching            = "true"
-  stack_name                      = local.stack_name
+  source                            = "../../modules/lambda"
+  eligibility_lambda_role_arn       = aws_iam_role.eligibility_lambda_role.arn
+  eligibility_lambda_role_name      = aws_iam_role.eligibility_lambda_role.name
+  workspace                         = local.workspace
+  environment                       = var.environment
+  runtime                           = "python3.13"
+  lambda_func_name                  = "${terraform.workspace == "default" ? "" : "${terraform.workspace}-"}eligibility_signposting_api"
+  security_group_ids                = [data.aws_security_group.main_sg.id]
+  vpc_intra_subnets                 = [for v in data.aws_subnet.private_subnets : v.id]
+  file_name                         = "../../../dist/lambda.zip"
+  handler                           = "eligibility_signposting_api.app.lambda_handler"
+  eligibility_rules_bucket_name     = module.s3_rules_bucket.storage_bucket_name
+  eligibility_status_table_name     = module.eligibility_status_table.table_name
+  kinesis_audit_stream_to_s3_name   = module.eligibility_audit_firehose_delivery_stream.firehose_stream_name
+  lambda_insights_extension_version = 38
+  log_level                         = "INFO"
+  enable_xray_patching              = "true"
+  stack_name                        = local.stack_name
+  provisioned_concurrency_count     = 5
 }
diff --git a/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf b/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf
index cb72a7966..22e3aa645 100644
--- a/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf
+++ b/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf
@@ -62,10 +62,17 @@ resource "aws_iam_policy" "lambda_management" {
           "lambda:ListAliases",
           "lambda:AddPermission",
           "lambda:RemovePermission",
-          "lambda:GetPolicy"
+          "lambda:GetPolicy",
+          "lambda:GetAlias",
+          "lambda:GetFunction",
+          "lambda:GetProvisionedConcurrencyConfig",
+          "lambda:GetLayerVersion",
+          "lambda:PutProvisionedConcurrencyConfig"
         ],
         Resource = [
-          "arn:aws:lambda:*:${data.aws_caller_identity.current.account_id}:function:*eligibility_signposting_api"
+          "arn:aws:lambda:*:${data.aws_caller_identity.current.account_id}:function:eligibility_signposting_api",
+          "arn:aws:lambda:*:${data.aws_caller_identity.current.account_id}:function:eligibility_signposting_api:*",
+          "arn:aws:lambda:*:580247275435:layer:LambdaInsightsExtension:*"
         ]
       }
     ]
@@ -465,29 +472,6 @@ data "aws_iam_policy_document" "github_actions_assume_role" {
   }
 }
 
-resource "aws_iam_policy" "cloudwatch_logging" {
-  name        = "cloudwatch-logging-management"
-  description = "Allow access to logging resources"
-  path        = "/service-policies/"
-
-  policy = jsonencode({
-    Version = "2012-10-17",
-    Statement = [
-      {
-        Effect = "Allow",
-        Action = [
-          "logs:ListTagsForResource",
-          "logs:DescribeLogGroups",
-          "logs:PutRetentionPolicy"
-        ],
-        Resource = "arn:aws:logs:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:log-group:/aws/kinesisfirehose/*"
-      }
-    ]
-  })
-
-  tags = merge(local.tags, { Name = "cloudwatch-logging-management" })
-}
-
 resource "aws_iam_policy" "firehose_readonly" {
   name        = "firehose-describe-access"
   description = "Allow GitHub Actions to describe Firehose delivery stream"
@@ -518,9 +502,9 @@ resource "aws_iam_policy" "firehose_readonly" {
   tags = merge(local.tags, { Name = "firehose-describe-access" })
 }
 
-resource "aws_iam_policy" "cloudwatch_alarms" {
-  name        = "cloudwatch-alarms-management"
-  description = "Allow GitHub Actions to manage CloudWatch alarms and SNS topics"
+resource "aws_iam_policy" "cloudwatch_management" {
+  name        = "cloudwatch-management"
+  description = "Allow GitHub Actions to manage CloudWatch logs, alarms, and SNS topics"
   path        = "/service-policies/"
 
   policy = jsonencode({
@@ -529,7 +513,10 @@ resource "aws_iam_policy" "cloudwatch_alarms" {
       {
         Effect = "Allow",
         Action = [
-          # CloudWatch Alarms management
+          "logs:ListTagsForResource",
+          "logs:DescribeLogGroups",
+          "logs:PutRetentionPolicy",
+
           "cloudwatch:PutMetricAlarm",
           "cloudwatch:DeleteAlarms",
           "cloudwatch:DescribeAlarms",
@@ -537,7 +524,7 @@ resource "aws_iam_policy" "cloudwatch_alarms" {
           "cloudwatch:ListTagsForResource",
           "cloudwatch:TagResource",
           "cloudwatch:UntagResource",
-          # SNS Topic management for alarm notifications
+
           "sns:CreateTopic",
           "sns:DeleteTopic",
           "sns:GetTopicAttributes",
@@ -552,6 +539,7 @@ resource "aws_iam_policy" "cloudwatch_alarms" {
           "sns:ListSubscriptionsByTopic"
         ],
         Resource = [
+          "arn:aws:logs:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:log-group:/aws/kinesisfirehose/*",
           "arn:aws:cloudwatch:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:alarm:*",
           "arn:aws:sns:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:cloudwatch-security-alarms*"
         ]
@@ -559,7 +547,33 @@ resource "aws_iam_policy" "cloudwatch_alarms" {
     ]
   })
 
-  tags = merge(local.tags, { Name = "cloudwatch-alarms-management" })
+  tags = merge(local.tags, { Name = "cloudwatch-management" })
+}
+
+# SQS Management Policy for GetQueueAttributes
+resource "aws_iam_policy" "sqs_management" {
+  name        = "sqs-management"
+  description = "Policy granting permissions to get SQS queue attributes"
+  path        = "/service-policies/"
+
+  policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      {
+        Effect = "Allow",
+        Action = [
+          "sqs:GetQueueAttributes",
+          "sqs:listqueuetags",
+          "sqs:createqueue"
+        ],
+        Resource = [
+          "arn:aws:sqs:eu-west-2:${data.aws_caller_identity.current.account_id}:*"
+        ]
+      }
+    ]
+  })
+
+  tags = merge(local.tags, { Name = "sqs-management" })
 }
 
 # Attach the policies to the role
@@ -598,17 +612,18 @@ resource "aws_iam_role_policy_attachment" "iam_management" {
   policy_arn = aws_iam_policy.iam_management.arn
 }
 
-resource "aws_iam_role_policy_attachment" "cloudwatch_logging" {
+resource "aws_iam_role_policy_attachment" "firehose_readonly_attach" {
   role       = aws_iam_role.github_actions.name
-  policy_arn = aws_iam_policy.cloudwatch_logging.arn
+  policy_arn = aws_iam_policy.firehose_readonly.arn
 }
 
-resource "aws_iam_role_policy_attachment" "firehose_readonly_attach" {
+resource "aws_iam_role_policy_attachment" "cloudwatch_management" {
   role       = aws_iam_role.github_actions.name
-  policy_arn = aws_iam_policy.firehose_readonly.arn
+  policy_arn = aws_iam_policy.cloudwatch_management.arn
 }
 
-resource "aws_iam_role_policy_attachment" "cloudwatch_alarms" {
+resource "aws_iam_role_policy_attachment" "sqs_management" {
   role       = aws_iam_role.github_actions.name
-  policy_arn = aws_iam_policy.cloudwatch_alarms.arn
+  policy_arn = aws_iam_policy.sqs_management.arn
 }
+
diff --git a/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf b/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf
index 4b37cde68..7631ab5a7 100644
--- a/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf
+++ b/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf
@@ -150,6 +150,10 @@ data "aws_iam_policy_document" "permissions_boundary" {
       "lambda:AddPermission",
       "lambda:RemovePermission",
       "lambda:GetPolicy",
+      "lambda:GetAlias",
+      "lambda:GetProvisionedConcurrencyConfig",
+      "lambda:GetLayerVersion",
+      "lambda:PutProvisionedConcurrencyConfig",
 
       # CloudWatch Logs - log management
       "logs:CreateLogGroup",
@@ -217,7 +221,13 @@ data "aws_iam_policy_document" "permissions_boundary" {
       "ssm:GetParameters",
       "ssm:ListTagsForResource",
       "ssm:PutParameter",
-      "ssm:AddTagsToResource"
+      "ssm:AddTagsToResource",
+
+      #SQS - message management
+      "sqs:SendMessage",
+      "sqs:GetQueueAttributes",
+      "sqs:listqueuetags",
+      "sqs:createqueue"
     ]
 
     resources = ["*"]

From 68fa0f1815cc4b5bbe933ef5a20a52b2a4343689 Mon Sep 17 00:00:00 2001
From: Karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com>
Date: Fri, 22 Aug 2025 09:46:54 +0100
Subject: [PATCH 22/22] Lambda versioning for provisioned concurrancy (#309)

* lambda versioning for provisioned concurrency

* dlq is not for RequestResponse (sync)

* checkov skip for dlq
---
 infrastructure/modules/lambda/data.tf         |  5 ---
 infrastructure/modules/lambda/lambda.tf       | 18 ++++-------
 infrastructure/modules/lambda/sqs.tf          | 23 -------------
 .../assumed_role_permissions_boundary.tf      |  5 +--
 .../github_actions_policies.tf                | 32 -------------------
 .../iams_permissions_boundary.tf              |  8 +----
 6 files changed, 8 insertions(+), 83 deletions(-)
 delete mode 100644 infrastructure/modules/lambda/sqs.tf

diff --git a/infrastructure/modules/lambda/data.tf b/infrastructure/modules/lambda/data.tf
index 9331249e2..8fc4b38cc 100644
--- a/infrastructure/modules/lambda/data.tf
+++ b/infrastructure/modules/lambda/data.tf
@@ -1,6 +1 @@
 data "aws_caller_identity" "current" {}
-
-data "aws_lambda_function" "existing" {
-  function_name = var.lambda_func_name
-  qualifier     = "$LATEST"
-}
diff --git a/infrastructure/modules/lambda/lambda.tf b/infrastructure/modules/lambda/lambda.tf
index 47466b3ba..717fbb0a7 100644
--- a/infrastructure/modules/lambda/lambda.tf
+++ b/infrastructure/modules/lambda/lambda.tf
@@ -1,4 +1,5 @@
 resource "aws_lambda_function" "eligibility_signposting_lambda" {
+  #checkov:skip=CKV_AWS_116: No deadletter queue is configured for this Lambda function, as the requests are synchronous
   #checkov:skip=CKV_AWS_115: Concurrent execution limit will be set at APIM level, not at Lambda level
   #checkov:skip=CKV_AWS_272: Skipping code signing but flagged to create ticket to investigate on ELI-238
   # If the file is not in the current working directory you will need to include a
@@ -27,15 +28,13 @@ resource "aws_lambda_function" "eligibility_signposting_lambda" {
 
   kms_key_arn = aws_kms_key.lambda_cmk.arn
 
+  publish = true
+
   vpc_config {
     subnet_ids         = var.vpc_intra_subnets
     security_group_ids = var.security_group_ids
   }
 
-  dead_letter_config {
-    target_arn = aws_sqs_queue.lambda_dlq.arn
-  }
-
   layers = compact([
   var.environment == "prod" ? "arn:aws:lambda:${var.region}:580247275435:layer:LambdaInsightsExtension:${var.lambda_insights_extension_version}" : null
   ])
@@ -49,14 +48,8 @@ resource "aws_lambda_function" "eligibility_signposting_lambda" {
 resource "aws_lambda_alias" "campaign_alias" {
   count            = var.environment == "prod" ? 1 : 0
   name             = "live"
-  function_name    = coalesce(
-    aws_lambda_function.eligibility_signposting_lambda.function_name,
-    data.aws_lambda_function.existing.function_name
-  )
-  function_version = coalesce(
-    aws_lambda_function.eligibility_signposting_lambda.version,
-    data.aws_lambda_function.existing.version
-  )
+  function_name    = aws_lambda_function.eligibility_signposting_lambda.function_name
+  function_version = aws_lambda_function.eligibility_signposting_lambda.version
 }
 
 # provisioned concurrency - number of pre-warmed lambda containers
@@ -66,3 +59,4 @@ resource "aws_lambda_provisioned_concurrency_config" "campaign_pc" {
   qualifier                         = aws_lambda_alias.campaign_alias[0].name
   provisioned_concurrent_executions = var.provisioned_concurrency_count
 }
+
diff --git a/infrastructure/modules/lambda/sqs.tf b/infrastructure/modules/lambda/sqs.tf
deleted file mode 100644
index f538b8383..000000000
--- a/infrastructure/modules/lambda/sqs.tf
+++ /dev/null
@@ -1,23 +0,0 @@
-resource "aws_sqs_queue" "lambda_dlq" {
-  name = "${var.lambda_func_name}_dead_letter_queue"
-  kms_master_key_id = aws_kms_key.lambda_cmk.id
-  tags = var.tags
-}
-
-# sql policy attachment
-resource "aws_iam_role_policy" "lambda_sqs_send_inline" {
-  name = "LambdaSQSMessageSendPolicy"
-  role = var.eligibility_lambda_role_name
-
-  policy = jsonencode({
-    Version = "2012-10-17",
-    Statement = [
-      {
-        Sid      = "AllowSQSSendMessage",
-        Effect   = "Allow",
-        Action   = ["sqs:SendMessage"],
-        Resource = aws_sqs_queue.lambda_dlq.arn
-      }
-    ]
-  })
-}
diff --git a/infrastructure/stacks/api-layer/assumed_role_permissions_boundary.tf b/infrastructure/stacks/api-layer/assumed_role_permissions_boundary.tf
index 7a9e28f94..efc3168b2 100644
--- a/infrastructure/stacks/api-layer/assumed_role_permissions_boundary.tf
+++ b/infrastructure/stacks/api-layer/assumed_role_permissions_boundary.tf
@@ -52,10 +52,7 @@ data "aws_iam_policy_document" "assumed_role_permissions_boundary" {
 
       # X-Ray - Lambda tracing
       "xray:PutTraceSegments",
-      "xray:PutTelemetryRecords",
-
-      #SQS - message management
-      "sqs:SendMessage"
+      "xray:PutTelemetryRecords"
     ]
 
     resources = ["*"]
diff --git a/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf b/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf
index 22e3aa645..9918d1b40 100644
--- a/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf
+++ b/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf
@@ -550,32 +550,6 @@ resource "aws_iam_policy" "cloudwatch_management" {
   tags = merge(local.tags, { Name = "cloudwatch-management" })
 }
 
-# SQS Management Policy for GetQueueAttributes
-resource "aws_iam_policy" "sqs_management" {
-  name        = "sqs-management"
-  description = "Policy granting permissions to get SQS queue attributes"
-  path        = "/service-policies/"
-
-  policy = jsonencode({
-    Version = "2012-10-17",
-    Statement = [
-      {
-        Effect = "Allow",
-        Action = [
-          "sqs:GetQueueAttributes",
-          "sqs:listqueuetags",
-          "sqs:createqueue"
-        ],
-        Resource = [
-          "arn:aws:sqs:eu-west-2:${data.aws_caller_identity.current.account_id}:*"
-        ]
-      }
-    ]
-  })
-
-  tags = merge(local.tags, { Name = "sqs-management" })
-}
-
 # Attach the policies to the role
 resource "aws_iam_role_policy_attachment" "terraform_state" {
   role       = aws_iam_role.github_actions.name
@@ -621,9 +595,3 @@ resource "aws_iam_role_policy_attachment" "cloudwatch_management" {
   role       = aws_iam_role.github_actions.name
   policy_arn = aws_iam_policy.cloudwatch_management.arn
 }
-
-resource "aws_iam_role_policy_attachment" "sqs_management" {
-  role       = aws_iam_role.github_actions.name
-  policy_arn = aws_iam_policy.sqs_management.arn
-}
-
diff --git a/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf b/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf
index 7631ab5a7..a403aa8a7 100644
--- a/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf
+++ b/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf
@@ -221,13 +221,7 @@ data "aws_iam_policy_document" "permissions_boundary" {
       "ssm:GetParameters",
       "ssm:ListTagsForResource",
       "ssm:PutParameter",
-      "ssm:AddTagsToResource",
-
-      #SQS - message management
-      "sqs:SendMessage",
-      "sqs:GetQueueAttributes",
-      "sqs:listqueuetags",
-      "sqs:createqueue"
+      "ssm:AddTagsToResource"
     ]
 
     resources = ["*"]