Merge remote-tracking branch 'origin/main' into feature/eli-418_performance-test-github-action-api-gateway-update

Author: ayeshalshukri1-nhs

.github/workflows/base-deploy.yml

@@ -125,7 +125,7 @@ jobs:
             terraform_version: ${{ needs.metadata.outputs.terraform_version }}
 
         - name: "Configure AWS Credentials"
-          uses: aws-actions/configure-aws-credentials@v5
+          uses: aws-actions/configure-aws-credentials@v6
           with:
             role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/service-roles/github-actions-api-deployment-role
             aws-region: eu-west-2
@@ -188,7 +188,7 @@ jobs:
           path: ./dist
 
       - name: "Configure AWS Credentials"
-        uses: aws-actions/configure-aws-credentials@v5
+        uses: aws-actions/configure-aws-credentials@v6
         with:
           role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/service-roles/github-actions-api-deployment-role
           aws-region: eu-west-2

.github/workflows/cicd-2-publish.yaml

@@ -88,7 +88,7 @@ jobs:
           path: dist/lambda.zip
 
       - name: "Configure AWS Credentials"
-        uses: aws-actions/configure-aws-credentials@v5
+        uses: aws-actions/configure-aws-credentials@v6
         with:
           role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/service-roles/github-actions-api-deployment-role
           aws-region: eu-west-2

.github/workflows/cicd-3-test-deploy.yaml

@@ -67,7 +67,7 @@ jobs:
           terraform_version: ${{ needs.metadata.outputs.terraform_version }}
 
       - name: "Configure AWS Credentials"
-        uses: aws-actions/configure-aws-credentials@v5
+        uses: aws-actions/configure-aws-credentials@v6
         with:
           role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/service-roles/github-actions-api-deployment-role
           aws-region: eu-west-2
@@ -130,3 +130,4 @@ jobs:
       ENVIRONMENT: "test"
       VERSION_NUMBER: "main"
     secrets: inherit
+

.github/workflows/cicd-4-preprod-deploy.yaml

@@ -36,7 +36,7 @@ jobs:
       this_sha:     ${{ steps.resolver.outputs.this_sha }}
       latest_sha:   ${{ steps.resolver.outputs.latest_test_sha }}
       release_type: ${{ steps.release_type.outputs.release_type }}
-
+    if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }}
     env:
       TEST_WORKFLOW_ID: "190123511" # this will need updating if the workflow is recreated
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -97,3 +97,4 @@ jobs:
       ref:          ${{ needs.metadata.outputs.ref }}
       release_type: ${{ needs.metadata.outputs.release_type }}
     secrets: inherit
+    if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }}

.github/workflows/monthly-capacity-report.yml

@@ -42,7 +42,7 @@ jobs:
           python-version: "3.11"
 
       - name: Configure AWS Credentials (${{ matrix.env_config.name }})
-        uses: aws-actions/configure-aws-credentials@v5
+        uses: aws-actions/configure-aws-credentials@v6
         with:
           role-to-assume: arn:aws:iam::${{ secrets[matrix.env_config.account_secret] }}:role/service-roles/github-actions-api-deployment-role
           aws-region: eu-west-2

.github/workflows/release-candidate.yml

@@ -95,7 +95,7 @@ jobs:
           terraform_version: $(grep '^terraform' .tool-versions | cut -f2 -d' ')
 
       - name: "Configure AWS Credentials (dev)"
-        uses: aws-actions/configure-aws-credentials@v5
+        uses: aws-actions/configure-aws-credentials@v6
         with:
           role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/service-roles/github-actions-api-deployment-role
           aws-region: eu-west-2
@@ -170,7 +170,7 @@ jobs:
           make build
 
       - name: "Configure AWS Credentials"
-        uses: aws-actions/configure-aws-credentials@v5
+        uses: aws-actions/configure-aws-credentials@v6
         with:
           role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/service-roles/github-actions-api-deployment-role
           aws-region: eu-west-2
@@ -222,7 +222,7 @@ jobs:
           path: dist
 
       - name: "Configure AWS Credentials (test)"
-        uses: aws-actions/configure-aws-credentials@v5
+        uses: aws-actions/configure-aws-credentials@v6
         with:
           role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/service-roles/github-actions-api-deployment-role
           aws-region: eu-west-2
@@ -323,7 +323,7 @@ jobs:
           path: dist
 
       - name: "Configure AWS Credentials (preprod)"
-        uses: aws-actions/configure-aws-credentials@v5
+        uses: aws-actions/configure-aws-credentials@v6
         with:
           role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/service-roles/github-actions-api-deployment-role
           aws-region: eu-west-2

infrastructure/modules/bootstrap/tfstate/s3.tf

@@ -80,7 +80,7 @@ resource "aws_s3_bucket_lifecycle_configuration" "tfstate_bucket" {
     }
 
     expiration {
-      days = 90
+      days = 1200
     }
 
     noncurrent_version_transition {

infrastructure/modules/s3/variables.tf

@@ -4,7 +4,7 @@ variable "bucket_name" {
 }
 
 variable "bucket_expiration_days" {
-  default     = 90
+  default     = 1200
   description = "How long to keep bucket contents before expiring"
   type        = number
 }

infrastructure/stacks/api-layer/api_gateway.tf

@@ -56,7 +56,8 @@ resource "aws_api_gateway_stage" "eligibility-signposting-api" {
   # A subscription filter (see csoc_log_forwarding.tf) forwards these logs to CSOC
   access_log_settings {
     destination_arn = module.eligibility_signposting_api_gateway.cloudwatch_destination_arn
-    format          = "{ \"requestId\":\"$context.requestId\", \"ip\": \"$context.identity.sourceIp\", \"caller\":\"$context.identity.caller\", \"user\":\"$context.identity.user\", \"requestTime\":\"$context.requestTime\", \"httpMethod\":\"$context.httpMethod\", \"resourcePath\":\"$context.resourcePath\", \"status\":\"$context.status\", \"protocol\":\"$context.protocol\", \"responseLength\":\"$context.responseLength\", \"accountId\":\"$context.accountId\", \"apiId\":\"$context.apiId\", \"stage\":\"$context.stage\", \"api_key\":\"$context.identity.apiKey\", \"responseLatency\":\"$context.responseLatency\",  \"integrationLatency\":\"$context.integrationLatency\" }"
+    format = "{ \"requestId\":\"$context.requestId\", \"ip\": \"$context.identity.sourceIp\", \"caller\":\"$context.identity.caller\", \"user\":\"$context.identity.user\", \"requestTime\":\"$context.requestTime\", \"httpMethod\":\"$context.httpMethod\", \"resourcePath\":\"$context.resourcePath\", \"status\":\"$context.status\", \"protocol\":\"$context.protocol\", \"responseLength\":\"$context.responseLength\", \"accountId\":\"$context.accountId\", \"apiId\":\"$context.apiId\", \"stage\":\"$context.stage\", \"api_key\":\"$context.identity.apiKey\" }"
+
   }
 
   depends_on = [

infrastructure/stacks/api-layer/iam_policies.tf

@@ -273,7 +273,7 @@ data "aws_iam_policy_document" "s3_audit_bucket_policy" {
   }
 }
 
-# Attach s3 write policy to external write role
+# Attach s3 write policy to lambda write role - rename below to lambda_s3_audit_write_policy
 resource "aws_iam_role_policy" "external_s3_write_policy" {
   name   = "S3WriteAccess"
   role   = aws_iam_role.eligibility_lambda_role.id
@@ -351,6 +351,7 @@ resource "aws_kms_key_policy" "s3_rules_kms_key" {
   policy = data.aws_iam_policy_document.s3_rules_kms_key_policy.json
 }
 
+# KMS key policy for consumer mapping file
 data "aws_iam_policy_document" "s3_consumer_mapping_kms_key_policy" {
   #checkov:skip=CKV_AWS_111: Root user needs full KMS key management
   #checkov:skip=CKV_AWS_356: Root user needs full KMS key management
@@ -755,3 +756,60 @@ resource "aws_iam_role_policy_attachment" "attach_rotation_sfn" {
   role       = aws_iam_role.rotation_sfn_role.name
   policy_arn = aws_iam_policy.rotation_sfn_policy.arn
 }
+
+#DQ policies
+# Policy doc for S3 DQ bucket
+data "aws_iam_policy_document" "s3_dq_bucket_policy" {
+  statement {
+    sid = "AllowSSLRequestsOnly"
+    actions = [
+      "s3:ListBucket",
+      "s3:GetBucketLocation",
+      "s3:GetObject",
+      "s3:PutObject",
+      "s3:DeleteObject", // Critical for Spark staging/committing
+      "s3:AbortMultipartUpload" // Important for large dataframes if job fails
+    ]
+    resources = [
+      module.s3_dq_metrics_bucket.storage_bucket_arn,
+      "${module.s3_dq_metrics_bucket.storage_bucket_arn}/*",
+    ]
+    condition {
+      test     = "Bool"
+      values = ["true"]
+      variable = "aws:SecureTransport"
+    }
+  }
+}
+
+# Attach DQ s3 write policy to external write role
+resource "aws_iam_role_policy" "external_dq_s3_write_policy" {
+  count = length(aws_iam_role.write_access_role)
+  name   = "S3DQWriteAccess"
+  role   = aws_iam_role.write_access_role[count.index].id
+  policy = data.aws_iam_policy_document.s3_dq_bucket_policy.json
+}
+
+# KMS access policy for S3 DQ bucket to external write role
+data "aws_iam_policy_document" "s3_dq_kms_access_policy" {
+  statement {
+    actions = [
+      "kms:Encrypt",
+      "kms:Decrypt",
+      "kms:ReEncrypt*",
+      "kms:GenerateDataKey*",
+      "kms:DescribeKey"
+    ]
+    resources = [
+      module.s3_dq_metrics_bucket.storage_bucket_kms_key_arn
+    ]
+  }
+}
+
+# Attach KMS policy to external write role
+resource "aws_iam_role_policy" "external_s3_kms_access_policy" {
+  count = length(aws_iam_role.write_access_role)
+  name   = "KMSAccessForS3DQ"
+  role   = aws_iam_role.write_access_role[count.index].id
+  policy = data.aws_iam_policy_document.s3_dq_kms_access_policy.json
+}