Eli 417 preprod db seeding

.github/workflows/cicd-4b-preprod-seed-users.yml

@@ -0,0 +1,59 @@
+name: preprod - Seed DynamoDB table
+
+concurrency:
+  group: seed-preprod-dynamodb
+  cancel-in-progress: false
+
+on:
+  workflow_run:
+    workflows: [ "Preprod Deploy" ]
+    types:
+      - completed
+    filters:
+      conclusion:
+        - success
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: Target environment
+        required: true
+        type: choice
+        options:
+          - preprod
+
+jobs:
+  seed-dynamodb:
+    runs-on: ubuntu-latest
+    environment: "preprod"
+    permissions:
+      id-token: write
+      contents: read
+    env:
+      AWS_REGION: eu-west-2
+      DATA_FOLDER: tests/e2e/data/dynamoDB/vitaIntegrationTestData
+      DYNAMODB_TABLE: eligibility-signposting-api-preprod-eligibility_datastore
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v5
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
+      - name: Install dependencies
+        run: pip install boto3
+
+      - name: "Configure AWS Credentials"
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/service-roles/github-actions-api-deployment-role
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Run seed script
+        run: |
+          python scripts/seed_users/seed_dynamodb.py \
+            --table-name "${{ env.DYNAMODB_TABLE }}" \
+            --region "${{ env.AWS_REGION }}" \
+            --data-folder "${{ env.DATA_FOLDER }}"

infrastructure/stacks/iams-developer-roles/github_actions_policies.tf

@@ -89,24 +89,42 @@ resource "aws_iam_policy" "dynamodb_management" {
 
   policy = jsonencode({
     Version = "2012-10-17",
-    Statement = [
-      {
-        Effect = "Allow",
-        Action = [
-          "dynamodb:DescribeTimeToLive",
-          "dynamodb:DescribeTable",
-          "dynamodb:DescribeContinuousBackups",
-          "dynamodb:ListTables",
-          "dynamodb:DeleteTable",
-          "dynamodb:CreateTable",
-          "dynamodb:TagResource",
-          "dynamodb:ListTagsOfResource",
-        ],
-        Resource = [
-          "arn:aws:dynamodb:*:${data.aws_caller_identity.current.account_id}:table/*eligibility-signposting-api-${var.environment}-eligibility_datastore"
-        ]
-      }
-    ]
+    Statement = concat(
+      [
+        {
+          Effect = "Allow",
+          Action = [
+            "dynamodb:DescribeTimeToLive",
+            "dynamodb:DescribeTable",
+            "dynamodb:DescribeContinuousBackups",
+            "dynamodb:ListTables",
+            "dynamodb:DeleteTable",
+            "dynamodb:CreateTable",
+            "dynamodb:TagResource",
+            "dynamodb:ListTagsOfResource",
+          ],
+          Resource = [
+            "arn:aws:dynamodb:*:${data.aws_caller_identity.current.account_id}:table/*eligibility-signposting-api-${var.environment}-eligibility_datastore"
+          ]
+        }
+      ],
+      # to create test users in preprod
+      var.environment == "preprod" ? [
+        {
+          Effect = "Allow",
+          Action = [
+            "dynamodb:GetItem",
+            "dynamodb:PutItem",
+            "dynamodb:DeleteItem",
+            "dynamodb:Scan",
+            "dynamodb:BatchWriteItem"
+          ],
+          Resource = [
+            "arn:aws:dynamodb:*:${data.aws_caller_identity.current.account_id}:table/*eligibility-signposting-api-${var.environment}-eligibility_datastore"
+          ]
+        }
+      ] : []
+    )
   })
 
   tags = merge(local.tags, { Name = "dynamodb-management" })
@@ -465,8 +483,8 @@ resource "aws_iam_policy" "iam_management" {
 # Assume role policy document for GitHub Actions
 data "aws_iam_policy_document" "github_actions_assume_role" {
   statement {
-    sid     = "OidcAssumeRoleWithWebIdentity"
-    effect  = "Allow"
+    sid    = "OidcAssumeRoleWithWebIdentity"
+    effect = "Allow"
     actions = ["sts:AssumeRoleWithWebIdentity"]
 
     principals {
@@ -479,13 +497,13 @@ data "aws_iam_policy_document" "github_actions_assume_role" {
     condition {
       test     = "StringLike"
       variable = "token.actions.githubusercontent.com:sub"
-      values   = ["repo:${var.github_org}/${var.github_repo}:*"]
+      values = ["repo:${var.github_org}/${var.github_repo}:*"]
     }
 
     condition {
       test     = "StringEquals"
       variable = "token.actions.githubusercontent.com:aud"
-      values   = ["sts.amazonaws.com"]
+      values = ["sts.amazonaws.com"]
     }
   }
 }
@@ -514,8 +532,8 @@ resource "aws_iam_policy" "firehose_readonly" {
           "firehose:StopDeliveryStreamEncryption"
         ]
         Resource = [
-            "arn:aws:firehose:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:deliverystream/eligibility-signposting-api*",
-            "arn:aws:firehose:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:deliverystream/splunk-alarm-events*"
+          "arn:aws:firehose:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:deliverystream/eligibility-signposting-api*",
+          "arn:aws:firehose:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:deliverystream/splunk-alarm-events*"
         ]
       }
     ]

infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf

@@ -233,7 +233,22 @@ data "aws_iam_policy_document" "permissions_boundary" {
       values   = [var.default_aws_region]
     }
   }
-
+  # Environment-specific actions
+  dynamic "statement" {
+    for_each = var.environment == "preprod" ? [1] : []
+    content {
+      sid    = "AllowPreprodDynamoDBItemOps"
+      effect = "Allow"
+      actions = [
+        "dynamodb:GetItem",
+        "dynamodb:PutItem",
+        "dynamodb:DeleteItem",
+        "dynamodb:Scan",
+        "dynamodb:BatchWriteItem"
+      ]
+      resources = ["*"]
+    }
+  }
   # Allow access to IAM actions for us-east-1 region only
   statement {
     sid       = "AllowIamActionsInUsEast1"

scripts/seed_users/README.md

@@ -0,0 +1,71 @@
+# 🧬 DynamoDB Seeder Script
+
+This script deletes and inserts items into a DynamoDB table using JSON seed data. It’s designed for integration testing and local development workflows.
+This script is user in the Preprod seed workflow.
+
+---
+
+## 📦 Requirements
+
+- Python 3.13
+- AWS credentials configured (via `~/.aws/credentials`, environment variables, or IAM role)
+- Required Python packages:
+
+  ```bash
+  pip install boto3
+  ```
+
+---
+
+## 🚀 Usage
+
+From the project root, run:
+
+```bash
+python scripts/seed_users/seed_dynamodb.py \
+  --table-name <your-dynamodb-table-name> \
+  --region <aws-region> \
+  --data-folder <path-to-json-folder>
+```
+
+### Example
+
+```bash
+python scripts/seed_users/seed_dynamodb.py \
+  --table-name eligibility-signposting-api-dev-eligibility_datastore \
+  --region eu-west-2 \
+  --data-folder tests/e2e/data/dynamoDB/vitaIntegrationTestData
+```
+
+---
+
+## 📁 JSON Data Format
+
+Each `.json` file in the specified folder should follow this structure:
+
+```json
+{
+  "data": [
+    {
+        "NHS_NUMBER": "1234567890",
+        "ATTRIBUTE_TYPE": "COHORTS",
+        "otherAttribute1": "value",
+        "otherAttribute2": "value"
+    }
+  ]
+}
+```
+
+## 🧹 What It Does
+
+1. **Deletes** existing items in the table matching `NHS_NUMBER` from all JSON files.
+2. **Inserts** all items from the same files into the table.
+
+---
+
+## 🛡️ Safety Notes
+
+- This script performs destructive operations — do not use this in prod environment.
+- Ensure your AWS credentials have appropriate permissions for `dynamodb:DeleteItem` and `dynamodb:PutItem`.
+
+---

scripts/seed_users/seed_dynamodb.py

@@ -0,0 +1,82 @@
+import argparse
+import glob
+import json
+import os
+
+import boto3
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Seed DynamoDB table with JSON data.")
+    parser.add_argument("--table-name", required=True, help="Name of the DynamoDB table")
+    parser.add_argument("--region", default="eu-west-2", help="AWS region")
+    parser.add_argument("--data-folder", default="vitaIntegrationTestData/", help="Folder containing JSON seed data")
+    return parser.parse_args()
+
+
+def resolve_data_folder(path):
+    return os.path.abspath(path)
+
+
+def get_unique_nhs_numbers(data_folder):
+    nhs_numbers = set()
+    json_files = glob.glob(os.path.join(data_folder, "*.json"))
+    for file_path in json_files:
+        with open(file_path) as f:
+            payload = json.load(f)
+            items = payload.get("data", [])
+            for item in items:
+                nhs_number = item.get("NHS_NUMBER")
+                if nhs_number:
+                    nhs_numbers.add(nhs_number)
+    return list(nhs_numbers)
+
+
+def delete_all_items_for_nhs_numbers(table, nhs_numbers):
+    for nhs_number in nhs_numbers:
+        response = table.query(
+            KeyConditionExpression=boto3.dynamodb.conditions.Key("NHS_NUMBER").eq(nhs_number)
+        )
+        items = response.get("Items", [])
+        with table.batch_writer() as batch:
+            for item in items:
+                key = {
+                    "NHS_NUMBER": item["NHS_NUMBER"],
+                    "ATTRIBUTE_TYPE": item["ATTRIBUTE_TYPE"]
+                }
+                batch.delete_item(Key=key)
+
+
+def insert_data_from_folder(table, data_folder):
+    json_files = glob.glob(os.path.join(data_folder, "*.json"))
+    for file_path in json_files:
+        with open(file_path) as f:
+            payload = json.load(f)
+            items = payload.get("data", [])
+
+        with table.batch_writer() as batch:
+            for item in items:
+                nhs_number = item.get("NHS_NUMBER")
+                attr_type = item.get("ATTRIBUTE_TYPE")
+                if nhs_number and attr_type:
+                    item["id"] = nhs_number
+                    batch.put_item(Item=item)
+
+
+def main():
+    args = parse_args()
+
+    dynamodb = boto3.resource("dynamodb", region_name=args.region)
+    table = dynamodb.Table(args.table_name)
+
+    data_folder = resolve_data_folder(args.data_folder)
+    if not os.path.isdir(data_folder):
+        raise ValueError(f"Data folder '{data_folder}' does not exist or is not a directory.")
+
+    nhs_numbers = get_unique_nhs_numbers(data_folder)
+    delete_all_items_for_nhs_numbers(table, nhs_numbers)
+    insert_data_from_folder(table, data_folder)
+
+
+if __name__ == "__main__":
+    main()