diff --git a/.github/workflows/cicd-4-preprod-deploy.yml b/.github/workflows/cicd-4a-preprod-deploy.yml similarity index 100% rename from .github/workflows/cicd-4-preprod-deploy.yml rename to .github/workflows/cicd-4a-preprod-deploy.yml diff --git a/.github/workflows/cicd-4b-preprod-seed-users.yml b/.github/workflows/cicd-4b-preprod-seed-users.yml new file mode 100644 index 000000000..af04f0f5f --- /dev/null +++ b/.github/workflows/cicd-4b-preprod-seed-users.yml @@ -0,0 +1,59 @@ +name: preprod - Seed DynamoDB table + +concurrency: + group: seed-preprod-dynamodb + cancel-in-progress: false + +on: + workflow_run: + workflows: [ "Preprod Deploy" ] + types: + - completed + filters: + conclusion: + - success + workflow_dispatch: + inputs: + environment: + description: Target environment + required: true + type: choice + options: + - preprod + +jobs: + seed-dynamodb: + runs-on: ubuntu-latest + environment: "preprod" + permissions: + id-token: write + contents: read + env: + AWS_REGION: eu-west-2 + DATA_FOLDER: tests/e2e/data/dynamoDB/vitaIntegrationTestData + DYNAMODB_TABLE: eligibility-signposting-api-preprod-eligibility_datastore + + steps: + - name: Checkout repo + uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install dependencies + run: pip install boto3 + + - name: "Configure AWS Credentials" + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/service-roles/github-actions-api-deployment-role + aws-region: ${{ env.AWS_REGION }} + + - name: Run seed script + run: | + python scripts/seed_users/seed_dynamodb.py \ + --table-name "${{ env.DYNAMODB_TABLE }}" \ + --region "${{ env.AWS_REGION }}" \ + --data-folder "${{ env.DATA_FOLDER }}" diff --git a/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf b/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf index 020926f1e..ee671e8fa 100644 --- a/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf +++ b/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf @@ -89,24 +89,42 @@ resource "aws_iam_policy" "dynamodb_management" { policy = jsonencode({ Version = "2012-10-17", - Statement = [ - { - Effect = "Allow", - Action = [ - "dynamodb:DescribeTimeToLive", - "dynamodb:DescribeTable", - "dynamodb:DescribeContinuousBackups", - "dynamodb:ListTables", - "dynamodb:DeleteTable", - "dynamodb:CreateTable", - "dynamodb:TagResource", - "dynamodb:ListTagsOfResource", - ], - Resource = [ - "arn:aws:dynamodb:*:${data.aws_caller_identity.current.account_id}:table/*eligibility-signposting-api-${var.environment}-eligibility_datastore" - ] - } - ] + Statement = concat( + [ + { + Effect = "Allow", + Action = [ + "dynamodb:DescribeTimeToLive", + "dynamodb:DescribeTable", + "dynamodb:DescribeContinuousBackups", + "dynamodb:ListTables", + "dynamodb:DeleteTable", + "dynamodb:CreateTable", + "dynamodb:TagResource", + "dynamodb:ListTagsOfResource", + ], + Resource = [ + "arn:aws:dynamodb:*:${data.aws_caller_identity.current.account_id}:table/*eligibility-signposting-api-${var.environment}-eligibility_datastore" + ] + } + ], + # to create test users in preprod + var.environment == "preprod" ? [ + { + Effect = "Allow", + Action = [ + "dynamodb:GetItem", + "dynamodb:PutItem", + "dynamodb:DeleteItem", + "dynamodb:Scan", + "dynamodb:BatchWriteItem" + ], + Resource = [ + "arn:aws:dynamodb:*:${data.aws_caller_identity.current.account_id}:table/*eligibility-signposting-api-${var.environment}-eligibility_datastore" + ] + } + ] : [] + ) }) tags = merge(local.tags, { Name = "dynamodb-management" }) @@ -465,8 +483,8 @@ resource "aws_iam_policy" "iam_management" { # Assume role policy document for GitHub Actions data "aws_iam_policy_document" "github_actions_assume_role" { statement { - sid = "OidcAssumeRoleWithWebIdentity" - effect = "Allow" + sid = "OidcAssumeRoleWithWebIdentity" + effect = "Allow" actions = ["sts:AssumeRoleWithWebIdentity"] principals { @@ -479,13 +497,13 @@ data "aws_iam_policy_document" "github_actions_assume_role" { condition { test = "StringLike" variable = "token.actions.githubusercontent.com:sub" - values = ["repo:${var.github_org}/${var.github_repo}:*"] + values = ["repo:${var.github_org}/${var.github_repo}:*"] } condition { test = "StringEquals" variable = "token.actions.githubusercontent.com:aud" - values = ["sts.amazonaws.com"] + values = ["sts.amazonaws.com"] } } } @@ -514,8 +532,8 @@ resource "aws_iam_policy" "firehose_readonly" { "firehose:StopDeliveryStreamEncryption" ] Resource = [ - "arn:aws:firehose:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:deliverystream/eligibility-signposting-api*", - "arn:aws:firehose:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:deliverystream/splunk-alarm-events*" + "arn:aws:firehose:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:deliverystream/eligibility-signposting-api*", + "arn:aws:firehose:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:deliverystream/splunk-alarm-events*" ] } ] diff --git a/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf b/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf index 10f3669c0..c31f806d7 100644 --- a/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf +++ b/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf @@ -233,7 +233,22 @@ data "aws_iam_policy_document" "permissions_boundary" { values = [var.default_aws_region] } } - + # Environment-specific actions + dynamic "statement" { + for_each = var.environment == "preprod" ? [1] : [] + content { + sid = "AllowPreprodDynamoDBItemOps" + effect = "Allow" + actions = [ + "dynamodb:GetItem", + "dynamodb:PutItem", + "dynamodb:DeleteItem", + "dynamodb:Scan", + "dynamodb:BatchWriteItem" + ] + resources = ["*"] + } + } # Allow access to IAM actions for us-east-1 region only statement { sid = "AllowIamActionsInUsEast1" diff --git a/scripts/seed_users/README.md b/scripts/seed_users/README.md new file mode 100644 index 000000000..3116ce9ef --- /dev/null +++ b/scripts/seed_users/README.md @@ -0,0 +1,71 @@ +# 🧬 DynamoDB Seeder Script + +This script deletes and inserts items into a DynamoDB table using JSON seed data. It’s designed for integration testing and local development workflows. +This script is user in the Preprod seed workflow. + +--- + +## πŸ“¦ Requirements + +- Python 3.13 +- AWS credentials configured (via `~/.aws/credentials`, environment variables, or IAM role) +- Required Python packages: + + ```bash + pip install boto3 + ``` + +--- + +## πŸš€ Usage + +From the project root, run: + +```bash +python scripts/seed_users/seed_dynamodb.py \ + --table-name \ + --region \ + --data-folder +``` + +### Example + +```bash +python scripts/seed_users/seed_dynamodb.py \ + --table-name eligibility-signposting-api-dev-eligibility_datastore \ + --region eu-west-2 \ + --data-folder tests/e2e/data/dynamoDB/vitaIntegrationTestData +``` + +--- + +## πŸ“ JSON Data Format + +Each `.json` file in the specified folder should follow this structure: + +```json +{ + "data": [ + { + "NHS_NUMBER": "1234567890", + "ATTRIBUTE_TYPE": "COHORTS", + "otherAttribute1": "value", + "otherAttribute2": "value" + } + ] +} +``` + +## 🧹 What It Does + +1. **Deletes** existing items in the table matching `NHS_NUMBER` from all JSON files. +2. **Inserts** all items from the same files into the table. + +--- + +## πŸ›‘οΈ Safety Notes + +- This script performs destructive operations β€” do not use this in prod environment. +- Ensure your AWS credentials have appropriate permissions for `dynamodb:DeleteItem` and `dynamodb:PutItem`. + +--- diff --git a/scripts/seed_users/__init__.py b/scripts/seed_users/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/seed_users/seed_dynamodb.py b/scripts/seed_users/seed_dynamodb.py new file mode 100644 index 000000000..d5e4dea6e --- /dev/null +++ b/scripts/seed_users/seed_dynamodb.py @@ -0,0 +1,82 @@ +import argparse +import glob +import json +import os + +import boto3 + + +def parse_args(): + parser = argparse.ArgumentParser(description="Seed DynamoDB table with JSON data.") + parser.add_argument("--table-name", required=True, help="Name of the DynamoDB table") + parser.add_argument("--region", default="eu-west-2", help="AWS region") + parser.add_argument("--data-folder", default="vitaIntegrationTestData/", help="Folder containing JSON seed data") + return parser.parse_args() + + +def resolve_data_folder(path): + return os.path.abspath(path) + + +def get_unique_nhs_numbers(data_folder): + nhs_numbers = set() + json_files = glob.glob(os.path.join(data_folder, "*.json")) + for file_path in json_files: + with open(file_path) as f: + payload = json.load(f) + items = payload.get("data", []) + for item in items: + nhs_number = item.get("NHS_NUMBER") + if nhs_number: + nhs_numbers.add(nhs_number) + return list(nhs_numbers) + + +def delete_all_items_for_nhs_numbers(table, nhs_numbers): + for nhs_number in nhs_numbers: + response = table.query( + KeyConditionExpression=boto3.dynamodb.conditions.Key("NHS_NUMBER").eq(nhs_number) + ) + items = response.get("Items", []) + with table.batch_writer() as batch: + for item in items: + key = { + "NHS_NUMBER": item["NHS_NUMBER"], + "ATTRIBUTE_TYPE": item["ATTRIBUTE_TYPE"] + } + batch.delete_item(Key=key) + + +def insert_data_from_folder(table, data_folder): + json_files = glob.glob(os.path.join(data_folder, "*.json")) + for file_path in json_files: + with open(file_path) as f: + payload = json.load(f) + items = payload.get("data", []) + + with table.batch_writer() as batch: + for item in items: + nhs_number = item.get("NHS_NUMBER") + attr_type = item.get("ATTRIBUTE_TYPE") + if nhs_number and attr_type: + item["id"] = nhs_number + batch.put_item(Item=item) + + +def main(): + args = parse_args() + + dynamodb = boto3.resource("dynamodb", region_name=args.region) + table = dynamodb.Table(args.table_name) + + data_folder = resolve_data_folder(args.data_folder) + if not os.path.isdir(data_folder): + raise ValueError(f"Data folder '{data_folder}' does not exist or is not a directory.") + + nhs_numbers = get_unique_nhs_numbers(data_folder) + delete_all_items_for_nhs_numbers(table, nhs_numbers) + insert_data_from_folder(table, data_folder) + + +if __name__ == "__main__": + main()