From 24050f228bb78c257eb3382ef30aa2bb60c550eb Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Fri, 29 Aug 2025 10:40:02 +0100 Subject: [PATCH 01/14] preprod db seeding --- ...-deploy.yml => cicd-4a-preprod-deploy.yml} | 0 .../workflows/cicd-4b-preprod-seed-users.yml | 43 +++++++++++++ .../github_actions_policies.tf | 64 ++++++++++++------- .../iams_permissions_boundary.tf | 4 ++ scripts/seed_users/__init__.py | 0 scripts/seed_users/seed_dynamodb.py | 56 ++++++++++++++++ .../AUTO_RSV_VITA_INT_001.json | 40 ++++++++++++ .../AUTO_RSV_VITA_INT_002.json | 39 +++++++++++ .../AUTO_RSV_VITA_INT_003.json | 39 +++++++++++ .../AUTO_RSV_VITA_INT_004.json | 35 ++++++++++ .../AUTO_RSV_VITA_INT_005.json | 35 ++++++++++ .../AUTO_RSV_VITA_INT_006.json | 34 ++++++++++ .../AUTO_RSV_VITA_INT_007.json | 39 +++++++++++ .../AUTO_RSV_VITA_INT_009.json | 39 +++++++++++ .../AUTO_RSV_VITA_INT_011.json | 39 +++++++++++ .../AUTO_RSV_VITA_INT_012.json | 39 +++++++++++ .../AUTO_RSV_VITA_INT_013.json | 34 ++++++++++ .../AUTO_RSV_VITA_INT_014.json | 34 ++++++++++ .../AUTO_RSV_VITA_INT_015.json | 35 ++++++++++ .../AUTO_RSV_VITA_INT_500.json | 40 ++++++++++++ 20 files changed, 664 insertions(+), 24 deletions(-) rename .github/workflows/{cicd-4-preprod-deploy.yml => cicd-4a-preprod-deploy.yml} (100%) create mode 100644 .github/workflows/cicd-4b-preprod-seed-users.yml create mode 100644 scripts/seed_users/__init__.py create mode 100644 scripts/seed_users/seed_dynamodb.py create mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_001.json create mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_002.json create mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_003.json create mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_004.json create mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_005.json create mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_006.json create mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_007.json create mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_009.json create mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_011.json create mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_012.json create mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_013.json create mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_014.json create mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_015.json create mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_500.json diff --git a/.github/workflows/cicd-4-preprod-deploy.yml b/.github/workflows/cicd-4a-preprod-deploy.yml similarity index 100% rename from .github/workflows/cicd-4-preprod-deploy.yml rename to .github/workflows/cicd-4a-preprod-deploy.yml diff --git a/.github/workflows/cicd-4b-preprod-seed-users.yml b/.github/workflows/cicd-4b-preprod-seed-users.yml new file mode 100644 index 000000000..cc874b259 --- /dev/null +++ b/.github/workflows/cicd-4b-preprod-seed-users.yml @@ -0,0 +1,43 @@ +name: Preprod - Seed DynamoDB table + +on: + push: + branches: + - ELI-417-preprod-db-seeding + +jobs: + seed: + runs-on: ubuntu-latest + environment: "dev" + permissions: + id-token: write + contents: read + env: + AWS_REGION: eu-west-2 + DATA_FOLDER: vitaIntegrationTestData/ + DYNAMODB_TABLE: eligibility-signposting-api-dev-eligibility_datastore + + steps: + - name: Checkout repo + uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install dependencies + run: pip install boto3 + + - name: "Configure AWS Credentials" + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/service-roles/github-actions-api-deployment-role + aws-region: ${{ env.AWS_REGION }} + + - name: Run seed script + run: | + python .github/scripts/seed_dynamodb.py \ + --table-name "${{ env.DYNAMODB_TABLE }}" \ + --region "${{ env.AWS_REGION }}" \ + --data-folder "${{ env.DATA_FOLDER }}" diff --git a/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf b/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf index 020926f1e..0514c1dd6 100644 --- a/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf +++ b/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf @@ -89,24 +89,40 @@ resource "aws_iam_policy" "dynamodb_management" { policy = jsonencode({ Version = "2012-10-17", - Statement = [ - { - Effect = "Allow", - Action = [ - "dynamodb:DescribeTimeToLive", - "dynamodb:DescribeTable", - "dynamodb:DescribeContinuousBackups", - "dynamodb:ListTables", - "dynamodb:DeleteTable", - "dynamodb:CreateTable", - "dynamodb:TagResource", - "dynamodb:ListTagsOfResource", - ], - Resource = [ - "arn:aws:dynamodb:*:${data.aws_caller_identity.current.account_id}:table/*eligibility-signposting-api-${var.environment}-eligibility_datastore" - ] - } - ] + Statement = concat( + [ + { + Effect = "Allow", + Action = [ + "dynamodb:DescribeTimeToLive", + "dynamodb:DescribeTable", + "dynamodb:DescribeContinuousBackups", + "dynamodb:ListTables", + "dynamodb:DeleteTable", + "dynamodb:CreateTable", + "dynamodb:TagResource", + "dynamodb:ListTagsOfResource", + ], + Resource = [ + "arn:aws:dynamodb:*:${data.aws_caller_identity.current.account_id}:table/*eligibility-signposting-api-${var.environment}-eligibility_datastore" + ] + } + ], + # to create test users in preprod + var.environment == "preprod" ? [ + { + Effect = "Allow", + Action = [ + "dynamodb:GetItem", + "dynamodb:PutItem", + "dynamodb:DeleteItem", + ], + Resource = [ + "arn:aws:dynamodb:*:${data.aws_caller_identity.current.account_id}:table/*eligibility-signposting-api-preprod-eligibility_datastore" + ] + } + ] : [] + ) }) tags = merge(local.tags, { Name = "dynamodb-management" }) @@ -465,8 +481,8 @@ resource "aws_iam_policy" "iam_management" { # Assume role policy document for GitHub Actions data "aws_iam_policy_document" "github_actions_assume_role" { statement { - sid = "OidcAssumeRoleWithWebIdentity" - effect = "Allow" + sid = "OidcAssumeRoleWithWebIdentity" + effect = "Allow" actions = ["sts:AssumeRoleWithWebIdentity"] principals { @@ -479,13 +495,13 @@ data "aws_iam_policy_document" "github_actions_assume_role" { condition { test = "StringLike" variable = "token.actions.githubusercontent.com:sub" - values = ["repo:${var.github_org}/${var.github_repo}:*"] + values = ["repo:${var.github_org}/${var.github_repo}:*"] } condition { test = "StringEquals" variable = "token.actions.githubusercontent.com:aud" - values = ["sts.amazonaws.com"] + values = ["sts.amazonaws.com"] } } } @@ -514,8 +530,8 @@ resource "aws_iam_policy" "firehose_readonly" { "firehose:StopDeliveryStreamEncryption" ] Resource = [ - "arn:aws:firehose:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:deliverystream/eligibility-signposting-api*", - "arn:aws:firehose:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:deliverystream/splunk-alarm-events*" + "arn:aws:firehose:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:deliverystream/eligibility-signposting-api*", + "arn:aws:firehose:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:deliverystream/splunk-alarm-events*" ] } ] diff --git a/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf b/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf index 10f3669c0..3325d315f 100644 --- a/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf +++ b/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf @@ -36,6 +36,10 @@ data "aws_iam_policy_document" "permissions_boundary" { "dynamodb:CreateTable", "dynamodb:TagResource", "dynamodb:ListTagsOfResource", + # Only used in preprod + "dynamodb:GetItem", + "dynamodb:PutItem", + "dynamodb:DeleteItem", # EC2 - networking infrastructure "ec2:Describe*", diff --git a/scripts/seed_users/__init__.py b/scripts/seed_users/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/seed_users/seed_dynamodb.py b/scripts/seed_users/seed_dynamodb.py new file mode 100644 index 000000000..744016080 --- /dev/null +++ b/scripts/seed_users/seed_dynamodb.py @@ -0,0 +1,56 @@ +import argparse +import glob +import json +import os + +import boto3 + + +def parse_args(): + parser = argparse.ArgumentParser(description="Seed DynamoDB table with JSON data.") + parser.add_argument("--table-name", required=True, help="Name of the DynamoDB table") + parser.add_argument("--region", default="eu-west-2", help="AWS region") + parser.add_argument("--data-folder", default="vitaIntegrationTestData/", help="Folder containing JSON seed data") + return parser.parse_args() + + +def clear_table(table): + scan = table.scan( + ProjectionExpression="#nhs, #type", ExpressionAttributeNames={"#nhs": "NHS_NUMBER", "#type": "ATTRIBUTE_TYPE"} + ) + with table.batch_writer() as batch: + for item in scan["Items"]: + batch.delete_item(Key={"NHS_NUMBER": item["NHS_NUMBER"], "ATTRIBUTE_TYPE": item["ATTRIBUTE_TYPE"]}) + + +def insert_data_from_folder(table, data_folder): + json_files = glob.glob(os.path.join(data_folder, "*.json")) + for file_path in json_files: + with open(file_path) as f: + try: + payload = json.load(f) + items = payload.get("data", []) + except Exception as e: + print(f"Skipping {file_path}: {e}") + continue + + with table.batch_writer() as batch: + for item in items: + nhs_number = item.get("NHS_NUMBER") + attr_type = item.get("ATTRIBUTE_TYPE") + if nhs_number and attr_type: + item["id"] = nhs_number + batch.put_item(Item=item) + + +def main(): + args = parse_args() + dynamodb = boto3.resource("dynamodb", region_name=args.region) + table = dynamodb.Table(args.table_name) + + clear_table(table) + insert_data_from_folder(table, args.data_folder) + + +if __name__ == "__main__": + main() diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_001.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_001.json new file mode 100644 index 000000000..c744f8e5b --- /dev/null +++ b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_001.json @@ -0,0 +1,40 @@ +{ + "scenario_name": "RSV - Vita Integration - Actionable - You should have the RSV vaccine ( in CP area ) - age rolling - BookNBS", + "request_headers": { + "nhs-login-nhs-number": "9686368973" + }, + "config_filenames": [ + "vita_integration_test_config.json" + ], + "data": [ + { + "NHS_NUMBER": "9686368973", + "ATTRIBUTE_TYPE": "COHORTS", + "COHORT_MEMBERSHIPS": [ + { + "COHORT_LABEL": "rsv_75to79", + "DATE_JOINED": "20231020" + } + ] + }, + { + "NHS_NUMBER": "9686368973", + "ATTRIBUTE_TYPE": "PERSON", + "DATE_OF_BIRTH": "<>", + "GENDER": "0", + "POSTCODE": "SG8 6EG", + "POSTCODE_SECTOR": "SG86", + "POSTCODE_OUTCODE": "SG8", + "MSOA": "E02003792", + "LSOA": "E01018267", + "LOCAL_AUTHORITY": "E08000011", + "GP_PRACTICE_CODE": "D81046", + "PCN": "U75549", + "ICB": "QUE", + "COMMISSIONING_REGION": "Y61", + "13Q_FLAG": "N", + "CARE_HOME_FLAG": "N", + "DE_FLAG": "N" + } + ] +} diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_002.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_002.json new file mode 100644 index 000000000..35bcb3efc --- /dev/null +++ b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_002.json @@ -0,0 +1,39 @@ +{ + "scenario_name": "RSV - Vita Integration - Actionable - You should have the RSV vaccine ( out CP area ) - age rolling - age rolling - BookLocal", + "request_headers": { + "nhs-login-nhs-number": "9686368906" + }, + "config_filenames": [ + "vita_integration_test_config.json" + ], + "data": [ + { + "NHS_NUMBER": "9686368906", + "ATTRIBUTE_TYPE": "COHORTS", + "COHORT_MEMBERSHIPS": [ + { + "COHORT_LABEL": "rsv_75to79", + "DATE_JOINED": "20231020" + } + ] + }, + { + "NHS_NUMBER": "9686368906", + "ATTRIBUTE_TYPE": "PERSON", + "DATE_OF_BIRTH": "19500601", + "GENDER": "2", + "POSTCODE": "CB3 8DX", + "POSTCODE_SECTOR": "CB38", + "POSTCODE_OUTCODE": "CB3", + "MSOA": "E02007085", + "LSOA": "E01018223", + "GP_PRACTICE_CODE": "D81046", + "PCN": "U75549", + "ICB": "QUE", + "COMMISSIONING_REGION": "Y61", + "13Q_FLAG": "N", + "CARE_HOME_FLAG": "N", + "DE_FLAG": "N" + } + ] +} diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_003.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_003.json new file mode 100644 index 000000000..0e8cdd257 --- /dev/null +++ b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_003.json @@ -0,0 +1,39 @@ +{ + "scenario_name": "RSV - Vita Integration - Actionable - You should have the RSV vaccine ( out CP area ) - age catchup - BookLocal", + "request_headers": { + "nhs-login-nhs-number": "9658218873" + }, + "config_filenames": [ + "vita_integration_test_config.json" + ], + "data": [ + { + "NHS_NUMBER": "9658218873", + "ATTRIBUTE_TYPE": "COHORTS", + "COHORT_MEMBERSHIPS": [ + { + "COHORT_LABEL": "rsv_80_since_02_Sept_2024", + "DATE_JOINED": "20231020" + } + ] + }, + { + "NHS_NUMBER": "9658218873", + "ATTRIBUTE_TYPE": "PERSON", + "DATE_OF_BIRTH": "19500601", + "GENDER": "2", + "POSTCODE": "CB3 8DX", + "POSTCODE_SECTOR": "CB38", + "POSTCODE_OUTCODE": "CB3", + "MSOA": "E02007085", + "LSOA": "E01018223", + "GP_PRACTICE_CODE": "D81046", + "PCN": "U75549", + "ICB": "QUE", + "COMMISSIONING_REGION": "Y61", + "13Q_FLAG": "N", + "CARE_HOME_FLAG": "N", + "DE_FLAG": "N" + } + ] +} diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_004.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_004.json new file mode 100644 index 000000000..ed20d7d35 --- /dev/null +++ b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_004.json @@ -0,0 +1,35 @@ +{ + "scenario_name": "RSV - Vita Integration - Actionable - You should have the RSV vaccine ( existing NBS booking ) - empty cohorts - AmendNBS", + "request_headers": { + "nhs-login-nhs-number": "9658218881" + }, + "config_filenames": [ + "vita_integration_test_config.json" + ], + "data": [ + { + "NHS_NUMBER": "9658218881", + "ATTRIBUTE_TYPE": "PERSON", + "DATE_OF_BIRTH": "19500601", + "GENDER": "2", + "POSTCODE": "CB3 8DX", + "POSTCODE_SECTOR": "CB38", + "POSTCODE_OUTCODE": "CB3", + "MSOA": "E02007085", + "LSOA": "E01018223", + "GP_PRACTICE_CODE": "D81046", + "PCN": "U75549", + "ICB": "QUE", + "COMMISSIONING_REGION": "Y61", + "13Q_FLAG": "N", + "CARE_HOME_FLAG": "N", + "DE_FLAG": "N" + }, + { + "NHS_NUMBER": "9658218881", + "ATTRIBUTE_TYPE": "RSV", + "BOOKED_APPOINTMENT_DATE": "<>", + "BOOKED_APPOINTMENT_PROVIDER": "NBS" + } + ] +} diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_005.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_005.json new file mode 100644 index 000000000..2ef47a620 --- /dev/null +++ b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_005.json @@ -0,0 +1,35 @@ +{ + "scenario_name": "RSV - Vita Integration - Actionable - You should have the RSV vaccine ( existing non-NBS booking ) - empty cohorts - ManageLocal", + "request_headers": { + "nhs-login-nhs-number": "9658218903" + }, + "config_filenames": [ + "vita_integration_test_config.json" + ], + "data": [ + { + "NHS_NUMBER": "9658218903", + "ATTRIBUTE_TYPE": "PERSON", + "DATE_OF_BIRTH": "19500601", + "GENDER": "2", + "POSTCODE": "CB3 8DX", + "POSTCODE_SECTOR": "CB38", + "POSTCODE_OUTCODE": "CB3", + "MSOA": "E02007085", + "LSOA": "E01018223", + "GP_PRACTICE_CODE": "D81046", + "PCN": "U75549", + "ICB": "QUE", + "COMMISSIONING_REGION": "Y61", + "13Q_FLAG": "N", + "CARE_HOME_FLAG": "N", + "DE_FLAG": "N" + }, + { + "NHS_NUMBER": "9658218903", + "ATTRIBUTE_TYPE": "RSV", + "BOOKED_APPOINTMENT_DATE": "<>", + "BOOKED_APPOINTMENT_PROVIDER": "ACC" + } + ] +} diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_006.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_006.json new file mode 100644 index 000000000..9cda11395 --- /dev/null +++ b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_006.json @@ -0,0 +1,34 @@ +{ + "scenario_name": "RSV - Vita Integration - NotActionable - You should have the RSV vaccine (already vaccd) - AlreadyVaccinated", + "request_headers": { + "nhs-login-nhs-number": "9658218989" + }, + "config_filenames": [ + "vita_integration_test_config.json" + ], + "data": [ + { + "NHS_NUMBER": "9658218989", + "ATTRIBUTE_TYPE": "PERSON", + "DATE_OF_BIRTH": "19500601", + "GENDER": "2", + "POSTCODE": "CB3 8DX", + "POSTCODE_SECTOR": "CB38", + "POSTCODE_OUTCODE": "CB3", + "MSOA": "E02007085", + "LSOA": "E01018223", + "GP_PRACTICE_CODE": "D81046", + "PCN": "U75549", + "ICB": "QUE", + "COMMISSIONING_REGION": "Y61", + "13Q_FLAG": "N", + "CARE_HOME_FLAG": "N", + "DE_FLAG": "N" + }, + { + "NHS_NUMBER": "9658218989", + "ATTRIBUTE_TYPE": "RSV", + "LAST_SUCCESSFUL_DATE": "<>" + } + ] +} diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_007.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_007.json new file mode 100644 index 000000000..e7b4d9a30 --- /dev/null +++ b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_007.json @@ -0,0 +1,39 @@ +{ + "scenario_name": "RSV - Vita Integration - NotActionable - You should have the RSV vaccine ( not available ) - age rolling - NotAvailable", + "request_headers": { + "nhs-login-nhs-number": "9658218997" + }, + "config_filenames": [ + "vita_integration_test_config.json" + ], + "data": [ + { + "NHS_NUMBER": "9658218997", + "ATTRIBUTE_TYPE": "COHORTS", + "COHORT_MEMBERSHIPS": [ + { + "COHORT_LABEL": "rsv_75to79", + "DATE_JOINED": "20231020" + } + ] + }, + { + "NHS_NUMBER": "9658218997", + "ATTRIBUTE_TYPE": "PERSON", + "DATE_OF_BIRTH": "19500601", + "GENDER": "2", + "POSTCODE": "CB3 8DX", + "POSTCODE_SECTOR": "CB38", + "POSTCODE_OUTCODE": "CB3", + "MSOA": "E02007085", + "LSOA": "E01018223", + "GP_PRACTICE_CODE": "D81046", + "PCN": "U75549", + "ICB": "QUE", + "COMMISSIONING_REGION": "Y61", + "13Q_FLAG": "N", + "CARE_HOME_FLAG": "N", + "DE_FLAG": "N" + } + ] +} diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_009.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_009.json new file mode 100644 index 000000000..f0c0971a2 --- /dev/null +++ b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_009.json @@ -0,0 +1,39 @@ +{ + "scenario_name": "RSV - Vita Integration - NotActionable - You should have the RSV vaccine - age rolling - NotYetDue", + "request_headers": { + "nhs-login-nhs-number": "9658219012" + }, + "config_filenames": [ + "vita_integration_test_config.json" + ], + "data": [ + { + "NHS_NUMBER": "9658219012", + "ATTRIBUTE_TYPE": "COHORTS", + "COHORT_MEMBERSHIPS": [ + { + "COHORT_LABEL": "rsv_75to79", + "DATE_JOINED": "20231020" + } + ] + }, + { + "NHS_NUMBER": "9658219012", + "ATTRIBUTE_TYPE": "PERSON", + "DATE_OF_BIRTH": "19500601", + "GENDER": "2", + "POSTCODE": "CB3 8DX", + "POSTCODE_SECTOR": "CB38", + "POSTCODE_OUTCODE": "CB3", + "MSOA": "E02007085", + "LSOA": "E01018223", + "GP_PRACTICE_CODE": "D81046", + "PCN": "U75549", + "ICB": "QUE", + "COMMISSIONING_REGION": "Y61", + "13Q_FLAG": "N", + "CARE_HOME_FLAG": "N", + "DE_FLAG": "N" + } + ] +} diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_011.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_011.json new file mode 100644 index 000000000..6ebd3beb1 --- /dev/null +++ b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_011.json @@ -0,0 +1,39 @@ +{ + "scenario_name": "RSV - Vita Integration - NotActionable - You should have the RSV vaccine - age rolling - TooClose", + "request_headers": { + "nhs-login-nhs-number": "9658220142" + }, + "config_filenames": [ + "vita_integration_test_config.json" + ], + "data": [ + { + "NHS_NUMBER": "9658220142", + "ATTRIBUTE_TYPE": "COHORTS", + "COHORT_MEMBERSHIPS": [ + { + "COHORT_LABEL": "rsv_75to79", + "DATE_JOINED": "20231020" + } + ] + }, + { + "NHS_NUMBER": "9658220142", + "ATTRIBUTE_TYPE": "PERSON", + "DATE_OF_BIRTH": "19500601", + "GENDER": "2", + "POSTCODE": "CB3 8DX", + "POSTCODE_SECTOR": "CB38", + "POSTCODE_OUTCODE": "CB3", + "MSOA": "E02007085", + "LSOA": "E01018223", + "GP_PRACTICE_CODE": "D81046", + "PCN": "U75549", + "ICB": "QUE", + "COMMISSIONING_REGION": "Y61", + "13Q_FLAG": "N", + "CARE_HOME_FLAG": "N", + "DE_FLAG": "N" + } + ] +} diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_012.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_012.json new file mode 100644 index 000000000..a91993317 --- /dev/null +++ b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_012.json @@ -0,0 +1,39 @@ +{ + "scenario_name": "RSV - Vita Integration - NotActionable - You should have the RSV vaccine ( managed setting ) - age rolling - OtherSetting", + "request_headers": { + "nhs-login-nhs-number": "9658220150" + }, + "config_filenames": [ + "vita_integration_test_config.json" + ], + "data": [ + { + "NHS_NUMBER": "9658220150", + "ATTRIBUTE_TYPE": "PERSON", + "DATE_OF_BIRTH": "19500601", + "GENDER": "2", + "POSTCODE": "CB3 8DX", + "POSTCODE_SECTOR": "CB38", + "POSTCODE_OUTCODE": "CB3", + "MSOA": "E02007085", + "LSOA": "E01018223", + "GP_PRACTICE_CODE": "D81046", + "PCN": "U75549", + "ICB": "QUE", + "COMMISSIONING_REGION": "Y61", + "13Q_FLAG": "N", + "CARE_HOME_FLAG": "Y", + "DE_FLAG": "N" + }, + { + "NHS_NUMBER": "9658220150", + "ATTRIBUTE_TYPE": "COHORTS", + "COHORT_MEMBERSHIPS": [ + { + "COHORT_LABEL": "rsv_75to79", + "DATE_JOINED": "20231020" + } + ] + } + ] +} diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_013.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_013.json new file mode 100644 index 000000000..77057e50b --- /dev/null +++ b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_013.json @@ -0,0 +1,34 @@ +{ + "scenario_name": "RSV - Vita Integration - NotActionable - You should have the RSV vaccine (already vaccinated) - unknown membership - AlreadyVaccinated", + "request_headers": { + "nhs-login-nhs-number": "9450114080" + }, + "config_filenames": [ + "vita_integration_test_config.json" + ], + "data": [ + { + "NHS_NUMBER": "9450114080", + "ATTRIBUTE_TYPE": "PERSON", + "DATE_OF_BIRTH": "19500601", + "GENDER": "2", + "POSTCODE": "CB3 8DX", + "POSTCODE_SECTOR": "CB38", + "POSTCODE_OUTCODE": "CB3", + "MSOA": "E02007085", + "LSOA": "E01018223", + "GP_PRACTICE_CODE": "D81046", + "PCN": "U75549", + "ICB": "QUE", + "COMMISSIONING_REGION": "Y61", + "13Q_FLAG": "N", + "CARE_HOME_FLAG": "Y", + "DE_FLAG": "N" + }, + { + "NHS_NUMBER": "9450114080", + "ATTRIBUTE_TYPE": "RSV", + "LAST_SUCCESSFUL_DATE": "<>" + } + ] +} diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_014.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_014.json new file mode 100644 index 000000000..7ec5f92d9 --- /dev/null +++ b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_014.json @@ -0,0 +1,34 @@ +{ + "scenario_name": "RSV - Vita Integration - NotActionable - You should have the RSV vaccine (already vaccinated) - empty cohorts - AlreadyVaccinated", + "request_headers": { + "nhs-login-nhs-number": "9466447939" + }, + "config_filenames": [ + "vita_integration_test_config.json" + ], + "data": [ + { + "NHS_NUMBER": "9466447939", + "ATTRIBUTE_TYPE": "RSV", + "LAST_SUCCESSFUL_DATE": "<>" + }, + { + "NHS_NUMBER": "9466447939", + "ATTRIBUTE_TYPE": "PERSON", + "DATE_OF_BIRTH": "19500601", + "GENDER": "2", + "POSTCODE": "CB3 8DX", + "POSTCODE_SECTOR": "CB38", + "POSTCODE_OUTCODE": "CB3", + "MSOA": "E02007085", + "LSOA": "E01018223", + "GP_PRACTICE_CODE": "D81046", + "PCN": "U75549", + "ICB": "QUE", + "COMMISSIONING_REGION": "Y61", + "13Q_FLAG": "N", + "CARE_HOME_FLAG": "Y", + "DE_FLAG": "N" + } + ] +} diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_015.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_015.json new file mode 100644 index 000000000..2f663767e --- /dev/null +++ b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_015.json @@ -0,0 +1,35 @@ +{ + "scenario_name": "RSV - Vita Integration - - Not Eligible", + "request_headers": { + "nhs-login-nhs-number": "9657933617" + }, + "config_filenames": [ + "vita_integration_test_config.json" + ], + "data": [ + { + "NHS_NUMBER": "9657933617", + "ATTRIBUTE_TYPE": "PERSON", + "DATE_OF_BIRTH": "19500601", + "GENDER": "2", + "POSTCODE": "CB3 8DX", + "POSTCODE_SECTOR": "CB38", + "POSTCODE_OUTCODE": "CB3", + "MSOA": "E02007085", + "LSOA": "E01018223", + "GP_PRACTICE_CODE": "D81046", + "PCN": "U75549", + "ICB": "QUE", + "COMMISSIONING_REGION": "Y61", + "13Q_FLAG": "N", + "CARE_HOME_FLAG": "Y", + "DE_FLAG": "N" + }, + { + "NHS_NUMBER": "9657933617", + "ATTRIBUTE_TYPE": "RSV", + "LAST_SUCCESSFUL_DATE": null, + "BOOKED_APPOINTMENT_DATE": null + } + ] +} diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_500.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_500.json new file mode 100644 index 000000000..79aff5d5e --- /dev/null +++ b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_500.json @@ -0,0 +1,40 @@ +{ + "scenario_name": "RSV - Vita Integration - 500", + "request_headers": { + "nhs-login-nhs-number": "9436793375" + }, + "expected_response_code": 500, + "config_filenames": [ + "vita_integration_test_config.json" + ], + "data": [ + { + "NHS_NUMBER": "9436793375", + "ATTRIBUTE_TYPE": "COHORTS", + "COHORT_MEMBERSHIPS": [ + { + "COHORT_LABEL": "rsv_75to79", + "DATE_JOINED": "20231020" + } + ] + }, + { + "NHS_NUMBER": "9436793375", + "ATTRIBUTE_TYPE": "PERSON", + "DATE_OF_BIRTH": 19500601, + "GENDER": "0", + "POSTCODE": "SG8 6EG", + "POSTCODE_SECTOR": "SG86", + "POSTCODE_OUTCODE": "SG8", + "MSOA": "E02003792", + "LSOA": "E01018267", + "GP_PRACTICE_CODE": "D81046", + "PCN": "U75549", + "ICB": "QUE", + "COMMISSIONING_REGION": "Y61", + "13Q_FLAG": "N", + "CARE_HOME_FLAG": "N", + "DE_FLAG": "N" + } + ] +} From a6884cd07a42bf52bb28d3701e507293476eafa6 Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Fri, 29 Aug 2025 13:52:04 +0100 Subject: [PATCH 02/14] added permissons and given token id --- .../workflows/cicd-4b-preprod-seed-users.yml | 12 +++++-- .../github_actions_policies.tf | 6 ++-- .../iams_permissions_boundary.tf | 2 ++ scripts/seed_users/seed_dynamodb.py | 35 ++++++++++++------- 4 files changed, 38 insertions(+), 17 deletions(-) diff --git a/.github/workflows/cicd-4b-preprod-seed-users.yml b/.github/workflows/cicd-4b-preprod-seed-users.yml index cc874b259..e3f5cb09d 100644 --- a/.github/workflows/cicd-4b-preprod-seed-users.yml +++ b/.github/workflows/cicd-4b-preprod-seed-users.yml @@ -4,9 +4,17 @@ on: push: branches: - ELI-417-preprod-db-seeding + workflow_dispatch: + inputs: + environment: + description: Target environment + required: true + type: choice + options: + - dev jobs: - seed: + seed-dynamodb: runs-on: ubuntu-latest environment: "dev" permissions: @@ -37,7 +45,7 @@ jobs: - name: Run seed script run: | - python .github/scripts/seed_dynamodb.py \ + python scripts/seed_users/seed_dynamodb.py \ --table-name "${{ env.DYNAMODB_TABLE }}" \ --region "${{ env.AWS_REGION }}" \ --data-folder "${{ env.DATA_FOLDER }}" diff --git a/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf b/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf index 0514c1dd6..0647a45c6 100644 --- a/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf +++ b/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf @@ -109,16 +109,18 @@ resource "aws_iam_policy" "dynamodb_management" { } ], # to create test users in preprod - var.environment == "preprod" ? [ + var.environment == "dev" ? [ { Effect = "Allow", Action = [ "dynamodb:GetItem", "dynamodb:PutItem", "dynamodb:DeleteItem", + "dynamodb:Scan", + "dynamodb:BatchWriteItem" ], Resource = [ - "arn:aws:dynamodb:*:${data.aws_caller_identity.current.account_id}:table/*eligibility-signposting-api-preprod-eligibility_datastore" + "arn:aws:dynamodb:*:${data.aws_caller_identity.current.account_id}:table/*eligibility-signposting-api-${var.environment}-eligibility_datastore" ] } ] : [] diff --git a/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf b/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf index 3325d315f..1971631f4 100644 --- a/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf +++ b/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf @@ -40,6 +40,8 @@ data "aws_iam_policy_document" "permissions_boundary" { "dynamodb:GetItem", "dynamodb:PutItem", "dynamodb:DeleteItem", + "dynamodb:Scan", + "dynamodb:BatchWriteItem", # EC2 - networking infrastructure "ec2:Describe*", diff --git a/scripts/seed_users/seed_dynamodb.py b/scripts/seed_users/seed_dynamodb.py index 744016080..4a51681d5 100644 --- a/scripts/seed_users/seed_dynamodb.py +++ b/scripts/seed_users/seed_dynamodb.py @@ -14,25 +14,33 @@ def parse_args(): return parser.parse_args() -def clear_table(table): - scan = table.scan( - ProjectionExpression="#nhs, #type", ExpressionAttributeNames={"#nhs": "NHS_NUMBER", "#type": "ATTRIBUTE_TYPE"} - ) +def get_keys_from_folder(data_folder): + keys_to_delete = [] + json_files = glob.glob(os.path.join(data_folder, "*.json")) + for file_path in json_files: + with open(file_path) as f: + payload = json.load(f) + items = payload.get("data", []) + for item in items: + nhs_number = item.get("NHS_NUMBER") + attr_type = item.get("ATTRIBUTE_TYPE") + if nhs_number and attr_type: + keys_to_delete.append({"NHS_NUMBER": nhs_number, "ATTRIBUTE_TYPE": attr_type}) + return keys_to_delete + + +def delete_specific_items(table, keys): with table.batch_writer() as batch: - for item in scan["Items"]: - batch.delete_item(Key={"NHS_NUMBER": item["NHS_NUMBER"], "ATTRIBUTE_TYPE": item["ATTRIBUTE_TYPE"]}) + for key in keys: + batch.delete_item(Key=key) def insert_data_from_folder(table, data_folder): json_files = glob.glob(os.path.join(data_folder, "*.json")) for file_path in json_files: with open(file_path) as f: - try: - payload = json.load(f) - items = payload.get("data", []) - except Exception as e: - print(f"Skipping {file_path}: {e}") - continue + payload = json.load(f) + items = payload.get("data", []) with table.batch_writer() as batch: for item in items: @@ -48,7 +56,8 @@ def main(): dynamodb = boto3.resource("dynamodb", region_name=args.region) table = dynamodb.Table(args.table_name) - clear_table(table) + keys_to_delete = get_keys_from_folder(args.data_folder) + delete_specific_items(table, keys_to_delete) insert_data_from_folder(table, args.data_folder) From 5ca518681dc94f5aec3d24ec2780972aff92473a Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Fri, 29 Aug 2025 16:20:08 +0100 Subject: [PATCH 03/14] grouping --- .github/workflows/cicd-4b-preprod-seed-users.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/cicd-4b-preprod-seed-users.yml b/.github/workflows/cicd-4b-preprod-seed-users.yml index e3f5cb09d..43e589918 100644 --- a/.github/workflows/cicd-4b-preprod-seed-users.yml +++ b/.github/workflows/cicd-4b-preprod-seed-users.yml @@ -1,5 +1,9 @@ name: Preprod - Seed DynamoDB table +concurrency: + group: seed-preprod-dynamodb + cancel-in-progress: false + on: push: branches: From 1449d2b514a7e41ed0ea623c92afd32ca59ee64f Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Mon, 1 Sep 2025 09:55:35 +0100 Subject: [PATCH 04/14] dev -> preprod after testing --- .../workflows/cicd-4b-preprod-seed-users.yml | 13 +++++++------ .../github_actions_policies.tf | 2 +- .../iams_permissions_boundary.tf | 17 ++++++++++++++++- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/.github/workflows/cicd-4b-preprod-seed-users.yml b/.github/workflows/cicd-4b-preprod-seed-users.yml index 43e589918..b38af6a2b 100644 --- a/.github/workflows/cicd-4b-preprod-seed-users.yml +++ b/.github/workflows/cicd-4b-preprod-seed-users.yml @@ -5,9 +5,10 @@ concurrency: cancel-in-progress: false on: - push: - branches: - - ELI-417-preprod-db-seeding + workflow_run: + workflows: [ "Preprod Deploy" ] + types: + - completed workflow_dispatch: inputs: environment: @@ -15,19 +16,19 @@ on: required: true type: choice options: - - dev + - preprod jobs: seed-dynamodb: runs-on: ubuntu-latest - environment: "dev" + environment: "preprod" permissions: id-token: write contents: read env: AWS_REGION: eu-west-2 DATA_FOLDER: vitaIntegrationTestData/ - DYNAMODB_TABLE: eligibility-signposting-api-dev-eligibility_datastore + DYNAMODB_TABLE: eligibility-signposting-api-preprod-eligibility_datastore steps: - name: Checkout repo diff --git a/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf b/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf index 0647a45c6..ee671e8fa 100644 --- a/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf +++ b/infrastructure/stacks/iams-developer-roles/github_actions_policies.tf @@ -109,7 +109,7 @@ resource "aws_iam_policy" "dynamodb_management" { } ], # to create test users in preprod - var.environment == "dev" ? [ + var.environment == "preprod" ? [ { Effect = "Allow", Action = [ diff --git a/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf b/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf index 1971631f4..dcf791bfc 100644 --- a/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf +++ b/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf @@ -239,7 +239,22 @@ data "aws_iam_policy_document" "permissions_boundary" { values = [var.default_aws_region] } } - + # Environment-specific actions + dynamic "statement" { + for_each = var.environment == "preprod" ? [1] : [] + content { + sid = "AllowPreprodDynamoDBItemOps" + effect = "Allow" + actions = [ + "dynamodb:GetItem", + "dynamodb:PutItem", + "dynamodb:DeleteItem", + "dynamodb:Scan", + "dynamodb:BatchWriteItem" + ] + resources = ["*"] + } + } # Allow access to IAM actions for us-east-1 region only statement { sid = "AllowIamActionsInUsEast1" From 013714da1f848c180e648bc9cf28897340577959 Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Mon, 1 Sep 2025 11:07:33 +0100 Subject: [PATCH 05/14] added folder validation --- scripts/seed_users/seed_dynamodb.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/seed_users/seed_dynamodb.py b/scripts/seed_users/seed_dynamodb.py index 4a51681d5..173a0defd 100644 --- a/scripts/seed_users/seed_dynamodb.py +++ b/scripts/seed_users/seed_dynamodb.py @@ -56,6 +56,9 @@ def main(): dynamodb = boto3.resource("dynamodb", region_name=args.region) table = dynamodb.Table(args.table_name) + if not os.path.isdir(args.data_folder): + raise ValueError(f"Data folder '{args.data_folder}' does not exist or is not a directory.") + keys_to_delete = get_keys_from_folder(args.data_folder) delete_specific_items(table, keys_to_delete) insert_data_from_folder(table, args.data_folder) From d0c980bdf03de1e47b427d53766033f46a3ce9e2 Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Mon, 1 Sep 2025 11:15:01 +0100 Subject: [PATCH 06/14] referring to the users in e2e tests folder --- scripts/seed_users/seed_dynamodb.py | 12 ++++-- .../AUTO_RSV_VITA_INT_001.json | 40 ------------------- .../AUTO_RSV_VITA_INT_002.json | 39 ------------------ .../AUTO_RSV_VITA_INT_003.json | 39 ------------------ .../AUTO_RSV_VITA_INT_004.json | 35 ---------------- .../AUTO_RSV_VITA_INT_005.json | 35 ---------------- .../AUTO_RSV_VITA_INT_006.json | 34 ---------------- .../AUTO_RSV_VITA_INT_007.json | 39 ------------------ .../AUTO_RSV_VITA_INT_009.json | 39 ------------------ .../AUTO_RSV_VITA_INT_011.json | 39 ------------------ .../AUTO_RSV_VITA_INT_012.json | 39 ------------------ .../AUTO_RSV_VITA_INT_013.json | 34 ---------------- .../AUTO_RSV_VITA_INT_014.json | 34 ---------------- .../AUTO_RSV_VITA_INT_015.json | 35 ---------------- .../AUTO_RSV_VITA_INT_500.json | 40 ------------------- 15 files changed, 8 insertions(+), 525 deletions(-) delete mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_001.json delete mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_002.json delete mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_003.json delete mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_004.json delete mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_005.json delete mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_006.json delete mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_007.json delete mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_009.json delete mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_011.json delete mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_012.json delete mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_013.json delete mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_014.json delete mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_015.json delete mode 100644 scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_500.json diff --git a/scripts/seed_users/seed_dynamodb.py b/scripts/seed_users/seed_dynamodb.py index 173a0defd..161bfe7f8 100644 --- a/scripts/seed_users/seed_dynamodb.py +++ b/scripts/seed_users/seed_dynamodb.py @@ -13,6 +13,8 @@ def parse_args(): parser.add_argument("--data-folder", default="vitaIntegrationTestData/", help="Folder containing JSON seed data") return parser.parse_args() +def resolve_data_folder(path): + return os.path.abspath(path) def get_keys_from_folder(data_folder): keys_to_delete = [] @@ -53,15 +55,17 @@ def insert_data_from_folder(table, data_folder): def main(): args = parse_args() + dynamodb = boto3.resource("dynamodb", region_name=args.region) table = dynamodb.Table(args.table_name) - if not os.path.isdir(args.data_folder): - raise ValueError(f"Data folder '{args.data_folder}' does not exist or is not a directory.") + data_folder = resolve_data_folder(args.data_folder) + if not os.path.isdir(data_folder): + raise ValueError(f"Data folder '{data_folder}' does not exist or is not a directory.") - keys_to_delete = get_keys_from_folder(args.data_folder) + keys_to_delete = get_keys_from_folder(data_folder) delete_specific_items(table, keys_to_delete) - insert_data_from_folder(table, args.data_folder) + insert_data_from_folder(table, data_folder) if __name__ == "__main__": diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_001.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_001.json deleted file mode 100644 index c744f8e5b..000000000 --- a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_001.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "scenario_name": "RSV - Vita Integration - Actionable - You should have the RSV vaccine ( in CP area ) - age rolling - BookNBS", - "request_headers": { - "nhs-login-nhs-number": "9686368973" - }, - "config_filenames": [ - "vita_integration_test_config.json" - ], - "data": [ - { - "NHS_NUMBER": "9686368973", - "ATTRIBUTE_TYPE": "COHORTS", - "COHORT_MEMBERSHIPS": [ - { - "COHORT_LABEL": "rsv_75to79", - "DATE_JOINED": "20231020" - } - ] - }, - { - "NHS_NUMBER": "9686368973", - "ATTRIBUTE_TYPE": "PERSON", - "DATE_OF_BIRTH": "<>", - "GENDER": "0", - "POSTCODE": "SG8 6EG", - "POSTCODE_SECTOR": "SG86", - "POSTCODE_OUTCODE": "SG8", - "MSOA": "E02003792", - "LSOA": "E01018267", - "LOCAL_AUTHORITY": "E08000011", - "GP_PRACTICE_CODE": "D81046", - "PCN": "U75549", - "ICB": "QUE", - "COMMISSIONING_REGION": "Y61", - "13Q_FLAG": "N", - "CARE_HOME_FLAG": "N", - "DE_FLAG": "N" - } - ] -} diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_002.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_002.json deleted file mode 100644 index 35bcb3efc..000000000 --- a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_002.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "scenario_name": "RSV - Vita Integration - Actionable - You should have the RSV vaccine ( out CP area ) - age rolling - age rolling - BookLocal", - "request_headers": { - "nhs-login-nhs-number": "9686368906" - }, - "config_filenames": [ - "vita_integration_test_config.json" - ], - "data": [ - { - "NHS_NUMBER": "9686368906", - "ATTRIBUTE_TYPE": "COHORTS", - "COHORT_MEMBERSHIPS": [ - { - "COHORT_LABEL": "rsv_75to79", - "DATE_JOINED": "20231020" - } - ] - }, - { - "NHS_NUMBER": "9686368906", - "ATTRIBUTE_TYPE": "PERSON", - "DATE_OF_BIRTH": "19500601", - "GENDER": "2", - "POSTCODE": "CB3 8DX", - "POSTCODE_SECTOR": "CB38", - "POSTCODE_OUTCODE": "CB3", - "MSOA": "E02007085", - "LSOA": "E01018223", - "GP_PRACTICE_CODE": "D81046", - "PCN": "U75549", - "ICB": "QUE", - "COMMISSIONING_REGION": "Y61", - "13Q_FLAG": "N", - "CARE_HOME_FLAG": "N", - "DE_FLAG": "N" - } - ] -} diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_003.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_003.json deleted file mode 100644 index 0e8cdd257..000000000 --- a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_003.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "scenario_name": "RSV - Vita Integration - Actionable - You should have the RSV vaccine ( out CP area ) - age catchup - BookLocal", - "request_headers": { - "nhs-login-nhs-number": "9658218873" - }, - "config_filenames": [ - "vita_integration_test_config.json" - ], - "data": [ - { - "NHS_NUMBER": "9658218873", - "ATTRIBUTE_TYPE": "COHORTS", - "COHORT_MEMBERSHIPS": [ - { - "COHORT_LABEL": "rsv_80_since_02_Sept_2024", - "DATE_JOINED": "20231020" - } - ] - }, - { - "NHS_NUMBER": "9658218873", - "ATTRIBUTE_TYPE": "PERSON", - "DATE_OF_BIRTH": "19500601", - "GENDER": "2", - "POSTCODE": "CB3 8DX", - "POSTCODE_SECTOR": "CB38", - "POSTCODE_OUTCODE": "CB3", - "MSOA": "E02007085", - "LSOA": "E01018223", - "GP_PRACTICE_CODE": "D81046", - "PCN": "U75549", - "ICB": "QUE", - "COMMISSIONING_REGION": "Y61", - "13Q_FLAG": "N", - "CARE_HOME_FLAG": "N", - "DE_FLAG": "N" - } - ] -} diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_004.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_004.json deleted file mode 100644 index ed20d7d35..000000000 --- a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_004.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "scenario_name": "RSV - Vita Integration - Actionable - You should have the RSV vaccine ( existing NBS booking ) - empty cohorts - AmendNBS", - "request_headers": { - "nhs-login-nhs-number": "9658218881" - }, - "config_filenames": [ - "vita_integration_test_config.json" - ], - "data": [ - { - "NHS_NUMBER": "9658218881", - "ATTRIBUTE_TYPE": "PERSON", - "DATE_OF_BIRTH": "19500601", - "GENDER": "2", - "POSTCODE": "CB3 8DX", - "POSTCODE_SECTOR": "CB38", - "POSTCODE_OUTCODE": "CB3", - "MSOA": "E02007085", - "LSOA": "E01018223", - "GP_PRACTICE_CODE": "D81046", - "PCN": "U75549", - "ICB": "QUE", - "COMMISSIONING_REGION": "Y61", - "13Q_FLAG": "N", - "CARE_HOME_FLAG": "N", - "DE_FLAG": "N" - }, - { - "NHS_NUMBER": "9658218881", - "ATTRIBUTE_TYPE": "RSV", - "BOOKED_APPOINTMENT_DATE": "<>", - "BOOKED_APPOINTMENT_PROVIDER": "NBS" - } - ] -} diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_005.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_005.json deleted file mode 100644 index 2ef47a620..000000000 --- a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_005.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "scenario_name": "RSV - Vita Integration - Actionable - You should have the RSV vaccine ( existing non-NBS booking ) - empty cohorts - ManageLocal", - "request_headers": { - "nhs-login-nhs-number": "9658218903" - }, - "config_filenames": [ - "vita_integration_test_config.json" - ], - "data": [ - { - "NHS_NUMBER": "9658218903", - "ATTRIBUTE_TYPE": "PERSON", - "DATE_OF_BIRTH": "19500601", - "GENDER": "2", - "POSTCODE": "CB3 8DX", - "POSTCODE_SECTOR": "CB38", - "POSTCODE_OUTCODE": "CB3", - "MSOA": "E02007085", - "LSOA": "E01018223", - "GP_PRACTICE_CODE": "D81046", - "PCN": "U75549", - "ICB": "QUE", - "COMMISSIONING_REGION": "Y61", - "13Q_FLAG": "N", - "CARE_HOME_FLAG": "N", - "DE_FLAG": "N" - }, - { - "NHS_NUMBER": "9658218903", - "ATTRIBUTE_TYPE": "RSV", - "BOOKED_APPOINTMENT_DATE": "<>", - "BOOKED_APPOINTMENT_PROVIDER": "ACC" - } - ] -} diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_006.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_006.json deleted file mode 100644 index 9cda11395..000000000 --- a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_006.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "scenario_name": "RSV - Vita Integration - NotActionable - You should have the RSV vaccine (already vaccd) - AlreadyVaccinated", - "request_headers": { - "nhs-login-nhs-number": "9658218989" - }, - "config_filenames": [ - "vita_integration_test_config.json" - ], - "data": [ - { - "NHS_NUMBER": "9658218989", - "ATTRIBUTE_TYPE": "PERSON", - "DATE_OF_BIRTH": "19500601", - "GENDER": "2", - "POSTCODE": "CB3 8DX", - "POSTCODE_SECTOR": "CB38", - "POSTCODE_OUTCODE": "CB3", - "MSOA": "E02007085", - "LSOA": "E01018223", - "GP_PRACTICE_CODE": "D81046", - "PCN": "U75549", - "ICB": "QUE", - "COMMISSIONING_REGION": "Y61", - "13Q_FLAG": "N", - "CARE_HOME_FLAG": "N", - "DE_FLAG": "N" - }, - { - "NHS_NUMBER": "9658218989", - "ATTRIBUTE_TYPE": "RSV", - "LAST_SUCCESSFUL_DATE": "<>" - } - ] -} diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_007.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_007.json deleted file mode 100644 index e7b4d9a30..000000000 --- a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_007.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "scenario_name": "RSV - Vita Integration - NotActionable - You should have the RSV vaccine ( not available ) - age rolling - NotAvailable", - "request_headers": { - "nhs-login-nhs-number": "9658218997" - }, - "config_filenames": [ - "vita_integration_test_config.json" - ], - "data": [ - { - "NHS_NUMBER": "9658218997", - "ATTRIBUTE_TYPE": "COHORTS", - "COHORT_MEMBERSHIPS": [ - { - "COHORT_LABEL": "rsv_75to79", - "DATE_JOINED": "20231020" - } - ] - }, - { - "NHS_NUMBER": "9658218997", - "ATTRIBUTE_TYPE": "PERSON", - "DATE_OF_BIRTH": "19500601", - "GENDER": "2", - "POSTCODE": "CB3 8DX", - "POSTCODE_SECTOR": "CB38", - "POSTCODE_OUTCODE": "CB3", - "MSOA": "E02007085", - "LSOA": "E01018223", - "GP_PRACTICE_CODE": "D81046", - "PCN": "U75549", - "ICB": "QUE", - "COMMISSIONING_REGION": "Y61", - "13Q_FLAG": "N", - "CARE_HOME_FLAG": "N", - "DE_FLAG": "N" - } - ] -} diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_009.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_009.json deleted file mode 100644 index f0c0971a2..000000000 --- a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_009.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "scenario_name": "RSV - Vita Integration - NotActionable - You should have the RSV vaccine - age rolling - NotYetDue", - "request_headers": { - "nhs-login-nhs-number": "9658219012" - }, - "config_filenames": [ - "vita_integration_test_config.json" - ], - "data": [ - { - "NHS_NUMBER": "9658219012", - "ATTRIBUTE_TYPE": "COHORTS", - "COHORT_MEMBERSHIPS": [ - { - "COHORT_LABEL": "rsv_75to79", - "DATE_JOINED": "20231020" - } - ] - }, - { - "NHS_NUMBER": "9658219012", - "ATTRIBUTE_TYPE": "PERSON", - "DATE_OF_BIRTH": "19500601", - "GENDER": "2", - "POSTCODE": "CB3 8DX", - "POSTCODE_SECTOR": "CB38", - "POSTCODE_OUTCODE": "CB3", - "MSOA": "E02007085", - "LSOA": "E01018223", - "GP_PRACTICE_CODE": "D81046", - "PCN": "U75549", - "ICB": "QUE", - "COMMISSIONING_REGION": "Y61", - "13Q_FLAG": "N", - "CARE_HOME_FLAG": "N", - "DE_FLAG": "N" - } - ] -} diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_011.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_011.json deleted file mode 100644 index 6ebd3beb1..000000000 --- a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_011.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "scenario_name": "RSV - Vita Integration - NotActionable - You should have the RSV vaccine - age rolling - TooClose", - "request_headers": { - "nhs-login-nhs-number": "9658220142" - }, - "config_filenames": [ - "vita_integration_test_config.json" - ], - "data": [ - { - "NHS_NUMBER": "9658220142", - "ATTRIBUTE_TYPE": "COHORTS", - "COHORT_MEMBERSHIPS": [ - { - "COHORT_LABEL": "rsv_75to79", - "DATE_JOINED": "20231020" - } - ] - }, - { - "NHS_NUMBER": "9658220142", - "ATTRIBUTE_TYPE": "PERSON", - "DATE_OF_BIRTH": "19500601", - "GENDER": "2", - "POSTCODE": "CB3 8DX", - "POSTCODE_SECTOR": "CB38", - "POSTCODE_OUTCODE": "CB3", - "MSOA": "E02007085", - "LSOA": "E01018223", - "GP_PRACTICE_CODE": "D81046", - "PCN": "U75549", - "ICB": "QUE", - "COMMISSIONING_REGION": "Y61", - "13Q_FLAG": "N", - "CARE_HOME_FLAG": "N", - "DE_FLAG": "N" - } - ] -} diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_012.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_012.json deleted file mode 100644 index a91993317..000000000 --- a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_012.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "scenario_name": "RSV - Vita Integration - NotActionable - You should have the RSV vaccine ( managed setting ) - age rolling - OtherSetting", - "request_headers": { - "nhs-login-nhs-number": "9658220150" - }, - "config_filenames": [ - "vita_integration_test_config.json" - ], - "data": [ - { - "NHS_NUMBER": "9658220150", - "ATTRIBUTE_TYPE": "PERSON", - "DATE_OF_BIRTH": "19500601", - "GENDER": "2", - "POSTCODE": "CB3 8DX", - "POSTCODE_SECTOR": "CB38", - "POSTCODE_OUTCODE": "CB3", - "MSOA": "E02007085", - "LSOA": "E01018223", - "GP_PRACTICE_CODE": "D81046", - "PCN": "U75549", - "ICB": "QUE", - "COMMISSIONING_REGION": "Y61", - "13Q_FLAG": "N", - "CARE_HOME_FLAG": "Y", - "DE_FLAG": "N" - }, - { - "NHS_NUMBER": "9658220150", - "ATTRIBUTE_TYPE": "COHORTS", - "COHORT_MEMBERSHIPS": [ - { - "COHORT_LABEL": "rsv_75to79", - "DATE_JOINED": "20231020" - } - ] - } - ] -} diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_013.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_013.json deleted file mode 100644 index 77057e50b..000000000 --- a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_013.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "scenario_name": "RSV - Vita Integration - NotActionable - You should have the RSV vaccine (already vaccinated) - unknown membership - AlreadyVaccinated", - "request_headers": { - "nhs-login-nhs-number": "9450114080" - }, - "config_filenames": [ - "vita_integration_test_config.json" - ], - "data": [ - { - "NHS_NUMBER": "9450114080", - "ATTRIBUTE_TYPE": "PERSON", - "DATE_OF_BIRTH": "19500601", - "GENDER": "2", - "POSTCODE": "CB3 8DX", - "POSTCODE_SECTOR": "CB38", - "POSTCODE_OUTCODE": "CB3", - "MSOA": "E02007085", - "LSOA": "E01018223", - "GP_PRACTICE_CODE": "D81046", - "PCN": "U75549", - "ICB": "QUE", - "COMMISSIONING_REGION": "Y61", - "13Q_FLAG": "N", - "CARE_HOME_FLAG": "Y", - "DE_FLAG": "N" - }, - { - "NHS_NUMBER": "9450114080", - "ATTRIBUTE_TYPE": "RSV", - "LAST_SUCCESSFUL_DATE": "<>" - } - ] -} diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_014.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_014.json deleted file mode 100644 index 7ec5f92d9..000000000 --- a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_014.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "scenario_name": "RSV - Vita Integration - NotActionable - You should have the RSV vaccine (already vaccinated) - empty cohorts - AlreadyVaccinated", - "request_headers": { - "nhs-login-nhs-number": "9466447939" - }, - "config_filenames": [ - "vita_integration_test_config.json" - ], - "data": [ - { - "NHS_NUMBER": "9466447939", - "ATTRIBUTE_TYPE": "RSV", - "LAST_SUCCESSFUL_DATE": "<>" - }, - { - "NHS_NUMBER": "9466447939", - "ATTRIBUTE_TYPE": "PERSON", - "DATE_OF_BIRTH": "19500601", - "GENDER": "2", - "POSTCODE": "CB3 8DX", - "POSTCODE_SECTOR": "CB38", - "POSTCODE_OUTCODE": "CB3", - "MSOA": "E02007085", - "LSOA": "E01018223", - "GP_PRACTICE_CODE": "D81046", - "PCN": "U75549", - "ICB": "QUE", - "COMMISSIONING_REGION": "Y61", - "13Q_FLAG": "N", - "CARE_HOME_FLAG": "Y", - "DE_FLAG": "N" - } - ] -} diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_015.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_015.json deleted file mode 100644 index 2f663767e..000000000 --- a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_015.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "scenario_name": "RSV - Vita Integration - - Not Eligible", - "request_headers": { - "nhs-login-nhs-number": "9657933617" - }, - "config_filenames": [ - "vita_integration_test_config.json" - ], - "data": [ - { - "NHS_NUMBER": "9657933617", - "ATTRIBUTE_TYPE": "PERSON", - "DATE_OF_BIRTH": "19500601", - "GENDER": "2", - "POSTCODE": "CB3 8DX", - "POSTCODE_SECTOR": "CB38", - "POSTCODE_OUTCODE": "CB3", - "MSOA": "E02007085", - "LSOA": "E01018223", - "GP_PRACTICE_CODE": "D81046", - "PCN": "U75549", - "ICB": "QUE", - "COMMISSIONING_REGION": "Y61", - "13Q_FLAG": "N", - "CARE_HOME_FLAG": "Y", - "DE_FLAG": "N" - }, - { - "NHS_NUMBER": "9657933617", - "ATTRIBUTE_TYPE": "RSV", - "LAST_SUCCESSFUL_DATE": null, - "BOOKED_APPOINTMENT_DATE": null - } - ] -} diff --git a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_500.json b/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_500.json deleted file mode 100644 index 79aff5d5e..000000000 --- a/scripts/seed_users/vitaIntegrationTestData/AUTO_RSV_VITA_INT_500.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "scenario_name": "RSV - Vita Integration - 500", - "request_headers": { - "nhs-login-nhs-number": "9436793375" - }, - "expected_response_code": 500, - "config_filenames": [ - "vita_integration_test_config.json" - ], - "data": [ - { - "NHS_NUMBER": "9436793375", - "ATTRIBUTE_TYPE": "COHORTS", - "COHORT_MEMBERSHIPS": [ - { - "COHORT_LABEL": "rsv_75to79", - "DATE_JOINED": "20231020" - } - ] - }, - { - "NHS_NUMBER": "9436793375", - "ATTRIBUTE_TYPE": "PERSON", - "DATE_OF_BIRTH": 19500601, - "GENDER": "0", - "POSTCODE": "SG8 6EG", - "POSTCODE_SECTOR": "SG86", - "POSTCODE_OUTCODE": "SG8", - "MSOA": "E02003792", - "LSOA": "E01018267", - "GP_PRACTICE_CODE": "D81046", - "PCN": "U75549", - "ICB": "QUE", - "COMMISSIONING_REGION": "Y61", - "13Q_FLAG": "N", - "CARE_HOME_FLAG": "N", - "DE_FLAG": "N" - } - ] -} From 724227f67d13fbafc77f9f62b239e21ff2b4a41c Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Mon, 1 Sep 2025 11:26:39 +0100 Subject: [PATCH 07/14] delete all items for nhs number and reinsert --- scripts/seed_users/README.md | 70 +++++++++++++++++++++++++++++ scripts/seed_users/seed_dynamodb.py | 34 +++++++++----- 2 files changed, 92 insertions(+), 12 deletions(-) create mode 100644 scripts/seed_users/README.md diff --git a/scripts/seed_users/README.md b/scripts/seed_users/README.md new file mode 100644 index 000000000..e72160707 --- /dev/null +++ b/scripts/seed_users/README.md @@ -0,0 +1,70 @@ +# 🧬 DynamoDB Seeder Script + +This script deletes and inserts items into a DynamoDB table using JSON seed data. It’s designed for integration testing and local development workflows. +This script is user in the preprod seed workflow. + +--- + +## πŸ“¦ Requirements + +- Python 3.13 +- AWS credentials configured (via `~/.aws/credentials`, environment variables, or IAM role) +- Required Python packages: + ```bash + pip install boto3 + ``` + +--- + +## πŸš€ Usage + +From the project root, run: + +```bash +python scripts/seed_users/seed_dynamodb.py \ + --table-name \ + --region \ + --data-folder +``` + +### Example: + +```bash +python scripts/seed_users/seed_dynamodb.py \ + --table-name eligibility-signposting-api-dev-eligibility_datastore \ + --region eu-west-2 \ + --data-folder tests/e2e/data/dynamoDB/vitaIntegrationTestData +``` + +--- + +## πŸ“ JSON Data Format + +Each `.json` file in the specified folder should follow this structure: + +```json +{ + "data": [ + { + "NHS_NUMBER": "1234567890", + "ATTRIBUTE_TYPE": "COHORTS", + "otherAttribute1": "value", + "otherAttribute2": "value" + } + ] +} +``` + +## 🧹 What It Does + +1. **Deletes** existing items in the table matching `NHS_NUMBER` from all JSON files. +2. **Inserts** all items from the same files into the table. + +--- + +## πŸ›‘οΈ Safety Notes + +- This script performs destructive operations β€” do not use this in prod environment. +- Ensure your AWS credentials have appropriate permissions for `dynamodb:DeleteItem` and `dynamodb:PutItem`. + +--- diff --git a/scripts/seed_users/seed_dynamodb.py b/scripts/seed_users/seed_dynamodb.py index 161bfe7f8..d5e4dea6e 100644 --- a/scripts/seed_users/seed_dynamodb.py +++ b/scripts/seed_users/seed_dynamodb.py @@ -13,11 +13,13 @@ def parse_args(): parser.add_argument("--data-folder", default="vitaIntegrationTestData/", help="Folder containing JSON seed data") return parser.parse_args() + def resolve_data_folder(path): return os.path.abspath(path) -def get_keys_from_folder(data_folder): - keys_to_delete = [] + +def get_unique_nhs_numbers(data_folder): + nhs_numbers = set() json_files = glob.glob(os.path.join(data_folder, "*.json")) for file_path in json_files: with open(file_path) as f: @@ -25,16 +27,24 @@ def get_keys_from_folder(data_folder): items = payload.get("data", []) for item in items: nhs_number = item.get("NHS_NUMBER") - attr_type = item.get("ATTRIBUTE_TYPE") - if nhs_number and attr_type: - keys_to_delete.append({"NHS_NUMBER": nhs_number, "ATTRIBUTE_TYPE": attr_type}) - return keys_to_delete + if nhs_number: + nhs_numbers.add(nhs_number) + return list(nhs_numbers) -def delete_specific_items(table, keys): - with table.batch_writer() as batch: - for key in keys: - batch.delete_item(Key=key) +def delete_all_items_for_nhs_numbers(table, nhs_numbers): + for nhs_number in nhs_numbers: + response = table.query( + KeyConditionExpression=boto3.dynamodb.conditions.Key("NHS_NUMBER").eq(nhs_number) + ) + items = response.get("Items", []) + with table.batch_writer() as batch: + for item in items: + key = { + "NHS_NUMBER": item["NHS_NUMBER"], + "ATTRIBUTE_TYPE": item["ATTRIBUTE_TYPE"] + } + batch.delete_item(Key=key) def insert_data_from_folder(table, data_folder): @@ -63,8 +73,8 @@ def main(): if not os.path.isdir(data_folder): raise ValueError(f"Data folder '{data_folder}' does not exist or is not a directory.") - keys_to_delete = get_keys_from_folder(data_folder) - delete_specific_items(table, keys_to_delete) + nhs_numbers = get_unique_nhs_numbers(data_folder) + delete_all_items_for_nhs_numbers(table, nhs_numbers) insert_data_from_folder(table, data_folder) From 2a548e5bccf2e79ba82e49df8149f70ade8e1931 Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Mon, 1 Sep 2025 11:31:14 +0100 Subject: [PATCH 08/14] data folder updated --- .github/workflows/cicd-4b-preprod-seed-users.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd-4b-preprod-seed-users.yml b/.github/workflows/cicd-4b-preprod-seed-users.yml index b38af6a2b..f600ff509 100644 --- a/.github/workflows/cicd-4b-preprod-seed-users.yml +++ b/.github/workflows/cicd-4b-preprod-seed-users.yml @@ -27,7 +27,7 @@ jobs: contents: read env: AWS_REGION: eu-west-2 - DATA_FOLDER: vitaIntegrationTestData/ + DATA_FOLDER: tests/e2e/data/dynamoDB/vitaIntegrationTestData DYNAMODB_TABLE: eligibility-signposting-api-preprod-eligibility_datastore steps: From c9a3b6b76bf1194982caa31e5deda2e3b6b4a0f5 Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Mon, 1 Sep 2025 11:35:14 +0100 Subject: [PATCH 09/14] readme fix --- scripts/seed_users/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/seed_users/README.md b/scripts/seed_users/README.md index e72160707..3116ce9ef 100644 --- a/scripts/seed_users/README.md +++ b/scripts/seed_users/README.md @@ -1,7 +1,7 @@ # 🧬 DynamoDB Seeder Script This script deletes and inserts items into a DynamoDB table using JSON seed data. It’s designed for integration testing and local development workflows. -This script is user in the preprod seed workflow. +This script is user in the Preprod seed workflow. --- @@ -10,6 +10,7 @@ This script is user in the preprod seed workflow. - Python 3.13 - AWS credentials configured (via `~/.aws/credentials`, environment variables, or IAM role) - Required Python packages: + ```bash pip install boto3 ``` @@ -27,7 +28,7 @@ python scripts/seed_users/seed_dynamodb.py \ --data-folder ``` -### Example: +### Example ```bash python scripts/seed_users/seed_dynamodb.py \ From 671d7b04bdfb0bdc61ee29cd0d63115984b8421b Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Mon, 1 Sep 2025 11:53:16 +0100 Subject: [PATCH 10/14] final dev test --- .github/workflows/cicd-4b-preprod-seed-users.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cicd-4b-preprod-seed-users.yml b/.github/workflows/cicd-4b-preprod-seed-users.yml index f600ff509..0005259b1 100644 --- a/.github/workflows/cicd-4b-preprod-seed-users.yml +++ b/.github/workflows/cicd-4b-preprod-seed-users.yml @@ -1,12 +1,12 @@ -name: Preprod - Seed DynamoDB table +name: dev - Seed DynamoDB table concurrency: - group: seed-preprod-dynamodb + group: seed-dev-dynamodb cancel-in-progress: false on: workflow_run: - workflows: [ "Preprod Deploy" ] + workflows: [ "CI/CD pull request" ] types: - completed workflow_dispatch: @@ -16,19 +16,19 @@ on: required: true type: choice options: - - preprod + - dev jobs: seed-dynamodb: runs-on: ubuntu-latest - environment: "preprod" + environment: "dev" permissions: id-token: write contents: read env: AWS_REGION: eu-west-2 DATA_FOLDER: tests/e2e/data/dynamoDB/vitaIntegrationTestData - DYNAMODB_TABLE: eligibility-signposting-api-preprod-eligibility_datastore + DYNAMODB_TABLE: eligibility-signposting-api-dev-eligibility_datastore steps: - name: Checkout repo From 466f392e6f5dace62b7bc13d05e0ee5f3f4491d0 Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Mon, 1 Sep 2025 12:05:57 +0100 Subject: [PATCH 11/14] final dev test --- .github/workflows/cicd-4b-preprod-seed-users.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cicd-4b-preprod-seed-users.yml b/.github/workflows/cicd-4b-preprod-seed-users.yml index 0005259b1..b58845b2f 100644 --- a/.github/workflows/cicd-4b-preprod-seed-users.yml +++ b/.github/workflows/cicd-4b-preprod-seed-users.yml @@ -6,9 +6,12 @@ concurrency: on: workflow_run: - workflows: [ "CI/CD pull request" ] + workflows: [ "CI/CD publish" ] types: - completed + filters: + conclusion: + - success workflow_dispatch: inputs: environment: From a533a26c2891f3241bd457d25962151a59aa774e Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Mon, 1 Sep 2025 12:10:29 +0100 Subject: [PATCH 12/14] final dev test --- .github/workflows/cicd-4b-preprod-seed-users.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd-4b-preprod-seed-users.yml b/.github/workflows/cicd-4b-preprod-seed-users.yml index b58845b2f..748dba785 100644 --- a/.github/workflows/cicd-4b-preprod-seed-users.yml +++ b/.github/workflows/cicd-4b-preprod-seed-users.yml @@ -6,7 +6,7 @@ concurrency: on: workflow_run: - workflows: [ "CI/CD publish" ] + workflows: [ "CI/CD pull request" ] types: - completed filters: From 53c02bcde439df020af05b9ee61cb14e66f384eb Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Mon, 1 Sep 2025 12:16:13 +0100 Subject: [PATCH 13/14] seeding env dev -> preprod --- .github/workflows/cicd-4b-preprod-seed-users.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cicd-4b-preprod-seed-users.yml b/.github/workflows/cicd-4b-preprod-seed-users.yml index 748dba785..af04f0f5f 100644 --- a/.github/workflows/cicd-4b-preprod-seed-users.yml +++ b/.github/workflows/cicd-4b-preprod-seed-users.yml @@ -1,12 +1,12 @@ -name: dev - Seed DynamoDB table +name: preprod - Seed DynamoDB table concurrency: - group: seed-dev-dynamodb + group: seed-preprod-dynamodb cancel-in-progress: false on: workflow_run: - workflows: [ "CI/CD pull request" ] + workflows: [ "Preprod Deploy" ] types: - completed filters: @@ -19,19 +19,19 @@ on: required: true type: choice options: - - dev + - preprod jobs: seed-dynamodb: runs-on: ubuntu-latest - environment: "dev" + environment: "preprod" permissions: id-token: write contents: read env: AWS_REGION: eu-west-2 DATA_FOLDER: tests/e2e/data/dynamoDB/vitaIntegrationTestData - DYNAMODB_TABLE: eligibility-signposting-api-dev-eligibility_datastore + DYNAMODB_TABLE: eligibility-signposting-api-preprod-eligibility_datastore steps: - name: Checkout repo From 26171af00192d59c6c1fc66649a725b102954f26 Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Mon, 1 Sep 2025 12:21:11 +0100 Subject: [PATCH 14/14] removed redundant policies --- .../iams-developer-roles/iams_permissions_boundary.tf | 6 ------ 1 file changed, 6 deletions(-) diff --git a/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf b/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf index dcf791bfc..c31f806d7 100644 --- a/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf +++ b/infrastructure/stacks/iams-developer-roles/iams_permissions_boundary.tf @@ -36,12 +36,6 @@ data "aws_iam_policy_document" "permissions_boundary" { "dynamodb:CreateTable", "dynamodb:TagResource", "dynamodb:ListTagsOfResource", - # Only used in preprod - "dynamodb:GetItem", - "dynamodb:PutItem", - "dynamodb:DeleteItem", - "dynamodb:Scan", - "dynamodb:BatchWriteItem", # EC2 - networking infrastructure "ec2:Describe*",