From 3a8cf8bc4305ae66f945bf7e7caea8a1f5b73f58 Mon Sep 17 00:00:00 2001 From: SeanSteberis <103416906+seansteberisal@users.noreply.github.com> Date: Wed, 1 Oct 2025 14:13:25 +0100 Subject: [PATCH 1/6] rework of DynamoDB database reset functionality (too complex, needs further improvements) --- utils/common_utils.py | 16 +++++ utils/dynamo_helper.py | 143 ++++++++++++++++++++++++++++------------- 2 files changed, 115 insertions(+), 44 deletions(-) create mode 100644 utils/common_utils.py diff --git a/utils/common_utils.py b/utils/common_utils.py new file mode 100644 index 00000000..a4883745 --- /dev/null +++ b/utils/common_utils.py @@ -0,0 +1,16 @@ +from pathlib import Path + + +def save_to_file(file_name, data, directory=None): + if directory is not None: + Path.mkdir(directory, parents=True, exist_ok=True) + else: + directory = Path.cwd() + + with Path.open(directory / file_name, "w", encoding="utf-8") as f: + f.write(data) + + +def load_from_file(file_name): + with Path.open(file_name, "r", encoding="utf-8") as f: + return f.read() diff --git a/utils/dynamo_helper.py b/utils/dynamo_helper.py index ac9ff4dd..5e5f1e51 100644 --- a/utils/dynamo_helper.py +++ b/utils/dynamo_helper.py @@ -2,11 +2,14 @@ import os import boto3 +from utils.common_utils import save_to_file, load_from_file from botocore.exceptions import ClientError from dotenv import load_dotenv logger = logging.getLogger(__name__) load_dotenv() +ENVIRONMENT = os.getenv("ENVIRONMENT") +DYNAMODB_TABLE_NAME = os.getenv("DYNAMODB_TABLE_NAME") class DynamoDBHelper: @@ -71,7 +74,7 @@ def delete_item(self, key: dict): else: return response - def get_table_information(self): + def describe_table(self): logger.debug(f"Describe table: {self.table_name}") try: table_description = self.dynamodb_client.describe_table( @@ -82,42 +85,78 @@ def get_table_information(self): f"Failed to get table information: {e.response["Error"]["Message"]}" ) raise - table_arn = table_description["Table"]["TableArn"] - return table_description, table_arn + save_to_file( + "description.json", table_description, directory="data/dynamoDB/temp" + ) + return table_description + + def get_table_tags(self, table_arn): + tags = self.dynamodb_client.list_tags_of_resource(ResourceArn=table_arn) + save_to_file("tags.json", tags, directory="data/dynamoDB/temp") + return tags + + def set_table_tags(self, table_arn, tags): + self.dynamodb_client.tag_resource(ResourceArn=table_arn, Tags=tags["Tags"]) + + def create_table(self, attribute_definitions, key_schema): + logger.info(f"Creating table '{self.table_name}'...") + try: + self.dynamodb_client.create_table( + TableName=self.table_name, + KeySchema=key_schema, + AttributeDefinitions=attribute_definitions, + BillingMode="PAY_PER_REQUEST", + ) + + # Wait for the new table to become active + logger.debug( + f"Waiting for table '{self.table_name}' to be created and become active..." + ) + waiter = self.dynamodb_client.get_waiter("table_exists") + waiter.wait(TableName=self.table_name) + logger.info( + f"Table '{self.table_name}' successfully created and is now active." + ) + except ClientError as e: + logger.exception(f"Error creating table '{self.table_name}': {e}") + + def delete_table(self, table_name): + logger.info(f"Deleting table '{table_name}'...") + self.dynamodb_client.delete_table(TableName=table_name) + + # Wait for the table to be completely deleted + logger.debug(f"Waiting for table '{self.table_name}' to be deleted...") + waiter = self.dynamodb_client.get_waiter("table_not_exists") + waiter.wait(TableName=self.table_name) + logger.info(f"Table '{self.table_name}' successfully deleted.") def reset_dynamo_tables(): - environment = os.getenv("ENVIRONMENT") logger.info("Resetting DynamoDB. This may take a few moments, please be patient.") - if environment not in ["dev", "test"]: + if ENVIRONMENT not in ["dev", "test"]: logger.warning( - f"{environment} is not supported. Resetting DynamoDB is only supported in dev or test." + f"{ENVIRONMENT} is not supported. Resetting DynamoDB is only supported in dev or test." ) return - dynamo_db_table = DynamoDBHelper(os.getenv("DYNAMODB_TABLE_NAME")) + dynamo_db_table = DynamoDBHelper(DYNAMODB_TABLE_NAME) table_name = dynamo_db_table.table_name # --- Step 1: Fetch table information --- - table_description, table_arn = dynamo_db_table.get_table_information() + table_description = dynamo_db_table.describe_table() + table_arn = table_description["Table"]["TableArn"] + logger.debug(f"TableArn: {table_arn}") + key_schema = table_description["Table"]["KeySchema"] logger.debug(f"KeySchema: {key_schema}") + attribute_definitions = table_description["Table"]["AttributeDefinitions"] logger.debug(f"attribute_definitions: {attribute_definitions}") - tags = dynamo_db_table.dynamodb_client.list_tags_of_resource(ResourceArn=table_arn) + + tags = dynamo_db_table.get_table_tags(table_arn) logger.debug(f"tags: {tags}") # --- Step 2: Delete the table --- try: - logger.info(f"Deleting table '{table_name}'...") - dynamo_db_table.dynamodb_client.delete_table(TableName=table_name) - - # Wait for the table to be completely deleted - logger.debug( - f"Waiting for table '{dynamo_db_table.table_name}' to be deleted..." - ) - waiter = dynamo_db_table.dynamodb_client.get_waiter("table_not_exists") - waiter.wait(TableName=dynamo_db_table.table_name) - logger.info(f"Table '{dynamo_db_table.table_name}' successfully deleted.") - + dynamo_db_table.delete_table(table_name) except ClientError as e: if e.response["Error"]["Code"] == "ResourceNotFoundException": logger.warning( @@ -127,40 +166,56 @@ def reset_dynamo_tables(): logger.exception( f"Error deleting table '{dynamo_db_table.table_name}': {e}" ) - return + raise e # --- Step 3: Recreate the table --- - logger.info(f"Creating table '{dynamo_db_table.table_name}'...") - try: - dynamo_db_table.dynamodb_client.create_table( - TableName=dynamo_db_table.table_name, - KeySchema=key_schema, - AttributeDefinitions=attribute_definitions, - BillingMode="PAY_PER_REQUEST", + if attribute_definitions is None or key_schema is None: + logger.warning( + "Unable to create table because AttributeDefinitions or KeySchema are not defined. " + "Attempting to load from backup file..." ) + try: + table_description = load_from_file("data/dynamoDB/temp/description.json") + key_schema = table_description["Table"]["KeySchema"] + logger.warning(f"KeySchema loaded from backup file: {key_schema}") - # Wait for the new table to become active - logger.debug( - f"Waiting for table '{dynamo_db_table.table_name}' to be created and become active..." - ) - waiter = dynamo_db_table.dynamodb_client.get_waiter("table_exists") - waiter.wait(TableName=dynamo_db_table.table_name) - logger.info( - f"Table '{dynamo_db_table.table_name}' successfully created and is now active." - ) - except ClientError as e: - logger.exception(f"Error creating table '{dynamo_db_table.table_name}': {e}") + attribute_definitions = table_description["Table"]["AttributeDefinitions"] + logger.warning( + f"attribute_definitions loaded from backup file: {attribute_definitions}" + ) + except FileNotFoundError as e: + logger.exception( + "Failed to load table information and no backup file was found." + ) + raise e + dynamo_db_table.create_table(attribute_definitions, key_schema) # --- Step 4: Restore the tags --- - logger.debug(f"restoring tags to table '{dynamo_db_table.table_name}'...") - dynamo_db_table.dynamodb_client.tag_resource( - ResourceArn=table_arn, Tags=tags["Tags"] - ) + logger.debug(f"Adding tags to table '{dynamo_db_table.table_name}'...") + if table_arn is None: + logger.warning( + "Unable to find TableArn, attempting to load from backup file..." + ) + table_description = load_from_file("data/dynamoDB/temp/description.json") + table_arn = table_description["Table"]["TableArn"] + logger.warning(f"TableArn loaded from backup file: : {table_arn}") + if tags is None: + logger.warning("Unable to add tags, attempting to load from backup file...") + try: + tags = load_from_file("data/dynamoDB/temp/tags.json") + logger.warning(f"tags loaded from backup file: {tags}") + + logger.debug(f"TableArn: {table_arn}") + dynamo_db_table.set_table_tags(table_arn, tags) + except FileNotFoundError: + logger.error( + "Failed to load table information and no backup file was found." + ) def insert_into_dynamo(data): logger.debug("Inserting into Dynamo: %s", data) - table = DynamoDBHelper(os.getenv("DYNAMODB_TABLE_NAME")) + table = DynamoDBHelper(DYNAMODB_TABLE_NAME) for item in data: try: table.insert_item(item) From fa3a5070d9e3d670d2f8e6f9f9c70c672c30c86a Mon Sep 17 00:00:00 2001 From: SeanSteberis <103416906+seansteberisal@users.noreply.github.com> Date: Wed, 1 Oct 2025 14:59:46 +0100 Subject: [PATCH 2/6] reduced complexity of reset dynamodb table method --- utils/dynamo_helper.py | 70 +++++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 28 deletions(-) diff --git a/utils/dynamo_helper.py b/utils/dynamo_helper.py index 5e5f1e51..66a6fca0 100644 --- a/utils/dynamo_helper.py +++ b/utils/dynamo_helper.py @@ -8,6 +8,7 @@ logger = logging.getLogger(__name__) load_dotenv() + ENVIRONMENT = os.getenv("ENVIRONMENT") DYNAMODB_TABLE_NAME = os.getenv("DYNAMODB_TABLE_NAME") @@ -70,7 +71,7 @@ def delete_item(self, key: dict): logger.exception( "Failed to delete item: %s", e.response["Error"]["Message"] ) - raise + raise e else: return response @@ -131,6 +132,40 @@ def delete_table(self, table_name): logger.info(f"Table '{self.table_name}' successfully deleted.") +def restore_tags_to_table(dynamo_db_table: DynamoDBHelper, table_arn, tags): + if table_arn is None: + logger.warning( + "Unable to find TableArn, attempting to load from backup file..." + ) + table_description = load_from_file("data/dynamoDB/temp/description.json") + table_arn = table_description["Table"]["TableArn"] + logger.warning(f"TableArn loaded from backup file: : {table_arn}") + if tags is None: + logger.warning("Unable to add tags, attempting to load from backup file...") + try: + tags = load_from_file("data/dynamoDB/temp/tags.json") + logger.warning(f"tags loaded from backup file: {tags}") + + logger.debug(f"TableArn: {table_arn}") + dynamo_db_table.set_table_tags(table_arn, tags) + except FileNotFoundError: + logger.error("Failed to restore tags and no backup file was found.") + + +def get_attribute_definitions_and_key_schema_from_file( + attribute_definitions: str | int | bytes, key_schema: str | int | bytes +) -> tuple[str | int | bytes, str | int | bytes]: + table_description = load_from_file("data/dynamoDB/temp/description.json") + key_schema = table_description["Table"]["KeySchema"] + logger.warning(f"KeySchema loaded from backup file: {key_schema}") + + attribute_definitions = table_description["Table"]["AttributeDefinitions"] + logger.warning( + f"attribute_definitions loaded from backup file: {attribute_definitions}" + ) + return attribute_definitions, key_schema + + def reset_dynamo_tables(): logger.info("Resetting DynamoDB. This may take a few moments, please be patient.") if ENVIRONMENT not in ["dev", "test"]: @@ -139,7 +174,7 @@ def reset_dynamo_tables(): ) return dynamo_db_table = DynamoDBHelper(DYNAMODB_TABLE_NAME) - table_name = dynamo_db_table.table_name + table_name = DYNAMODB_TABLE_NAME # --- Step 1: Fetch table information --- table_description = dynamo_db_table.describe_table() @@ -175,13 +210,10 @@ def reset_dynamo_tables(): "Attempting to load from backup file..." ) try: - table_description = load_from_file("data/dynamoDB/temp/description.json") - key_schema = table_description["Table"]["KeySchema"] - logger.warning(f"KeySchema loaded from backup file: {key_schema}") - - attribute_definitions = table_description["Table"]["AttributeDefinitions"] - logger.warning( - f"attribute_definitions loaded from backup file: {attribute_definitions}" + attribute_definitions, key_schema = ( + get_attribute_definitions_and_key_schema_from_file( + attribute_definitions, key_schema + ) ) except FileNotFoundError as e: logger.exception( @@ -192,25 +224,7 @@ def reset_dynamo_tables(): # --- Step 4: Restore the tags --- logger.debug(f"Adding tags to table '{dynamo_db_table.table_name}'...") - if table_arn is None: - logger.warning( - "Unable to find TableArn, attempting to load from backup file..." - ) - table_description = load_from_file("data/dynamoDB/temp/description.json") - table_arn = table_description["Table"]["TableArn"] - logger.warning(f"TableArn loaded from backup file: : {table_arn}") - if tags is None: - logger.warning("Unable to add tags, attempting to load from backup file...") - try: - tags = load_from_file("data/dynamoDB/temp/tags.json") - logger.warning(f"tags loaded from backup file: {tags}") - - logger.debug(f"TableArn: {table_arn}") - dynamo_db_table.set_table_tags(table_arn, tags) - except FileNotFoundError: - logger.error( - "Failed to load table information and no backup file was found." - ) + restore_tags_to_table(dynamo_db_table, table_arn, tags) def insert_into_dynamo(data): From e97e75feb95fc17c27b03e6210aa8766c60d6ce8 Mon Sep 17 00:00:00 2001 From: SeanSteberis <103416906+seansteberisal@users.noreply.github.com> Date: Sat, 4 Oct 2025 22:35:29 +0100 Subject: [PATCH 3/6] changes to dynamoDB reset --- .gitignore | 1 + Makefile | 5 +- poetry.lock | 29 +++++--- tests/test_reset_db.py | 5 ++ utils/common_utils.py | 8 +-- utils/data_helper.py | 3 +- utils/dynamo_helper.py | 152 +++++++++++++++++++++++++---------------- 7 files changed, 127 insertions(+), 76 deletions(-) create mode 100644 tests/test_reset_db.py diff --git a/.gitignore b/.gitignore index e5ae5a8a..ee8d5fc6 100644 --- a/.gitignore +++ b/.gitignore @@ -255,3 +255,4 @@ vscode # Node Modules node_modules /certs/ +/data/dynamoDB/temp/ diff --git a/Makefile b/Makefile index e4165e95..5db7f927 100644 --- a/Makefile +++ b/Makefile @@ -72,7 +72,10 @@ deep-clean-install: pre-commit: poetry run pre-commit run --all-files -run-tests: guard-env guard-log_level +clear-db: guard-env guard-log_level + poetry run pytest --env=${env} --log-cli-level=${log_level} -s tests/test_reset_db.py + +run-tests: guard-env guard-log_level clear-db poetry run pytest --env=${env} --log-cli-level=${log_level} -s tests/test_story_tests.py poetry run pytest --env=${env} --log-cli-level=${log_level} -s tests/test_error_scenario_tests.py poetry run pytest --env=${env} --log-cli-level=${log_level} -s tests/test_vita_integration_tests.py diff --git a/poetry.lock b/poetry.lock index 235c4f42..997f38ae 100644 --- a/poetry.lock +++ b/poetry.lock @@ -377,7 +377,7 @@ description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" groups = ["main"] -markers = "platform_system == \"Windows\" or sys_platform == \"win32\"" +markers = "sys_platform == \"win32\" or platform_system == \"Windows\"" files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, @@ -1371,13 +1371,6 @@ optional = false python-versions = ">=3.8" groups = ["main", "dev"] files = [ - {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6"}, - {file = "PyYAML-6.0.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369"}, - {file = "PyYAML-6.0.3-cp38-cp38-win32.whl", hash = "sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295"}, - {file = "PyYAML-6.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b"}, {file = "pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b"}, {file = "pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956"}, {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8"}, @@ -1791,6 +1784,24 @@ files = [ {file = "tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9"}, ] +[[package]] +name = "tzlocal" +version = "5.3.1" +description = "tzinfo object for the local timezone" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d"}, + {file = "tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd"}, +] + +[package.dependencies] +tzdata = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +devenv = ["check-manifest", "pytest (>=4.3)", "pytest-cov", "pytest-mock (>=3.3)", "zest.releaser"] + [[package]] name = "urllib3" version = "2.5.0" @@ -1860,4 +1871,4 @@ test = ["pytest (>=3.0.0)"] [metadata] lock-version = "2.1" python-versions = "^3.13" -content-hash = "752c6c0403bd761d5290dfa5cf0a2be96614d921f4dce6453130dc0e8d7a80a9" +content-hash = "d5fc8ce1573caaa7ef506bd01afb8d8f5720d779aea1f48617d9bc198e2ed418" diff --git a/tests/test_reset_db.py b/tests/test_reset_db.py new file mode 100644 index 00000000..a5a5770e --- /dev/null +++ b/tests/test_reset_db.py @@ -0,0 +1,5 @@ +from utils.dynamo_helper import reset_dynamo_tables + + +def test_reset_db(): + reset_dynamo_tables() diff --git a/utils/common_utils.py b/utils/common_utils.py index a4883745..911fed78 100644 --- a/utils/common_utils.py +++ b/utils/common_utils.py @@ -1,16 +1,16 @@ from pathlib import Path -def save_to_file(file_name, data, directory=None): +def save_to_file(file_name: str, data, directory: str = None): if directory is not None: - Path.mkdir(directory, parents=True, exist_ok=True) + Path.mkdir(Path(directory), parents=True, exist_ok=True) else: directory = Path.cwd() - with Path.open(directory / file_name, "w", encoding="utf-8") as f: + with Path.open(Path(directory) / file_name, "w") as f: f.write(data) def load_from_file(file_name): - with Path.open(file_name, "r", encoding="utf-8") as f: + with Path.open(file_name, "r") as f: return f.read() diff --git a/utils/data_helper.py b/utils/data_helper.py index 263f5d85..891945ee 100644 --- a/utils/data_helper.py +++ b/utils/data_helper.py @@ -4,7 +4,7 @@ from dotenv import load_dotenv -from .dynamo_helper import insert_into_dynamo, reset_dynamo_tables +from .dynamo_helper import insert_into_dynamo from .placeholder_context import PlaceholderDTO, ResolvedPlaceholderContext from .placeholder_utils import resolve_placeholders @@ -17,7 +17,6 @@ def initialise_tests(folder): folder_path = Path(folder).resolve() all_data, dto = load_all_test_scenarios(folder_path) - reset_dynamo_tables() logger.info("Adding data into Dynamo") for scenario in all_data.values(): insert_into_dynamo(scenario["dynamo_items"]) diff --git a/utils/dynamo_helper.py b/utils/dynamo_helper.py index 66a6fca0..bc55f7a5 100644 --- a/utils/dynamo_helper.py +++ b/utils/dynamo_helper.py @@ -1,3 +1,4 @@ +import json import logging import os @@ -9,9 +10,6 @@ logger = logging.getLogger(__name__) load_dotenv() -ENVIRONMENT = os.getenv("ENVIRONMENT") -DYNAMODB_TABLE_NAME = os.getenv("DYNAMODB_TABLE_NAME") - class DynamoDBHelper: def __init__(self, table_name): @@ -20,6 +18,10 @@ def __init__(self, table_name): self.dynamodb_client = boto3.client("dynamodb", "eu-west-2") self.dynamodb_resource = boto3.resource("dynamodb", "eu-west-2") self.table = self.dynamodb_resource.Table(table_name) + self.table_arn = None + self.attribute_definitions = None + self.key_schema = None + self.tags = None def insert_item(self, item: dict): """ @@ -80,32 +82,43 @@ def describe_table(self): try: table_description = self.dynamodb_client.describe_table( TableName=self.table_name - ) + )["Table"] except ClientError as e: logger.exception( f"Failed to get table information: {e.response["Error"]["Message"]}" ) raise + self.table_arn = table_description["TableArn"] + save_to_file("table_arn.json", self.table_arn, directory="data/dynamoDB/temp") + self.attribute_definitions = table_description["attribute_definitions"] save_to_file( - "description.json", table_description, directory="data/dynamoDB/temp" + "attribute_definitions.json", + self.attribute_definitions, + directory="data/dynamoDB/temp", ) - return table_description + self.key_schema = table_description["key_schema"] + save_to_file("key_schema.json", self.key_schema, directory="data/dynamoDB/temp") - def get_table_tags(self, table_arn): - tags = self.dynamodb_client.list_tags_of_resource(ResourceArn=table_arn) - save_to_file("tags.json", tags, directory="data/dynamoDB/temp") - return tags + return self.table_arn, self.attribute_definitions, self.key_schema - def set_table_tags(self, table_arn, tags): - self.dynamodb_client.tag_resource(ResourceArn=table_arn, Tags=tags["Tags"]) + def get_table_tags(self): + self.tags = self.dynamodb_client.list_tags_of_resource( + ResourceArn=self.table_arn + )["Tags"] + logger.debug(f"tags: {self.tags}") + save_to_file("tags.json", self.tags, directory="data/dynamoDB/temp") + return self.tags - def create_table(self, attribute_definitions, key_schema): + def set_table_tags(self): + self.dynamodb_client.tag_resource(ResourceArn=self.table_arn, Tags=self.tags) + + def create_table(self): logger.info(f"Creating table '{self.table_name}'...") try: self.dynamodb_client.create_table( TableName=self.table_name, - KeySchema=key_schema, - AttributeDefinitions=attribute_definitions, + KeySchema=self.key_schema, + AttributeDefinitions=self.attribute_definitions, BillingMode="PAY_PER_REQUEST", ) @@ -132,22 +145,27 @@ def delete_table(self, table_name): logger.info(f"Table '{self.table_name}' successfully deleted.") -def restore_tags_to_table(dynamo_db_table: DynamoDBHelper, table_arn, tags): - if table_arn is None: +def restore_tags_to_table(dynamo_db_table: DynamoDBHelper): + if dynamo_db_table.table_arn is None: logger.warning( "Unable to find TableArn, attempting to load from backup file..." ) - table_description = load_from_file("data/dynamoDB/temp/description.json") - table_arn = table_description["Table"]["TableArn"] - logger.warning(f"TableArn loaded from backup file: : {table_arn}") - if tags is None: + dynamo_db_table.table_arn = json.loads( + load_from_file("data/dynamoDB/temp/table_arn.json") + ) + logger.warning( + f"TableArn loaded from backup file: : {dynamo_db_table.table_arn}" + ) + if dynamo_db_table.tags is None: logger.warning("Unable to add tags, attempting to load from backup file...") try: - tags = load_from_file("data/dynamoDB/temp/tags.json") - logger.warning(f"tags loaded from backup file: {tags}") + dynamo_db_table.tags = json.loads( + load_from_file("data/dynamoDB/temp/tags.json") + ) + logger.warning(f"tags loaded from backup file: {dynamo_db_table.tags}") - logger.debug(f"TableArn: {table_arn}") - dynamo_db_table.set_table_tags(table_arn, tags) + logger.debug(f"TableArn: {dynamo_db_table.table_arn}") + dynamo_db_table.set_table_tags() except FileNotFoundError: logger.error("Failed to restore tags and no backup file was found.") @@ -156,39 +174,68 @@ def get_attribute_definitions_and_key_schema_from_file( attribute_definitions: str | int | bytes, key_schema: str | int | bytes ) -> tuple[str | int | bytes, str | int | bytes]: table_description = load_from_file("data/dynamoDB/temp/description.json") - key_schema = table_description["Table"]["KeySchema"] + key_schema = table_description["KeySchema"] logger.warning(f"KeySchema loaded from backup file: {key_schema}") - attribute_definitions = table_description["Table"]["AttributeDefinitions"] + attribute_definitions = table_description["AttributeDefinitions"] logger.warning( f"attribute_definitions loaded from backup file: {attribute_definitions}" ) return attribute_definitions, key_schema +def file_backup_exists(): + try: + load_from_file("data/dynamoDB/temp/tags.json") + load_from_file("data/dynamoDB/temp/attributeDefinition.json") + load_from_file("data/dynamoDB/temp/keySchema.json") + load_from_file("data/dynamoDB/temp/tableArn.json") + return True + except FileNotFoundError: + return False + + def reset_dynamo_tables(): + environment = os.getenv("ENVIRONMENT") + dynamodb_table_name = os.getenv("DYNAMODB_TABLE_NAME") + logger.info("Resetting DynamoDB. This may take a few moments, please be patient.") - if ENVIRONMENT not in ["dev", "test"]: + if environment not in ["dev", "test"]: logger.warning( - f"{ENVIRONMENT} is not supported. Resetting DynamoDB is only supported in dev or test." + f"{environment} is not supported. Resetting DynamoDB is only supported in dev or test." ) return - dynamo_db_table = DynamoDBHelper(DYNAMODB_TABLE_NAME) - table_name = DYNAMODB_TABLE_NAME + dynamo_db_table = DynamoDBHelper(dynamodb_table_name) + table_name = dynamodb_table_name # --- Step 1: Fetch table information --- - table_description = dynamo_db_table.describe_table() - table_arn = table_description["Table"]["TableArn"] - logger.debug(f"TableArn: {table_arn}") - - key_schema = table_description["Table"]["KeySchema"] - logger.debug(f"KeySchema: {key_schema}") + try: + table_arn, attribute_definitions, key_schema = dynamo_db_table.describe_table() + logger.debug(f"TableArn: {table_arn}") + logger.debug(f"Attribute Definitions: {attribute_definitions}") + logger.debug(f"Key Schema: {key_schema}") - attribute_definitions = table_description["Table"]["AttributeDefinitions"] - logger.debug(f"attribute_definitions: {attribute_definitions}") + dynamo_db_table.tags = dynamo_db_table.get_table_tags() + except ClientError as e: + logger.warning(f"Error describing table: {e}") + if not file_backup_exists(): + logger.exception( + f"FATAL! Unable to get table information and no backup present: {e}" + ) + raise e + else: + logger.warning("Table information taken from backup files") + dynamo_db_table.tags = load_from_file("data/dynamoDB/temp/tags.json") + dynamo_db_table.attribute_definitions = load_from_file( + "data/dynamoDB/temp/attributeDefinition.json" + ) + dynamo_db_table.key_schema = load_from_file( + "data/dynamoDB/temp/keySchema.json" + ) + dynamo_db_table.table_arn = load_from_file( + "data/dynamoDB/temp/tableArn.json" + ) - tags = dynamo_db_table.get_table_tags(table_arn) - logger.debug(f"tags: {tags}") # --- Step 2: Delete the table --- try: dynamo_db_table.delete_table(table_name) @@ -204,32 +251,17 @@ def reset_dynamo_tables(): raise e # --- Step 3: Recreate the table --- - if attribute_definitions is None or key_schema is None: - logger.warning( - "Unable to create table because AttributeDefinitions or KeySchema are not defined. " - "Attempting to load from backup file..." - ) - try: - attribute_definitions, key_schema = ( - get_attribute_definitions_and_key_schema_from_file( - attribute_definitions, key_schema - ) - ) - except FileNotFoundError as e: - logger.exception( - "Failed to load table information and no backup file was found." - ) - raise e - dynamo_db_table.create_table(attribute_definitions, key_schema) + dynamo_db_table.create_table() # --- Step 4: Restore the tags --- logger.debug(f"Adding tags to table '{dynamo_db_table.table_name}'...") - restore_tags_to_table(dynamo_db_table, table_arn, tags) + restore_tags_to_table(dynamo_db_table) def insert_into_dynamo(data): logger.debug("Inserting into Dynamo: %s", data) - table = DynamoDBHelper(DYNAMODB_TABLE_NAME) + dynamodb_table_name = os.getenv("DYNAMODB_TABLE_NAME") + table = DynamoDBHelper(dynamodb_table_name) for item in data: try: table.insert_item(item) From 8a21103cc4a3c5fca2d34c22efaf0a63a228b8d5 Mon Sep 17 00:00:00 2001 From: SeanSteberis <103416906+seansteberisal@users.noreply.github.com> Date: Sun, 5 Oct 2025 00:13:52 +0100 Subject: [PATCH 4/6] DynamoDB reset now more resilient to changes on the fly --- utils/dynamo_helper.py | 88 ++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 55 deletions(-) diff --git a/utils/dynamo_helper.py b/utils/dynamo_helper.py index bc55f7a5..04ef1617 100644 --- a/utils/dynamo_helper.py +++ b/utils/dynamo_helper.py @@ -90,14 +90,18 @@ def describe_table(self): raise self.table_arn = table_description["TableArn"] save_to_file("table_arn.json", self.table_arn, directory="data/dynamoDB/temp") - self.attribute_definitions = table_description["attribute_definitions"] + self.attribute_definitions = table_description["AttributeDefinitions"] save_to_file( "attribute_definitions.json", - self.attribute_definitions, + json.dumps(self.attribute_definitions), + directory="data/dynamoDB/temp", + ) + self.key_schema = table_description["KeySchema"] + save_to_file( + "key_schema.json", + json.dumps(self.key_schema), directory="data/dynamoDB/temp", ) - self.key_schema = table_description["key_schema"] - save_to_file("key_schema.json", self.key_schema, directory="data/dynamoDB/temp") return self.table_arn, self.attribute_definitions, self.key_schema @@ -106,7 +110,7 @@ def get_table_tags(self): ResourceArn=self.table_arn )["Tags"] logger.debug(f"tags: {self.tags}") - save_to_file("tags.json", self.tags, directory="data/dynamoDB/temp") + save_to_file("tags.json", json.dumps(self.tags), directory="data/dynamoDB/temp") return self.tags def set_table_tags(self): @@ -146,50 +150,23 @@ def delete_table(self, table_name): def restore_tags_to_table(dynamo_db_table: DynamoDBHelper): - if dynamo_db_table.table_arn is None: - logger.warning( - "Unable to find TableArn, attempting to load from backup file..." - ) - dynamo_db_table.table_arn = json.loads( - load_from_file("data/dynamoDB/temp/table_arn.json") - ) + if dynamo_db_table.table_arn is None or dynamo_db_table.tags is None: logger.warning( - f"TableArn loaded from backup file: : {dynamo_db_table.table_arn}" + "Unable to find TableArn or Tags, attempting to load from backup files..." ) - if dynamo_db_table.tags is None: - logger.warning("Unable to add tags, attempting to load from backup file...") try: - dynamo_db_table.tags = json.loads( - load_from_file("data/dynamoDB/temp/tags.json") - ) - logger.warning(f"tags loaded from backup file: {dynamo_db_table.tags}") - - logger.debug(f"TableArn: {dynamo_db_table.table_arn}") - dynamo_db_table.set_table_tags() + load_information_from_backup_files(dynamo_db_table) except FileNotFoundError: logger.error("Failed to restore tags and no backup file was found.") - - -def get_attribute_definitions_and_key_schema_from_file( - attribute_definitions: str | int | bytes, key_schema: str | int | bytes -) -> tuple[str | int | bytes, str | int | bytes]: - table_description = load_from_file("data/dynamoDB/temp/description.json") - key_schema = table_description["KeySchema"] - logger.warning(f"KeySchema loaded from backup file: {key_schema}") - - attribute_definitions = table_description["AttributeDefinitions"] - logger.warning( - f"attribute_definitions loaded from backup file: {attribute_definitions}" - ) - return attribute_definitions, key_schema + dynamo_db_table.set_table_tags() def file_backup_exists(): try: - load_from_file("data/dynamoDB/temp/tags.json") - load_from_file("data/dynamoDB/temp/attributeDefinition.json") - load_from_file("data/dynamoDB/temp/keySchema.json") - load_from_file("data/dynamoDB/temp/tableArn.json") + json.loads(load_from_file("data/dynamoDB/temp/tags.json")) + json.loads(load_from_file("data/dynamoDB/temp/attribute_definitions.json")) + json.loads(load_from_file("data/dynamoDB/temp/key_schema.json")) + load_from_file("data/dynamoDB/temp/table_arn.json") return True except FileNotFoundError: return False @@ -197,7 +174,7 @@ def file_backup_exists(): def reset_dynamo_tables(): environment = os.getenv("ENVIRONMENT") - dynamodb_table_name = os.getenv("DYNAMODB_TABLE_NAME") + table_name = os.getenv("DYNAMODB_TABLE_NAME") logger.info("Resetting DynamoDB. This may take a few moments, please be patient.") if environment not in ["dev", "test"]: @@ -205,8 +182,7 @@ def reset_dynamo_tables(): f"{environment} is not supported. Resetting DynamoDB is only supported in dev or test." ) return - dynamo_db_table = DynamoDBHelper(dynamodb_table_name) - table_name = dynamodb_table_name + dynamo_db_table = DynamoDBHelper(table_name) # --- Step 1: Fetch table information --- try: @@ -219,22 +195,12 @@ def reset_dynamo_tables(): except ClientError as e: logger.warning(f"Error describing table: {e}") if not file_backup_exists(): - logger.exception( + logger.error( f"FATAL! Unable to get table information and no backup present: {e}" ) raise e else: - logger.warning("Table information taken from backup files") - dynamo_db_table.tags = load_from_file("data/dynamoDB/temp/tags.json") - dynamo_db_table.attribute_definitions = load_from_file( - "data/dynamoDB/temp/attributeDefinition.json" - ) - dynamo_db_table.key_schema = load_from_file( - "data/dynamoDB/temp/keySchema.json" - ) - dynamo_db_table.table_arn = load_from_file( - "data/dynamoDB/temp/tableArn.json" - ) + load_information_from_backup_files(dynamo_db_table) # --- Step 2: Delete the table --- try: @@ -258,6 +224,18 @@ def reset_dynamo_tables(): restore_tags_to_table(dynamo_db_table) +def load_information_from_backup_files(dynamo_db_table: DynamoDBHelper): + logger.warning("Table information taken from backup files") + dynamo_db_table.tags = json.loads(load_from_file("data/dynamoDB/temp/tags.json")) + dynamo_db_table.attribute_definitions = json.loads( + load_from_file("data/dynamoDB/temp/attribute_definitions.json") + ) + dynamo_db_table.key_schema = json.loads( + load_from_file("data/dynamoDB/temp/key_schema.json") + ) + dynamo_db_table.table_arn = load_from_file("data/dynamoDB/temp/table_arn.json") + + def insert_into_dynamo(data): logger.debug("Inserting into Dynamo: %s", data) dynamodb_table_name = os.getenv("DYNAMODB_TABLE_NAME") From aaeb5340fead9ac4e6c8df7c18549dac5e042ea6 Mon Sep 17 00:00:00 2001 From: SeanSteberis <103416906+seansteberisal@users.noreply.github.com> Date: Mon, 6 Oct 2025 10:35:55 +0100 Subject: [PATCH 5/6] add CODEOWNERS --- .github/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..62b0385f --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# Default owners for everything in the repo - unless a later match takes precedence +* @NHSDigital/ELID-Regression-Tests-Code-Owners From 9d865f59ca54bcc3d53648b50e6cd2c255596141 Mon Sep 17 00:00:00 2001 From: SeanSteberis <103416906+seansteberisal@users.noreply.github.com> Date: Tue, 7 Oct 2025 11:38:24 +0100 Subject: [PATCH 6/6] dynamodb temp folder to variable --- .gitignore | 2 +- utils/dynamo_helper.py | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index ee8d5fc6..5a4fde7e 100644 --- a/.gitignore +++ b/.gitignore @@ -255,4 +255,4 @@ vscode # Node Modules node_modules /certs/ -/data/dynamoDB/temp/ +*temp* diff --git a/utils/dynamo_helper.py b/utils/dynamo_helper.py index 04ef1617..0b40ae80 100644 --- a/utils/dynamo_helper.py +++ b/utils/dynamo_helper.py @@ -9,6 +9,7 @@ logger = logging.getLogger(__name__) load_dotenv() +dynamo_temp_location = "data/dynamoDB/temp/" class DynamoDBHelper: @@ -163,10 +164,10 @@ def restore_tags_to_table(dynamo_db_table: DynamoDBHelper): def file_backup_exists(): try: - json.loads(load_from_file("data/dynamoDB/temp/tags.json")) - json.loads(load_from_file("data/dynamoDB/temp/attribute_definitions.json")) - json.loads(load_from_file("data/dynamoDB/temp/key_schema.json")) - load_from_file("data/dynamoDB/temp/table_arn.json") + json.loads(load_from_file(f"{dynamo_temp_location}tags.json")) + json.loads(load_from_file(f"{dynamo_temp_location}attribute_definitions.json")) + json.loads(load_from_file(f"{dynamo_temp_location}key_schema.json")) + load_from_file(f"{dynamo_temp_location}table_arn.json") return True except FileNotFoundError: return False @@ -226,14 +227,16 @@ def reset_dynamo_tables(): def load_information_from_backup_files(dynamo_db_table: DynamoDBHelper): logger.warning("Table information taken from backup files") - dynamo_db_table.tags = json.loads(load_from_file("data/dynamoDB/temp/tags.json")) + dynamo_db_table.tags = json.loads( + load_from_file(f"{dynamo_temp_location}tags.json") + ) dynamo_db_table.attribute_definitions = json.loads( - load_from_file("data/dynamoDB/temp/attribute_definitions.json") + load_from_file(f"{dynamo_temp_location}attribute_definitions.json") ) dynamo_db_table.key_schema = json.loads( - load_from_file("data/dynamoDB/temp/key_schema.json") + load_from_file(f"{dynamo_temp_location}key_schema.json") ) - dynamo_db_table.table_arn = load_from_file("data/dynamoDB/temp/table_arn.json") + dynamo_db_table.table_arn = load_from_file(f"{dynamo_temp_location}table_arn.json") def insert_into_dynamo(data):