Merge pull request #116 from NHSDigital/feature/terb-ELI-275-manual-uploads

Feature/terb eli 275 manual uploads

Author: robbailiff2

poetry.lock

@@ -56,6 +56,7 @@ localstack = "^4.1.1"
 pytest-docker = "^3.2.0"
 stamina = "^25.1.0"
 pytest-freezer = "^0.4.9"
+moto = "^5.1.5"
 
 [tool.poetry-plugin-lambda-build]
 docker-image = "public.ecr.aws/sam/build-python3.13:1.139-x86_64"  # See https://gallery.ecr.aws/search?searchTerm=%22python%22&architecture=x86-64&popularRegistries=amazon&verified=verified&operatingSystems=Linux

pyproject.toml

@@ -0,0 +1,76 @@
+#!/bin/bash
+
+# === Config ===
+
+BUCKET_NAME="eligibility-signposting-api-dev-eli-rules"
+S3_PREFIX="manual-uploads"
+
+# === Usage Info ===
+# make it executable using - chmod +x manual_config_upload.sh
+
+if [ "$#" -lt 1 ]; then
+  echo "Usage: $0 <file.json> [bucket_name]"
+  echo
+  echo "Before running this script, you should either:"
+  echo "  • Run 'aws configure'"
+  echo "  • OR export these variables:"
+  echo "      export AWS_ACCESS_KEY_ID=..."
+  echo "      export AWS_SECRET_ACCESS_KEY=..."
+  echo "      export AWS_SESSION_TOKEN=...  # if using temporary credentials"
+  echo
+  exit 1
+fi
+FILE="$1"
+BUCKET="${2:-$BUCKET_NAME}"
+
+# === Check dependencies ===
+
+if ! command -v jq >/dev/null 2>&1; then
+  echo " 'jq' is not installed. Please install it (e.g., sudo apt install jq)"
+  exit 1
+fi
+
+# === Validate JSON ===
+
+if ! jq empty "$FILE" >/dev/null 2>&1; then
+  echo " Invalid JSON in file: $FILE"
+  exit 1
+fi
+
+# === Prompt for AWS credentials if not set ===
+
+if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ]; then
+  echo "AWS credentials not found in environment. Let's set them now:"
+  read -p "Enter AWS_ACCESS_KEY_ID: " AWS_ACCESS_KEY_ID
+  read -s -p "Enter AWS_SECRET_ACCESS_KEY: " AWS_SECRET_ACCESS_KEY
+  echo
+  read -s -p "Enter AWS_SESSION_TOKEN (leave blank if not needed): " AWS_SESSION_TOKEN
+  echo  export AWS_ACCESS_KEY_ID
+  export AWS_SECRET_ACCESS_KEY
+  export AWS_SESSION_TOKEN
+fi
+
+# === Confirm credentials work ===
+
+aws sts get-caller-identity >/dev/null 2>&1
+if [ $? -ne 0 ]; then
+  echo "Failed to authenticate with AWS. Please check your credentials."
+  exit 1
+
+fi
+
+# === Create unique key ===
+
+BASENAME=$(basename "$FILE" .json)
+S3_KEY="${S3_PREFIX}/${BASENAME}.json"
+
+# === Upload ===
+
+echo " JSON is valid. Uploading to s3://$BUCKET/$S3_KEY ..."
+aws s3 cp "$FILE" "s3://$BUCKET/$S3_KEY" --content-type "application/json"
+
+if [ $? -eq 0 ]; then
+  echo " Upload complete."
+else
+  echo "Upload failed."
+fi

scripts/manual_uploads/manual_config_upload.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+set -e
+
+# Start LocalStack (assumes localstack is installed and in path)
+echo "Starting LocalStack for S3..."
+localstack start -d
+sleep 5
+
+export AWS_ACCESS_KEY_ID=test
+export AWS_SECRET_ACCESS_KEY=test
+export AWS_DEFAULT_REGION=us-east-1
+export AWS_ENDPOINT_URL=http://localhost:4566
+
+# Create mock S3 bucket
+aws --endpoint-url=$AWS_ENDPOINT_URL s3 mb s3://test-bucket
+
+# Create test file
+echo '{ "test": "value" }' > testfile.json
+
+# Run the script
+echo "Testing manual_config_upload.sh..."
+./manual_config_upload.sh testfile.json test-bucket
+
+# List contents
+echo "Contents of bucket:"
+aws --endpoint-url=$AWS_ENDPOINT_URL s3 ls s3://test-bucket/manual-uploads/
+
+# Cleanup
+rm testfile.json
+localstack stop

scripts/manual_uploads/manual_config_upload_test.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+# === Config ===
+DEFAULT_TABLE="eligibility_data_store"
+REGION="eu-west-2"
+AWS_ENDPOINT_URL="${AWS_ENDPOINT_URL:-https://dynamodb.${REGION}.amazonaws.com}"
+
+# === Usage Check ===
+
+if [ "$#" -lt 1 ]; then
+  echo "Usage: $0 <item.json> [table_name]"
+  echo
+  echo "Before running this, make sure AWS credentials are set either via:"
+  echo "  • 'aws configure'"
+  echo "  • OR by exporting:"
+  echo "      export AWS_ACCESS_KEY_ID=..."
+  echo "      export AWS_SECRET_ACCESS_KEY=..."
+  echo "      export AWS_SESSION_TOKEN=..."
+  exit 1
+fi
+
+FILE="$1"
+TABLE="${2:-$DEFAULT_TABLE}"
+
+# === Validate JSON ===
+
+if ! jq empty "$FILE" >/dev/null 2>&1; then
+  echo "Invalid JSON in file: $FILE"
+  exit 1
+fi
+
+# === Prompt for AWS credentials if not set ===
+
+if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ]; then
+  echo "AWS credentials not found. Let's set them now:"
+  read -p "Enter AWS_ACCESS_KEY_ID: " AWS_ACCESS_KEY_ID
+  read -s -p "Enter AWS_SECRET_ACCESS_KEY: " AWS_SECRET_ACCESS_KEY
+  echo
+  read -s -p "Enter AWS_SESSION_TOKEN (leave blank if not needed): " AWS_SESSION_TOKEN
+  echo
+
+  export AWS_ACCESS_KEY_ID
+  export AWS_SECRET_ACCESS_KEY
+  export AWS_SESSION_TOKEN
+fi
+
+# === Check AWS auth ===
+
+aws sts get-caller-identity >/dev/null 2>&1
+if [ $? -ne 0 ]; then
+  echo "Failed to authenticate with AWS. Check credentials."
+  exit 1
+fi
+
+# === Upload to DynamoDB ===
+
+echo "Uploading item from $FILE to table $TABLE ..."
+aws dynamodb put-item \
+  --table-name "$TABLE" \
+  --item "file://$FILE" \
+  --region "$REGION" \
+  --endpoint-url "$AWS_ENDPOINT_URL"
+
+if [ $? -eq 0 ]; then
+  echo "Upload complete."
+else
+  echo "Upload failed."
+fi

scripts/manual_uploads/manual_dynamo_upload.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+set -e
+
+# Start LocalStack (assumes localstack is installed and in path)
+echo "Starting LocalStack for DynamoDB..."
+localstack start -d
+sleep 5
+
+export AWS_ACCESS_KEY_ID=test
+export AWS_SECRET_ACCESS_KEY=test
+export AWS_DEFAULT_REGION=eu-west-2
+export AWS_ENDPOINT_URL=http://localhost:4566
+
+# Create DynamoDB table
+aws --endpoint-url=$AWS_ENDPOINT_URL dynamodb create-table \
+  --table-name test-table \
+  --attribute-definitions AttributeName=UserId,AttributeType=S \
+  --key-schema AttributeName=UserId,KeyType=HASH \
+  --billing-mode PAY_PER_REQUEST
+
+# Create test item in DynamoDB JSON format
+cat <<EOF > testitem.json
+{
+  "UserId": { "S": "123" },
+  "Email": { "S": "localstack@test.com" },
+  "Score": { "N": "100" }
+}
+EOF
+
+# Run the script
+echo "Testing manual_dynamo_upload.sh..."
+./manual_dynamo_upload.sh testitem.json test-table
+
+# Query item
+echo "Querying DynamoDB table:"
+aws --endpoint-url=$AWS_ENDPOINT_URL dynamodb get-item \
+  --table-name test-table \
+  --key '{ "UserId": { "S": "123" } }'
+
+# Cleanup
+rm testitem.json
+localstack stop

scripts/manual_uploads/manual_dynamo_upload_test.sh

@@ -0,0 +1,125 @@
+import boto3
+import json
+import os
+import argparse
+import logging
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Union, Generator
+from decimal import Decimal
+
+
+def map_dynamo_type(value: Any) -> Dict[str, Any]:
+    if isinstance(value, str):
+        return {"S": value}
+    elif isinstance(value, bool):
+        return {"BOOL": value}
+    elif isinstance(value, (int, float, Decimal)):
+        return {"N": str(value)}
+    elif value is None:
+        return {"NULL": True}
+    elif isinstance(value, list):
+        return {"L": [map_dynamo_type(item) for item in value]}
+    elif isinstance(value, dict):
+        return {"M": {k: map_dynamo_type(v) for k, v in value.items()}}
+    else:
+        logging.warning(f"Unsupported value type: {type(value)}", "Converting it to string")
+        return {"S": str(value)}
+
+
+def load_json_lines(filepath: Union[str, Path]) -> Generator[Dict[str, Any], None, None]:
+    with Path.open(filepath) as f:
+        for line in f:
+            if line.strip():
+                yield json.loads(line)
+
+
+def upload_to_s3(
+    s3_client: Any,
+    bucket: str,
+    filepath: Union[str, Path],
+    dry_run: bool = False
+) -> None:
+
+    filename = os.path.basename(filepath)
+    s3_key = f"manual-uploads/{filename}"
+
+    if dry_run:
+        print(f"[DRY RUN] Would upload {filepath} to s3://{bucket}/{s3_key}")
+        return
+
+    try:
+        s3_client.upload_file(filepath, bucket, s3_key)
+        print(f"Uploaded {filepath} to s3://{bucket}/{s3_key}")
+    except Exception as e:
+        print(f"Failed to upload {filepath}: {e}")
+
+
+def upload_to_dynamo(
+    dynamo_client: Any,
+    table_name: str,
+    filepath: Union[str, Path],
+) -> None:
+    uploaded_items = 0
+    for item in load_json_lines(filepath):
+        try:
+            dynamo_client.put_item(
+                TableName=table_name, Item={key: map_dynamo_type(value) for key, value in item.items()}
+            )
+            uploaded_items += 1
+        except Exception as e:
+            partition_key = item.get("NHS_NUMBER", "Unknown")
+            sort_key = item.get("ATTRIBUTE_TYPE", "Unknown")
+            print(f"Failed to upload item (NHS_NUMBER: {partition_key}, ATTRIBUTE_TYPE: {sort_key}) from {filepath}: {e}")
+
+    if uploaded_items > 0:
+        print(f"Uploaded {uploaded_items} items from {filepath} to DynamoDB table {table_name}")
+
+def run_upload(args: Optional[List[str]] = None) -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--env")
+    parser.add_argument("--upload-s3", type=Path)
+    parser.add_argument("--upload-dynamo", type=Path)
+    parser.add_argument("--region", default="eu-west-2")
+    parser.add_argument("--s3-bucket")
+    parser.add_argument("--dynamo-table")
+    parser.add_argument("--dry-run", action="store_true")
+
+    if args is None:
+        parsed_args = parser.parse_args()
+    else:
+        parsed_args = parser.parse_args(args)
+
+    if not parsed_args.upload_s3 and not parsed_args.upload_dynamo:
+        logging.warning("Neither '--upload-s3' nor '--upload-dynamo' flags specified. No upload actions will be performed.")
+    if not parsed_args.s3_bucket:
+        parsed_args.s3_bucket = f"eligibility-signposting-api-{parsed_args.env}-eli-rules"
+    if not parsed_args.dynamo_table:
+        parsed_args.dynamo_table = f"eligibility-signposting-api-{parsed_args.env}-eligibility_datastore"
+
+    session = boto3.Session()
+    s3 = session.client("s3", region_name=parsed_args.region)
+    dynamo = session.client("dynamodb", region_name=parsed_args.region)
+
+    if parsed_args.upload_s3:
+        if parsed_args.upload_s3.is_dir():
+            files = parsed_args.upload_s3.glob("*.json")
+        else:
+            files = [parsed_args.upload_s3]
+
+        for filepath in files:
+            print(f"Uploading to S3 from {filepath}")
+            upload_to_s3(s3, parsed_args.s3_bucket, str(filepath), parsed_args.dry_run)
+
+    if parsed_args.upload_dynamo:
+        if parsed_args.upload_dynamo.is_dir():
+            paths = parsed_args.upload_dynamo.glob("*.json")
+        else:
+            paths = [parsed_args.upload_dynamo]
+
+        for filepath in paths:
+            print(f"Uploading to DynamoDB from {filepath}")
+            upload_to_dynamo(dynamo, parsed_args.dynamo_table, str(filepath))
+
+
+if __name__ == "__main__":
+    run_upload()