eli-599/replacing localstack with moto for local lambda IT testing (#582)

* localstack

* localstack

* localstack

* ELI-662: working sam

* ELI-662: working sam

* AWS RIE for lambda and moto for other services, Cloud watch cannot be configured for testing

* moved lambda client to lambda conftest

* use docker compose profiling

* lint fixes

* docker clean up

* docker clean up

* test - commit

* test - commit

* reduced subprocess usage

* lint fix

* fix services trigger

* simulate API Gateway requests in python

Author: Karthikeyannhs

poetry.lock

@@ -52,7 +52,6 @@ awscli-local = "^0.22.2"
 polyfactory = "^3.2.0"
 pyright = "^1.1.407"
 brunns-matchers = "^2.9.0"
-localstack = "^4.12.0"
 pytest-docker = "^3.2.3"
 stamina = "^25.2.0"
 pytest-freezer = "^0.4.9"
@@ -63,7 +62,7 @@ behave = "^1.3.3"
 python-dotenv = "^1.2.1"
 openapi-spec-validator = "^0.7.2"
 pip-licenses = "^5.5.0"
-
+cachetools = "^7.0.1"
 
 [tool.poetry-plugin-lambda-build]
 docker-image = "public.ecr.aws/sam/build-python3.13:1.139-x86_64"  # See https://gallery.ecr.aws/search?searchTerm=%22python%22&architecture=x86-64&popularRegistries=amazon&verified=verified&operatingSystems=Linux

pyproject.toml

@@ -26,7 +26,7 @@ def config() -> dict[str, Any]:
     audit_bucket_name = BucketName(os.getenv("AUDIT_BUCKET_NAME", "test-audit-bucket"))
     hashing_secret_name = HashSecretName(os.getenv("HASHING_SECRET_NAME", "test_secret"))
     aws_default_region = AwsRegion(os.getenv("AWS_DEFAULT_REGION", "eu-west-1"))
-    enable_xray_patching = bool(os.getenv("ENABLE_XRAY_PATCHING", "false"))
+    enable_xray_patching = os.getenv("ENABLE_XRAY_PATCHING", "false").lower() == "true"
     kinesis_audit_stream_to_s3 = AwsKinesisFirehoseStreamName(
         os.getenv("KINESIS_AUDIT_STREAM_TO_S3", "test_kinesis_audit_stream_to_s3")
     )
@@ -51,21 +51,21 @@ def config() -> dict[str, Any]:
             "log_level": log_level,
         }
 
-    local_stack_endpoint = "http://localhost:4566"
+    moto_server_endpoint = "http://localhost:4566"
     return {
         "aws_access_key_id": AwsAccessKey(os.getenv("AWS_ACCESS_KEY_ID", "dummy_key")),
         "aws_default_region": aws_default_region,
         "aws_secret_access_key": AwsSecretAccessKey(os.getenv("AWS_SECRET_ACCESS_KEY", "dummy_secret")),
-        "dynamodb_endpoint": URL(os.getenv("DYNAMODB_ENDPOINT", local_stack_endpoint)),
+        "dynamodb_endpoint": URL(os.getenv("DYNAMODB_ENDPOINT", moto_server_endpoint)),
         "person_table_name": person_table_name,
-        "s3_endpoint": URL(os.getenv("S3_ENDPOINT", local_stack_endpoint)),
+        "s3_endpoint": URL(os.getenv("S3_ENDPOINT", moto_server_endpoint)),
         "rules_bucket_name": rules_bucket_name,
         "audit_bucket_name": audit_bucket_name,
         "consumer_mapping_bucket_name": consumer_mapping_bucket_name,
-        "firehose_endpoint": URL(os.getenv("FIREHOSE_ENDPOINT", local_stack_endpoint)),
+        "firehose_endpoint": URL(os.getenv("FIREHOSE_ENDPOINT", moto_server_endpoint)),
         "kinesis_audit_stream_to_s3": kinesis_audit_stream_to_s3,
         "enable_xray_patching": enable_xray_patching,
-        "secretsmanager_endpoint": URL(os.getenv("SECRET_MANAGER_ENDPOINT", local_stack_endpoint)),
+        "secretsmanager_endpoint": URL(os.getenv("SECRET_MANAGER_ENDPOINT", moto_server_endpoint)),
         "hashing_secret_name": hashing_secret_name,
         "log_level": log_level,
     }

src/eligibility_signposting_api/config/config.py

@@ -0,0 +1,50 @@
+services:
+  moto-server:
+    #used for s3, dynamodb, kinesis, secret manager
+    # lambda cannot be used, because its 3.11 (older)
+    image: motoserver/moto:latest
+    container_name: moto-server
+    ports:
+      - "4566:5000"
+    networks:
+      - test-network
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:5000/" ]
+      interval: 1s
+      timeout: 1s
+      retries: 30
+  lambda-api:
+    image: public.ecr.aws/lambda/python:3.13
+    container_name: lambda-api
+    ports:
+      - "4567:8080"
+    platform: linux/amd64
+    volumes:
+      - ../dist/lambda.zip:/tmp/lambda.zip:ro
+    environment:
+      - AWS_ACCESS_KEY_ID=dummy_key
+      - AWS_SECRET_ACCESS_KEY=dummy_secret
+      - AWS_DEFAULT_REGION=eu-west-1
+      - PYTHONPATH=/var/task
+      - AWS_ENDPOINT_URL=http://moto-server:5000
+      - DYNAMODB_ENDPOINT=http://moto-server:5000
+      - S3_ENDPOINT=http://moto-server:5000
+      - SECRET_MANAGER_ENDPOINT=http://moto-server:5000
+      - FIREHOSE_ENDPOINT=http://moto-server:5000
+      - LOG_LEVEL=INFO
+    entrypoint: /bin/sh
+    command:
+      - "-c"
+      - |
+        mkdir -p /var/task &&
+        python3 -m zipfile -e /tmp/lambda.zip /var/task &&
+        exec /usr/local/bin/aws-lambda-rie python3 -m awslambdaric eligibility_signposting_api.app.lambda_handler
+    networks:
+      - test-network
+    depends_on:
+      moto-server:
+        condition: service_healthy
+
+networks:
+  test-network:
+    driver: bridge