From 30ade8d57a75387becda843615502bc8c98fec5e Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Thu, 24 Jul 2025 15:59:07 +0100 Subject: [PATCH 1/4] handled none headers from request --- src/eligibility_signposting_api/logging/logs_helper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/eligibility_signposting_api/logging/logs_helper.py b/src/eligibility_signposting_api/logging/logs_helper.py index cb71bc911..6b383e8ab 100644 --- a/src/eligibility_signposting_api/logging/logs_helper.py +++ b/src/eligibility_signposting_api/logging/logs_helper.py @@ -12,8 +12,8 @@ def log_request_ids() -> Callable: def decorator(func: Callable) -> Callable: @wraps(func) def wrapper(event: LambdaEvent, context: LambdaContext) -> dict[str, Any] | None: - gateway_request_id = event.get("requestContext", {}).get("requestId") - headers = event.get("headers", {}) + gateway_request_id = (event.get("requestContext") or {}).get("requestId") + headers = event.get("headers") or {} logger.info( "request trace metadata", extra={ From f54324e8551f8b697f8022dece249a8d5f11b314 Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Fri, 25 Jul 2025 16:09:41 +0100 Subject: [PATCH 2/4] x-ray tracing setup for dynamo, s3, firehose --- poetry.lock | 25 ++++++++++++++----- pyproject.toml | 1 + src/eligibility_signposting_api/app.py | 15 ++++++++--- .../config/config.py | 3 +++ .../logging/logs_helper.py | 2 +- .../logging/logs_manager.py | 2 +- .../logging/tracing_helper.py | 21 ++++++++++++++++ tests/unit/logging/test_logs_helper.py | 4 +-- tests/unit/logging/test_logs_manager.py | 10 ++++---- 9 files changed, 64 insertions(+), 19 deletions(-) create mode 100644 src/eligibility_signposting_api/logging/tracing_helper.py diff --git a/poetry.lock b/poetry.lock index 405be8852..ed37ac0d5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -218,6 +218,22 @@ files = [ [package.dependencies] cryptography = "*" +[[package]] +name = "aws-xray-sdk" +version = "2.14.0" +description = "The AWS X-Ray SDK for Python (the SDK) enables Python developers to record and emit information from within their applications to the AWS X-Ray service." +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "aws_xray_sdk-2.14.0-py2.py3-none-any.whl", hash = "sha256:cfbe6feea3d26613a2a869d14c9246a844285c97087ad8f296f901633554ad94"}, + {file = "aws_xray_sdk-2.14.0.tar.gz", hash = "sha256:aab843c331af9ab9ba5cefb3a303832a19db186140894a523edafc024cc0493c"}, +] + +[package.dependencies] +botocore = ">=1.11.3" +wrapt = "*" + [[package]] name = "awscli" version = "1.40.41" @@ -1531,11 +1547,8 @@ files = [ {file = "lxml-5.4.0-cp36-cp36m-win_amd64.whl", hash = "sha256:7ce1a171ec325192c6a636b64c94418e71a1964f56d002cc28122fceff0b6121"}, {file = "lxml-5.4.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:795f61bcaf8770e1b37eec24edf9771b307df3af74d1d6f27d812e15a9ff3872"}, {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:29f451a4b614a7b5b6c2e043d7b64a15bd8304d7e767055e8ab68387a8cacf4e"}, - {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:891f7f991a68d20c75cb13c5c9142b2a3f9eb161f1f12a9489c82172d1f133c0"}, {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4aa412a82e460571fad592d0f93ce9935a20090029ba08eca05c614f99b0cc92"}, - {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:ac7ba71f9561cd7d7b55e1ea5511543c0282e2b6450f122672a2694621d63b7e"}, {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:c5d32f5284012deaccd37da1e2cd42f081feaa76981f0eaa474351b68df813c5"}, - {file = "lxml-5.4.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:ce31158630a6ac85bddd6b830cffd46085ff90498b397bd0a259f59d27a12188"}, {file = "lxml-5.4.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:31e63621e073e04697c1b2d23fcb89991790eef370ec37ce4d5d469f40924ed6"}, {file = "lxml-5.4.0-cp37-cp37m-win32.whl", hash = "sha256:be2ba4c3c5b7900246a8f866580700ef0d538f2ca32535e991027bdaba944063"}, {file = "lxml-5.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:09846782b1ef650b321484ad429217f5154da4d6e786636c38e434fa32e94e49"}, @@ -3261,7 +3274,7 @@ version = "1.17.2" description = "Module for decorators, wrappers and monkey patching." optional = false python-versions = ">=3.8" -groups = ["dev"] +groups = ["main", "dev"] files = [ {file = "wrapt-1.17.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3d57c572081fed831ad2d26fd430d565b76aa277ed1d30ff4d40670b1c0dd984"}, {file = "wrapt-1.17.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5e251054542ae57ac7f3fba5d10bfff615b6c2fb09abeb37d2f1463f841ae22"}, @@ -3478,4 +3491,4 @@ propcache = ">=0.2.1" [metadata] lock-version = "2.1" python-versions = "^3.13" -content-hash = "ca918c8b9441327adf6c176055f47d8d3f9e058aa243c1c957e0f7a6a6e4159f" +content-hash = "e7dab798823725076f2001cb892f67cd73269205120bb323d75d3c9d755c1bb2" diff --git a/pyproject.toml b/pyproject.toml index 28d9d5b01..c2d21e645 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ python-json-logger = "^3.3.0" fhir-resources = "^8.0.0" python-dateutil = "^2.9.0" pyhamcrest = "^2.1.0" +aws-xray-sdk = "2.14.0" [tool.poetry.group.dev.dependencies] ruff = "^0.11.13" diff --git a/src/eligibility_signposting_api/app.py b/src/eligibility_signposting_api/app.py index a8772c855..076248bf5 100644 --- a/src/eligibility_signposting_api/app.py +++ b/src/eligibility_signposting_api/app.py @@ -1,8 +1,10 @@ import logging +import os from typing import Any import wireup.integration.flask from asgiref.wsgi import WsgiToAsgi +from aws_xray_sdk.core import patch_all from flask import Flask from mangum import Mangum from mangum.types import LambdaContext, LambdaEvent @@ -11,10 +13,14 @@ from eligibility_signposting_api.common.error_handler import handle_exception from eligibility_signposting_api.common.request_validator import validate_request_params from eligibility_signposting_api.config.config import config -from eligibility_signposting_api.logging.logs_helper import log_request_ids -from eligibility_signposting_api.logging.logs_manager import add_request_id_to_logs, init_logging +from eligibility_signposting_api.logging.logs_helper import log_request_ids_from_headers +from eligibility_signposting_api.logging.logs_manager import add_lambda_request_id_to_logger, init_logging +from eligibility_signposting_api.logging.tracing_helper import tracing_setup from eligibility_signposting_api.views import eligibility_blueprint +if os.getenv("ENABLE_XRAY_PATCHING", "false") == "true": + patch_all() + init_logging() logger = logging.getLogger(__name__) @@ -25,8 +31,9 @@ def main() -> None: # pragma: no cover app.run(debug=config()["log_level"] == logging.DEBUG) -@add_request_id_to_logs() -@log_request_ids() +@add_lambda_request_id_to_logger() +@tracing_setup() +@log_request_ids_from_headers() @validate_request_params() def lambda_handler(event: LambdaEvent, context: LambdaContext) -> dict[str, Any]: # pragma: no cover """Run the Flask app as an AWS Lambda.""" diff --git a/src/eligibility_signposting_api/config/config.py b/src/eligibility_signposting_api/config/config.py index 49faeff6b..914738ce0 100644 --- a/src/eligibility_signposting_api/config/config.py +++ b/src/eligibility_signposting_api/config/config.py @@ -22,6 +22,7 @@ def config() -> dict[str, Any]: rules_bucket_name = BucketName(os.getenv("RULES_BUCKET_NAME", "test-rules-bucket")) audit_bucket_name = BucketName(os.getenv("AUDIT_BUCKET_NAME", "test-audit-bucket")) aws_default_region = AwsRegion(os.getenv("AWS_DEFAULT_REGION", "eu-west-1")) + enable_xray_patching = bool(os.getenv("ENABLE_XRAY_PATCHING", "false") == "true") kinesis_audit_stream_to_s3 = AwsKinesisFirehoseStreamName( os.getenv("KINESIS_AUDIT_STREAM_TO_S3", "test_kinesis_audit_stream_to_s3") ) @@ -39,6 +40,7 @@ def config() -> dict[str, Any]: "audit_bucket_name": audit_bucket_name, "firehose_endpoint": None, "kinesis_audit_stream_to_s3": kinesis_audit_stream_to_s3, + "enable_xray_patching": enable_xray_patching, "log_level": log_level, } @@ -53,5 +55,6 @@ def config() -> dict[str, Any]: "audit_bucket_name": audit_bucket_name, "firehose_endpoint": URL(os.getenv("FIREHOSE_ENDPOINT", "http://localhost:4566")), "kinesis_audit_stream_to_s3": kinesis_audit_stream_to_s3, + "enable_xray_patching": enable_xray_patching, "log_level": log_level, } diff --git a/src/eligibility_signposting_api/logging/logs_helper.py b/src/eligibility_signposting_api/logging/logs_helper.py index 6b383e8ab..12d8a48db 100644 --- a/src/eligibility_signposting_api/logging/logs_helper.py +++ b/src/eligibility_signposting_api/logging/logs_helper.py @@ -8,7 +8,7 @@ logger = logging.getLogger(__name__) -def log_request_ids() -> Callable: +def log_request_ids_from_headers() -> Callable: def decorator(func: Callable) -> Callable: @wraps(func) def wrapper(event: LambdaEvent, context: LambdaContext) -> dict[str, Any] | None: diff --git a/src/eligibility_signposting_api/logging/logs_manager.py b/src/eligibility_signposting_api/logging/logs_manager.py index 093c06dda..6ca253ef2 100644 --- a/src/eligibility_signposting_api/logging/logs_manager.py +++ b/src/eligibility_signposting_api/logging/logs_manager.py @@ -14,7 +14,7 @@ LOG_FORMAT = "%(asctime)s %(levelname)-8s %(name)s %(module)s.py:%(funcName)s():%(lineno)d %(message)s" -def add_request_id_to_logs() -> Callable: +def add_lambda_request_id_to_logger() -> Callable: def decorator(func: Callable) -> Callable: @wraps(func) def wrapper(event: LambdaEvent, context: LambdaContext) -> dict[str, Any] | None: diff --git a/src/eligibility_signposting_api/logging/tracing_helper.py b/src/eligibility_signposting_api/logging/tracing_helper.py new file mode 100644 index 000000000..888adc507 --- /dev/null +++ b/src/eligibility_signposting_api/logging/tracing_helper.py @@ -0,0 +1,21 @@ +from collections.abc import Callable +from functools import wraps +from typing import Any + +from aws_xray_sdk.core import xray_recorder +from mangum.types import LambdaContext, LambdaEvent + + +def tracing_setup() -> Callable: + def decorator(func: Callable) -> Callable: + @wraps(func) + def wrapper(event: LambdaEvent, context: LambdaContext) -> dict[str, Any] | None: + xray_recorder.begin_subsegment("Lambda") + try: + return func(event, context) + finally: + xray_recorder.end_subsegment() + + return wrapper + + return decorator diff --git a/tests/unit/logging/test_logs_helper.py b/tests/unit/logging/test_logs_helper.py index 5594cbe36..d11885bc9 100644 --- a/tests/unit/logging/test_logs_helper.py +++ b/tests/unit/logging/test_logs_helper.py @@ -46,13 +46,13 @@ def lambda_context(): ], ) def test_log_request_ids_decorator_logs_metadata(headers, gateway_request_id, expected_extra, lambda_context, caplog): - from eligibility_signposting_api.app import log_request_ids + from eligibility_signposting_api.app import log_request_ids_from_headers event = {"headers": headers} if gateway_request_id is not None: event["requestContext"] = {"requestId": gateway_request_id} - @log_request_ids() + @log_request_ids_from_headers() def test_handler(event, context): # noqa : ARG001 logger = logging.getLogger("test_logger") logger.info("Inside test handler") diff --git a/tests/unit/logging/test_logs_manager.py b/tests/unit/logging/test_logs_manager.py index eb9c38a6c..f78c13451 100644 --- a/tests/unit/logging/test_logs_manager.py +++ b/tests/unit/logging/test_logs_manager.py @@ -11,7 +11,7 @@ from eligibility_signposting_api.logging.logs_manager import ( LOG_FORMAT, EnrichedJsonFormatter, - add_request_id_to_logs, + add_lambda_request_id_to_logger, request_id_context_var, ) @@ -21,7 +21,7 @@ def test_decorator_sets_request_id_in_context(): mock_context = MagicMock() mock_context.aws_request_id = test_request_id - @add_request_id_to_logs() + @add_lambda_request_id_to_logger() def decorated_handler(event, context): # noqa : ARG001 return request_id_context_var.get() @@ -35,7 +35,7 @@ def test_decorator_preserves_function_return_value(): mock_context = MagicMock() mock_context.aws_request_id = "any-id" - @add_request_id_to_logs() + @add_lambda_request_id_to_logger() def decorated_handler(event, context): # noqa : ARG001 return expected_result @@ -47,7 +47,7 @@ def decorated_handler(event, context): # noqa : ARG001 def test_request_id_context_is_properly_isolated(): results = {} - @add_request_id_to_logs() + @add_lambda_request_id_to_logger() def decorated_handler(event, context): # noqa : ARG001 rid = request_id_context_var.get() results[threading.current_thread().name] = rid @@ -86,7 +86,7 @@ def lambda_context(): def test_enriched_json_formatter_adds_all_fields(lambda_context): - @add_request_id_to_logs() + @add_lambda_request_id_to_logger() def test_handler(event, context): # noqa : ARG001 logger = logging.getLogger("test_logger") logger.info("Test log inside handler") From b3b76892337ab10bc954613739b0dee3c9079265 Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Fri, 25 Jul 2025 16:44:14 +0100 Subject: [PATCH 3/4] enable_xray_patching env variable for lambda --- infrastructure/modules/lambda/lambda.tf | 1 + infrastructure/modules/lambda/variables.tf | 5 +++++ infrastructure/stacks/api-layer/lambda.tf | 1 + 3 files changed, 7 insertions(+) diff --git a/infrastructure/modules/lambda/lambda.tf b/infrastructure/modules/lambda/lambda.tf index 9013c8386..f31a6e762 100644 --- a/infrastructure/modules/lambda/lambda.tf +++ b/infrastructure/modules/lambda/lambda.tf @@ -22,6 +22,7 @@ resource "aws_lambda_function" "eligibility_signposting_lambda" { KINESIS_AUDIT_STREAM_TO_S3 = var.kinesis_audit_stream_to_s3_name ENV = var.environment LOG_LEVEL = var.log_level + ENABLE_XRAY_PATCHING = var.enable_xray_patching } } diff --git a/infrastructure/modules/lambda/variables.tf b/infrastructure/modules/lambda/variables.tf index ca6d9b95d..229c1fbb4 100644 --- a/infrastructure/modules/lambda/variables.tf +++ b/infrastructure/modules/lambda/variables.tf @@ -47,3 +47,8 @@ variable "log_level" { description = "log level" type = string } + +variable "enable_xray_patching"{ + description = "flag to enable xray tracing, which puts an entry for dynamodb, s3 and firehose in trace map" + type = string +} diff --git a/infrastructure/stacks/api-layer/lambda.tf b/infrastructure/stacks/api-layer/lambda.tf index 09f56ac03..68885b6d7 100644 --- a/infrastructure/stacks/api-layer/lambda.tf +++ b/infrastructure/stacks/api-layer/lambda.tf @@ -24,5 +24,6 @@ module "eligibility_signposting_lambda_function" { eligibility_status_table_name = module.eligibility_status_table.table_name kinesis_audit_stream_to_s3_name = module.eligibility_audit_firehose_delivery_stream.firehose_stream_name log_level = "INFO" + enable_xray_patching = "true" stack_name = local.stack_name } From 48a406535da34bab6ff6492c771d46028dd1422c Mon Sep 17 00:00:00 2001 From: karthikeyannhs <174426205+Karthikeyannhs@users.noreply.github.com> Date: Mon, 28 Jul 2025 11:11:41 +0100 Subject: [PATCH 4/4] sonar fixes --- src/eligibility_signposting_api/app.py | 2 +- src/eligibility_signposting_api/config/config.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/eligibility_signposting_api/app.py b/src/eligibility_signposting_api/app.py index 076248bf5..ffa3cd14b 100644 --- a/src/eligibility_signposting_api/app.py +++ b/src/eligibility_signposting_api/app.py @@ -18,7 +18,7 @@ from eligibility_signposting_api.logging.tracing_helper import tracing_setup from eligibility_signposting_api.views import eligibility_blueprint -if os.getenv("ENABLE_XRAY_PATCHING", "false") == "true": +if os.getenv("ENABLE_XRAY_PATCHING"): patch_all() init_logging() diff --git a/src/eligibility_signposting_api/config/config.py b/src/eligibility_signposting_api/config/config.py index 914738ce0..58be70258 100644 --- a/src/eligibility_signposting_api/config/config.py +++ b/src/eligibility_signposting_api/config/config.py @@ -22,7 +22,7 @@ def config() -> dict[str, Any]: rules_bucket_name = BucketName(os.getenv("RULES_BUCKET_NAME", "test-rules-bucket")) audit_bucket_name = BucketName(os.getenv("AUDIT_BUCKET_NAME", "test-audit-bucket")) aws_default_region = AwsRegion(os.getenv("AWS_DEFAULT_REGION", "eu-west-1")) - enable_xray_patching = bool(os.getenv("ENABLE_XRAY_PATCHING", "false") == "true") + enable_xray_patching = bool(os.getenv("ENABLE_XRAY_PATCHING", "false")) kinesis_audit_stream_to_s3 = AwsKinesisFirehoseStreamName( os.getenv("KINESIS_AUDIT_STREAM_TO_S3", "test_kinesis_audit_stream_to_s3") ) @@ -44,16 +44,17 @@ def config() -> dict[str, Any]: "log_level": log_level, } + local_stack_endpoint = "http://localhost:4566" return { "aws_access_key_id": AwsAccessKey(os.getenv("AWS_ACCESS_KEY_ID", "dummy_key")), "aws_default_region": aws_default_region, "aws_secret_access_key": AwsSecretAccessKey(os.getenv("AWS_SECRET_ACCESS_KEY", "dummy_secret")), - "dynamodb_endpoint": URL(os.getenv("DYNAMODB_ENDPOINT", "http://localhost:4566")), + "dynamodb_endpoint": URL(os.getenv("DYNAMODB_ENDPOINT", local_stack_endpoint)), "person_table_name": person_table_name, - "s3_endpoint": URL(os.getenv("S3_ENDPOINT", "http://localhost:4566")), + "s3_endpoint": URL(os.getenv("S3_ENDPOINT", local_stack_endpoint)), "rules_bucket_name": rules_bucket_name, "audit_bucket_name": audit_bucket_name, - "firehose_endpoint": URL(os.getenv("FIREHOSE_ENDPOINT", "http://localhost:4566")), + "firehose_endpoint": URL(os.getenv("FIREHOSE_ENDPOINT", local_stack_endpoint)), "kinesis_audit_stream_to_s3": kinesis_audit_stream_to_s3, "enable_xray_patching": enable_xray_patching, "log_level": log_level,