Skip to content

Commit f1961d8

Browse files
TOEL2TOEL2
authored
Merge pull request #609 from NHSDigital/feature/ELI-688-kinesis-stream-buffer
[ELI-688] - Adding Kinesis Data Stream as a buffer for Firehose
2 parents 1bdd58e + 6363578 commit f1961d8

24 files changed

Lines changed: 493 additions & 66 deletions

‎infrastructure/modules/kinesis_firehose/kinesis_firehose_delivery_stream.tf‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,11 @@ resource "aws_kinesis_firehose_delivery_stream" "eligibility_audit_firehose_deli
22
name = "${terraform.workspace == "default" ? "" : "${terraform.workspace}-"}${var.project_name}-${var.environment}-${var.audit_firehose_delivery_stream_name}"
33
destination = "extended_s3"
44

5+
kinesis_source_configuration {
6+
kinesis_stream_arn = var.kinesis_source_stream_arn
7+
role_arn = var.audit_firehose_role.arn
8+
}
9+
510
extended_s3_configuration {
611
role_arn = var.audit_firehose_role.arn
712
bucket_arn = var.s3_audit_bucket_arn
@@ -14,20 +19,13 @@ resource "aws_kinesis_firehose_delivery_stream" "eligibility_audit_firehose_deli
1419

1520
cloudwatch_logging_options {
1621
enabled = true
17-
log_group_name = var.kinesis_cloud_watch_log_group_name
18-
log_stream_name = var.kinesis_cloud_watch_log_stream
22+
log_group_name = var.firehose_cloud_watch_log_group_name
23+
log_stream_name = var.firehose_cloud_watch_log_stream
1924
}
2025
}
21-
22-
server_side_encryption {
23-
enabled = true
24-
key_arn = aws_kms_key.firehose_cmk.arn
25-
key_type = "CUSTOMER_MANAGED_CMK"
26-
}
27-
26+
# Removed server_side_encryption_configuration as it is not supported for kinesis as source
2827
depends_on = [
2928
aws_kms_key.firehose_cmk,
30-
var.audit_firehose_role
3129
]
3230

3331
tags = var.tags

‎infrastructure/modules/kinesis_firehose/variables.tf‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,12 @@ variable "s3_audit_bucket_arn" {
1313
type = string
1414
}
1515

16-
variable "kinesis_cloud_watch_log_group_name" {
16+
variable "firehose_cloud_watch_log_group_name" {
1717
description = "kinesis cloud watch log group name"
1818
type = string
1919
}
2020

21-
variable "kinesis_cloud_watch_log_stream" {
21+
variable "firehose_cloud_watch_log_stream" {
2222
description = "kinesis cloud watch log stream"
2323
type = string
2424
}
@@ -28,6 +28,11 @@ variable "eligibility_lambda_role_arn" {
2828
type = any
2929
}
3030

31+
variable "kinesis_source_stream_arn" {
32+
description = "the arn of the kinesis data stream the lambda puts records to"
33+
type = any
34+
}
35+
3136

3237

3338

‎infrastructure/modules/lambda/lambda.tf‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,12 @@ resource "aws_lambda_function" "eligibility_signposting_lambda" {
2020
PERSON_TABLE_NAME = var.eligibility_status_table_name,
2121
RULES_BUCKET_NAME = var.eligibility_rules_bucket_name,
2222
CONSUMER_MAPPING_BUCKET_NAME = var.eligibility_consumer_mappings_bucket_name,
23-
KINESIS_AUDIT_STREAM_TO_S3 = var.kinesis_audit_stream_to_s3_name
24-
ENV = var.environment
25-
LOG_LEVEL = var.log_level
26-
ENABLE_XRAY_PATCHING = var.enable_xray_patching
27-
API_DOMAIN_NAME = var.api_domain_name
28-
HASHING_SECRET_NAME = var.hashing_secret_name
23+
KINESIS_AUDIT_STREAM = var.kinesis_audit_stream_name,
24+
ENV = var.environment,
25+
LOG_LEVEL = var.log_level,
26+
ENABLE_XRAY_PATCHING = var.enable_xray_patching,
27+
API_DOMAIN_NAME = var.api_domain_name,
28+
HASHING_SECRET_NAME = var.hashing_secret_name,
2929
}
3030
}
3131

‎infrastructure/modules/lambda/variables.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ variable "eligibility_status_table_name" {
5454
type = string
5555
}
5656

57-
variable "kinesis_audit_stream_to_s3_name" {
57+
variable "kinesis_audit_stream_name" {
5858
description = "kinesis audit stream to s3 name"
5959
type = string
6060
}

‎infrastructure/stacks/api-layer/assumed_role_permissions_boundary.tf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ data "aws_iam_policy_document" "assumed_role_permissions_boundary" {
5353
# Kinesis Firehose - Lambda writing audit data
5454
"firehose:PutRecord",
5555
"firehose:PutRecordBatch",
56+
"kinesis:*",
5657

5758
# X-Ray - Lambda tracing
5859
"xray:PutTraceSegments",

‎infrastructure/stacks/api-layer/iam_policies.tf‎

Lines changed: 105 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -224,15 +224,15 @@ resource "aws_iam_role_policy" "kinesis_firehose_logs_policy" {
224224
"logs:CreateLogStream",
225225
"logs:PutLogEvents"
226226
],
227-
Resource = "arn:aws:logs:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:log-group:/aws/kinesisfirehose/${module.eligibility_audit_firehose_delivery_stream.firehose_stream_name}:log-stream:*"
227+
Resource = "arn:aws:logs:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:log-group:/aws/kinesisfirehose/${local.firehose_stream_name}:log-stream:*"
228228
},
229229
{
230230
Effect = "Allow",
231231
Action = [
232232
"logs:DescribeLogGroups",
233233
"logs:DescribeLogStreams"
234234
],
235-
Resource = "arn:aws:logs:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:log-group:/aws/kinesisfirehose/${module.eligibility_audit_firehose_delivery_stream.firehose_stream_name}"
235+
Resource = "arn:aws:logs:${var.default_aws_region}:${data.aws_caller_identity.current.account_id}:log-group:/aws/kinesisfirehose/${local.firehose_stream_name}"
236236
}
237237
]
238238
})
@@ -256,6 +256,109 @@ resource "aws_iam_role_policy_attachment" "lambda_insights_policy" {
256256
policy_arn = "arn:aws:iam::aws:policy/CloudWatchLambdaInsightsExecutionRolePolicy"
257257
}
258258

259+
# Policy document to read from Kinesis Source stream
260+
data "aws_iam_policy_document" "kinesis_source_access" {
261+
statement {
262+
sid = "AllowReadFromKinesisSourceStream"
263+
effect = "Allow"
264+
265+
actions = [
266+
"kinesis:DescribeStream",
267+
"kinesis:DescribeStreamSummary",
268+
"kinesis:GetRecords",
269+
"kinesis:GetShardIterator",
270+
"kinesis:ListShards",
271+
"kinesis:SubscribeToShard",
272+
]
273+
274+
resources = [
275+
aws_kinesis_stream.kinesis_source_stream.arn,
276+
]
277+
}
278+
}
279+
280+
# Policy document to use KMS key for reading from Kinesis Source stream
281+
data "aws_iam_policy_document" "kinesis_source_kms_read_access" {
282+
statement {
283+
sid = "AllowUseOfKinesisSourceKeyForReads"
284+
effect = "Allow"
285+
286+
actions = [
287+
"kms:Decrypt",
288+
"kms:GenerateDataKey",
289+
"kms:DescribeKey"
290+
]
291+
292+
resources = [
293+
aws_kms_key.kinesis_data_stream_kms_key.arn
294+
]
295+
}
296+
}
297+
298+
# Attach kinesis read policy to firehose role
299+
resource "aws_iam_role_policy" "kinesis_firehose_read_policy" {
300+
name = "KinesisSourceReadAccess"
301+
role = aws_iam_role.eligibility_audit_firehose_role.id
302+
policy = data.aws_iam_policy_document.kinesis_source_access.json
303+
}
304+
305+
# Attach kinesis source stream KMS read policy to firehose role
306+
resource "aws_iam_role_policy" "firehose_kinesis_source_kms_policy" {
307+
name = "KinesisSourceKmsReadAccess"
308+
role = aws_iam_role.eligibility_audit_firehose_role.id
309+
policy = data.aws_iam_policy_document.kinesis_source_kms_read_access.json
310+
}
311+
312+
# Policy document for Lambda to write to Kinesis stream
313+
data "aws_iam_policy_document" "kinesis_write_access" {
314+
statement {
315+
sid = "AllowWriteToKinesisStream"
316+
effect = "Allow"
317+
318+
actions = [
319+
"kinesis:PutRecord",
320+
"kinesis:PutRecords"
321+
]
322+
323+
resources = [
324+
aws_kinesis_stream.kinesis_source_stream.arn
325+
]
326+
}
327+
}
328+
329+
# Policy document to use the KMS key
330+
data "aws_iam_policy_document" "kinesis_kms_write_access" {
331+
statement {
332+
sid = "AllowUseOfKinesisStreamKeyForWrites"
333+
effect = "Allow"
334+
335+
actions = [
336+
"kms:Encrypt",
337+
"kms:GenerateDataKey",
338+
"kms:DescribeKey"
339+
]
340+
341+
resources = [
342+
aws_kms_key.kinesis_data_stream_kms_key.arn
343+
]
344+
}
345+
}
346+
347+
348+
# Attach kinesis write policy to Lambda role
349+
resource "aws_iam_role_policy" "lambda_kinesis_write_policy" {
350+
name = "KinesisWriteAccess"
351+
role = aws_iam_role.eligibility_lambda_role.id
352+
policy = data.aws_iam_policy_document.kinesis_write_access.json
353+
}
354+
355+
# Attach kinesis KMS access policy to Lambda role
356+
resource "aws_iam_role_policy" "lambda_kinesis_kms_policy" {
357+
name = "KinesisStreamKmsWriteAccess"
358+
role = aws_iam_role.eligibility_lambda_role.id
359+
policy = data.aws_iam_policy_document.kinesis_kms_write_access.json
360+
}
361+
259362
# Policy doc for S3 Audit bucket
260363
data "aws_iam_policy_document" "s3_audit_bucket_policy" {
261364
statement {

‎infrastructure/stacks/api-layer/kinesis_data_stream.tf‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
resource "aws_kms_key" "kinesis_data_stream_kms_key" {
2+
description = "${terraform.workspace == "default" ? "" : "${terraform.workspace}-"} kinesis_data_stream_kms Master Key"
3+
deletion_window_in_days = 14
4+
is_enabled = true
5+
enable_key_rotation = true
6+
tags = local.tags
7+
}
8+
9+
resource "aws_kms_alias" "kinesis_data_stream_kms_key" {
10+
name = "alias/${var.project_name}-${var.environment}-kinesis-audit-stream"
11+
target_key_id = aws_kms_key.kinesis_data_stream_kms_key.key_id
12+
}
13+
14+
15+
data "aws_iam_policy_document" "kinesis_stream_kms_key_policy" {
16+
#checkov:skip=CKV_AWS_111 Root user needs full KMS key management
17+
#checkov:skip=CKV_AWS_109 Root user needs full KMS key management
18+
#checkov:skip=CKV_AWS_356 Root user needs full KMS key management
19+
statement {
20+
sid = "EnableRootPermissions"
21+
effect = "Allow"
22+
23+
principals {
24+
type = "AWS"
25+
identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"]
26+
}
27+
28+
actions = ["kms:*"]
29+
resources = ["*"]
30+
}
31+
32+
statement {
33+
sid = "AllowLambdaUseOfKey"
34+
effect = "Allow"
35+
36+
principals {
37+
type = "AWS"
38+
identifiers = [aws_iam_role.eligibility_lambda_role.arn]
39+
}
40+
41+
actions = [
42+
"kms:Encrypt",
43+
"kms:Decrypt",
44+
"kms:ReEncrypt*",
45+
"kms:GenerateDataKey*",
46+
"kms:DescribeKey"
47+
]
48+
49+
resources = ["*"]
50+
51+
condition {
52+
test = "StringEquals"
53+
variable = "kms:ViaService"
54+
values = ["kinesis.${var.default_aws_region}.amazonaws.com"]
55+
}
56+
}
57+
58+
statement {
59+
sid = "AllowFirehoseRoleUseOfKey"
60+
effect = "Allow"
61+
62+
principals {
63+
type = "AWS"
64+
identifiers = [aws_iam_role.eligibility_audit_firehose_role.arn]
65+
}
66+
67+
actions = [
68+
"kms:Decrypt",
69+
"kms:GenerateDataKey*",
70+
"kms:DescribeKey"
71+
]
72+
73+
resources = ["*"]
74+
75+
condition {
76+
test = "StringEquals"
77+
variable = "kms:ViaService"
78+
values = ["firehose.${var.default_aws_region}.amazonaws.com"]
79+
}
80+
}
81+
}
82+
83+
resource "aws_kms_key_policy" "kinesis_stream_kms_key_policy" {
84+
key_id = aws_kms_key.kinesis_data_stream_kms_key.id
85+
policy = data.aws_iam_policy_document.kinesis_stream_kms_key_policy.json
86+
}
87+
88+
resource "aws_kinesis_stream" "kinesis_source_stream" {
89+
name = "${var.project_name}-${var.environment}-kinesis-audit-stream"
90+
retention_period = 24
91+
92+
stream_mode_details {
93+
stream_mode = "ON_DEMAND" # can discuss later
94+
}
95+
96+
encryption_type = "KMS"
97+
kms_key_id = aws_kms_key.kinesis_data_stream_kms_key.arn
98+
tags = local.tags
99+
}

‎infrastructure/stacks/api-layer/kinesis_firehose.tf‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,21 @@
11
module "eligibility_audit_firehose_delivery_stream" {
2-
source = "../../modules/kinesis_firehose"
3-
audit_firehose_delivery_stream_name = "audit_stream_to_s3"
4-
audit_firehose_role = aws_iam_role.eligibility_audit_firehose_role
5-
s3_audit_bucket_arn = module.s3_audit_bucket.storage_bucket_arn
6-
environment = local.environment
7-
stack_name = local.stack_name
8-
workspace = local.workspace
9-
tags = local.tags
10-
kinesis_cloud_watch_log_group_name = aws_cloudwatch_log_group.firehose_audit.name
11-
kinesis_cloud_watch_log_stream = aws_cloudwatch_log_stream.firehose_audit_stream.name
12-
eligibility_lambda_role_arn = aws_iam_role.eligibility_lambda_role.arn
2+
source = "../../modules/kinesis_firehose"
3+
audit_firehose_delivery_stream_name = "audit_stream_to_s3"
4+
audit_firehose_role = aws_iam_role.eligibility_audit_firehose_role
5+
s3_audit_bucket_arn = module.s3_audit_bucket.storage_bucket_arn
6+
environment = local.environment
7+
stack_name = local.stack_name
8+
workspace = local.workspace
9+
tags = local.tags
10+
firehose_cloud_watch_log_group_name = aws_cloudwatch_log_group.firehose_audit.name
11+
firehose_cloud_watch_log_stream = aws_cloudwatch_log_stream.firehose_audit_stream.name
12+
eligibility_lambda_role_arn = aws_iam_role.eligibility_lambda_role.arn
13+
kinesis_source_stream_arn = aws_kinesis_stream.kinesis_source_stream.arn
14+
15+
depends_on = [
16+
aws_iam_role_policy.kinesis_firehose_read_policy,
17+
aws_iam_role_policy.firehose_kinesis_source_kms_policy,
18+
aws_iam_role_policy.kinesis_firehose_s3_write_policy,
19+
aws_iam_role_policy.kinesis_firehose_logs_policy,
20+
]
1321
}

‎infrastructure/stacks/api-layer/lambda.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ module "eligibility_signposting_lambda_function" {
2525
eligibility_rules_bucket_name = module.s3_rules_bucket.storage_bucket_name
2626
eligibility_consumer_mappings_bucket_name = module.s3_consumer_mappings_bucket.storage_bucket_name
2727
eligibility_status_table_name = module.eligibility_status_table.table_name
28-
kinesis_audit_stream_to_s3_name = module.eligibility_audit_firehose_delivery_stream.firehose_stream_name
28+
kinesis_audit_stream_name = aws_kinesis_stream.kinesis_source_stream.name
2929
hashing_secret_name = module.secrets_manager.aws_hashing_secret_name
3030
lambda_insights_extension_version = 38
3131
log_level = "INFO"

‎infrastructure/stacks/api-layer/locals.tf‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ locals {
1010
data.aws_ssm_parameter.mtls_api_ca_cert.value
1111
])
1212

13+
firehose_stream_name = "${terraform.workspace == "default" ? "" : "${terraform.workspace}-"}${var.project_name}-${var.environment}-audit_stream_to_s3"
14+
1315
# Toggle for deploying WAF resources in the current environment
1416
# True when var.environment is contained in var.waf_enabled_environments
1517
waf_enabled = contains(var.waf_enabled_environments, var.environment)

0 commit comments

Comments
 (0)