From 2cde648969b4a8284367c8b8ff16e1fb2d57c95e Mon Sep 17 00:00:00 2001 From: aditya Date: Wed, 31 Jul 2024 16:17:44 -0400 Subject: [PATCH] chore: ensure graphql lambda function has the correct IAM permissions (#371) * feat: FE and graphql pieces for manual treasury report generation * fix: linting * feat: add signedUrl support * fix: stale or unused comments * fix: ensure file URL is being parsed from the response correctly * chore: ensure graphql lambda function has the correct IAM permissions * feat: add stepfunctions * chore: add step function as an allowed trigger * fix: addresses removal of unnecessary console.log statements and refactor submit functions * fix: add unit tests for download function and ensure hard return when org is not found * feat: add tests for treasury generation execution * fix: untrack generated type files * chore: re-org terraform and format * fix: use jsonencode format * fix: ensure graphql lambda has start-execution permissions * fix: incorrect arn reference * chore: add environment variables for treasury report generation * chore: add some initial documentaiton to setup the env-var * fix: ensure there is one lambda function to handle project generation * fix: ensure project code is derived from step function event * fix: terraform linting * fix: unused import --- .env.defaults | 7 + .../src/functions/generate_treasury_report.py | 3 +- terraform/functions.tf | 192 ++---------------- .../treasury_generation_lambda_functions.tf | 172 ++++++++++++++++ .../treasury_generation_step_function.tf | 140 +++++++++++++ 5 files changed, 342 insertions(+), 172 deletions(-) create mode 100644 terraform/treasury_generation_lambda_functions.tf create mode 100644 terraform/treasury_generation_step_function.tf diff --git a/.env.defaults b/.env.defaults index b9bc531f..dbdf15dc 100644 --- a/.env.defaults +++ b/.env.defaults @@ -43,3 +43,10 @@ DD_RUM_TRACK_LONG_TASKS=true # Auth provider environment variables AUTH_PROVIDER=local + +# Treasury Report Generation +# Note: this requires creating a step function in localstack with the name `GenerateTreasuryReport` +# Example command to create the step function: +# Create a definition file that contains the step function definition as defined in `treasury_generation_step_function.tf` +# awslocal stepfunctions create-state-machine --name GenerateTreasuryReport --definition file://./step-functions/GenerateTreasuryReport.json --role-arn "arn:aws:iam::000000000000:role/stepfunctions-role" +TREASURY_STEP_FUNCTION_ARN="arn:aws:states:us-west-2:000000000000:stateMachine:GenerateTreasuryReport" diff --git a/python/src/functions/generate_treasury_report.py b/python/src/functions/generate_treasury_report.py index 13242664..51aec2a8 100644 --- a/python/src/functions/generate_treasury_report.py +++ b/python/src/functions/generate_treasury_report.py @@ -7,7 +7,6 @@ import json import tempfile from typing import Any, Optional, Set -import os import boto3 import structlog @@ -76,7 +75,7 @@ def handle(event: S3Event, context: Context): s3_client: S3Client = boto3.client("s3") - project_code = os.getenv("PROJECT_USE_CODE") + project_code = event["ProjectType"] if project_code == "1A": project_use_code = ProjectType._1A elif project_code == "1B": diff --git a/terraform/functions.tf b/terraform/functions.tf index 0229109f..aceb96e2 100644 --- a/terraform/functions.tf +++ b/terraform/functions.tf @@ -301,6 +301,27 @@ module "lambda_function-graphql" { "${module.reporting_data_bucket.bucket_arn}/uploads/*/*/*/*/*.xlsm", ] } + AllowDownloadTreasuryCSVFiles = { + effect = "Allow" + actions = [ + "s3:GetObject", + "s3:HeadObject", + ] + resources = [ + # Path: treasuryreports/{organization_id}/{reporting_period_id}/{filename}.csv + "${module.reporting_data_bucket.bucket_arn}/treasuryreports/*/*/*.csv", + ] + } + + AllowStepFunctionInvocation = { + effect = "Allow" + actions = [ + "states:StartExecution" + ] + resources = [ + module.treasury_generation_step_function.state_machine_arn + ] + } } // Artifacts @@ -336,6 +357,7 @@ module "lambda_function-graphql" { DD_LAMBDA_HANDLER = "graphql.handler" PASSAGE_API_KEY_SECRET_ARN = data.aws_ssm_parameter.passage_api_key_secret_arn.value AUTH_PROVIDER = "passage" + TREASURY_STEP_FUNCTION_ARN = module.treasury_generation_step_function.state_machine_arn }) // Triggers @@ -515,173 +537,3 @@ module "lambda_function-cpfValidation" { } } } - -module "lambda_function-subrecipientTreasuryReportGen" { - source = "terraform-aws-modules/lambda/aws" - version = "6.5.0" - - // Metadata - function_name = "${var.namespace}-subrecipientTreasuryReportGen" - description = "Generates subrecipients file for treasury report when called by step function." - - // Networking - vpc_subnet_ids = null - vpc_security_group_ids = null - attach_network_policy = false - - // Permissions - role_permissions_boundary = local.permissions_boundary_arn - attach_cloudwatch_logs_policy = true - cloudwatch_logs_retention_in_days = var.log_retention_in_days - attach_policy_jsons = length(local.lambda_default_execution_policies) > 0 - number_of_policy_jsons = length(local.lambda_default_execution_policies) - policy_jsons = local.lambda_default_execution_policies - attach_policy_statements = true - policy_statements = { - AllowDownloadSubrecipientsFile = { - effect = "Allow" - actions = [ - "s3:GetObject", - "s3:HeadObject", - ] - resources = [ - # Path: /{organization_id}/{reporting_period_id}/subrecipients - "${module.reporting_data_bucket.bucket_arn}/*/*/subrecipients", - ] - } - AllowUploadCSVReport = { - effect = "Allow" - actions = [ - "s3:PutObject" - ] - resources = [ - # Path: /treasuryreports/{organization_id}/{reporting_period_id}/{userId}/CPFSubrecipientTemplate.csv - "${module.reporting_data_bucket.bucket_arn}/treasuryReports/*/*/*/CPFSubrecipientTemplate.csv", - ] - } - } - - // Artifacts - publish = true - create_package = false - s3_existing_package = { - bucket = aws_s3_object.lambda_artifact-python.bucket - key = aws_s3_object.lambda_artifact-python.key - } - - // Runtime - handler = var.datadog_enabled ? local.datadog_lambda_py_handler : "src.functions.subrecipient_treasury_report_gen.handle" - runtime = var.lambda_py_runtime - architectures = [var.lambda_arch] - layers = local.lambda_py_layer_arns - timeout = 60 # 1 minute, in seconds - memory_size = 512 - environment_variables = merge(local.lambda_default_environment_variables, { - DD_LAMBDA_HANDLER = "src.functions.subrecipient_treasury_report_gen.handle" - DD_LOGS_INJECTION = "true" - }) - - // Triggers -- TODO uncomment the below when step function code is added - # allowed_triggers = { - # StepFunctionTrigger = { - # principal = "states.amazonaws.com" - # source_arn = put_correct_trigger_here.arn - # } - # } -} - -module "lambda_function-treasuryReportGeneration" { - for_each = toset(["1A", "1B", "1C"]) - source = "terraform-aws-modules/lambda/aws" - version = "6.5.0" - - // Metadata - function_name = "${var.namespace}-treasuryReportGeneration${each.key}" - description = "Creates the Treasury Report for Projects ${each.key}." - - // Networking - vpc_subnet_ids = null - vpc_security_group_ids = null - attach_network_policy = false - - // Permissions - role_permissions_boundary = local.permissions_boundary_arn - attach_cloudwatch_logs_policy = true - cloudwatch_logs_retention_in_days = var.log_retention_in_days - attach_policy_jsons = length(local.lambda_default_execution_policies) > 0 - number_of_policy_jsons = length(local.lambda_default_execution_policies) - policy_jsons = local.lambda_default_execution_policies - attach_policy_statements = true - policy_statements = { - AllowDownloadExcelObjects = { - effect = "Allow" - actions = [ - "s3:GetObject", - "s3:HeadObject", - ] - resources = [ - # Path: treasuryreports/{organization_id}/{reporting_period_id}/{filename}.xlsm - "${module.reporting_data_bucket.bucket_arn}/treasuryreports/*/*/*.xlsm", - ] - } - AllowUploadXlsxObjects = { - effect = "Allow" - actions = [ - "s3:PutObject" - ] - resources = [ - # Path: uploads/{organization_id}/{reporting_period_id}/{filename}.xlsx - "${module.reporting_data_bucket.bucket_arn}/uploads/*/*/*.xlsx", - ] - } - AllowUploadCsvObjects = { - effect = "Allow" - actions = [ - "s3:PutObject" - ] - resources = [ - # Path: uploads/{organization_id}/{reporting_period_id}/{filename}.csv - "${module.reporting_data_bucket.bucket_arn}/uploads/*/*/*.csv", - ] - } - AllowUploadJsonObjects = { - effect = "Allow" - actions = [ - "s3:PutObject" - ] - resources = [ - # Path: uploads/{organization_id}/{reporting_period_id}/{filename}.json - "${module.reporting_data_bucket.bucket_arn}/uploads/*/*/*.json", - ] - } - } - - // Artifacts - publish = true - create_package = false - s3_existing_package = { - bucket = aws_s3_object.lambda_artifact-python.bucket - key = aws_s3_object.lambda_artifact-python.key - } - - // Runtime - handler = var.datadog_enabled ? local.datadog_lambda_py_handler : "src.functions.generate_treasury_report.handle" - runtime = var.lambda_py_runtime - architectures = [var.lambda_arch] - layers = local.lambda_py_layer_arns - timeout = 60 # 1 minute, in seconds - memory_size = 512 - environment_variables = merge(local.lambda_default_environment_variables, { - DD_LAMBDA_HANDLER = "src.functions.generate_treasury_report.handle" - DD_LOGS_INJECTION = "true" - PROJECT_USE_CODE = "${each.key}" - }) - - // Triggers - allowed_triggers = { - S3BucketNotification = { - principal = "s3.amazonaws.com" - source_arn = module.reporting_data_bucket.bucket_arn - } - } -} diff --git a/terraform/treasury_generation_lambda_functions.tf b/terraform/treasury_generation_lambda_functions.tf new file mode 100644 index 00000000..7cbc5121 --- /dev/null +++ b/terraform/treasury_generation_lambda_functions.tf @@ -0,0 +1,172 @@ + +module "lambda_function-subrecipientTreasuryReportGen" { + source = "terraform-aws-modules/lambda/aws" + version = "6.5.0" + + // Metadata + function_name = "${var.namespace}-subrecipientTreasuryReportGen" + description = "Generates subrecipients file for treasury report when called by step function." + + // Networking + vpc_subnet_ids = null + vpc_security_group_ids = null + attach_network_policy = false + + // Permissions + role_permissions_boundary = local.permissions_boundary_arn + attach_cloudwatch_logs_policy = true + cloudwatch_logs_retention_in_days = var.log_retention_in_days + attach_policy_jsons = length(local.lambda_default_execution_policies) > 0 + number_of_policy_jsons = length(local.lambda_default_execution_policies) + policy_jsons = local.lambda_default_execution_policies + attach_policy_statements = true + policy_statements = { + AllowDownloadSubrecipientsFile = { + effect = "Allow" + actions = [ + "s3:GetObject", + "s3:HeadObject", + ] + resources = [ + # Path: /{organization_id}/{reporting_period_id}/subrecipients + "${module.reporting_data_bucket.bucket_arn}/*/*/subrecipients", + ] + } + AllowUploadCSVReport = { + effect = "Allow" + actions = [ + "s3:PutObject" + ] + resources = [ + # Path: /treasuryreports/{organization_id}/{reporting_period_id}/{userId}/CPFSubrecipientTemplate.csv + "${module.reporting_data_bucket.bucket_arn}/treasuryReports/*/*/*/CPFSubrecipientTemplate.csv", + ] + } + } + + // Artifacts + publish = true + create_package = false + s3_existing_package = { + bucket = aws_s3_object.lambda_artifact-python.bucket + key = aws_s3_object.lambda_artifact-python.key + } + + // Runtime + handler = var.datadog_enabled ? local.datadog_lambda_py_handler : "src.functions.subrecipient_treasury_report_gen.handle" + runtime = var.lambda_py_runtime + architectures = [var.lambda_arch] + layers = local.lambda_py_layer_arns + timeout = 60 # 1 minute, in seconds + memory_size = 512 + environment_variables = merge(local.lambda_default_environment_variables, { + DD_LAMBDA_HANDLER = "src.functions.subrecipient_treasury_report_gen.handle" + DD_LOGS_INJECTION = "true" + }) + + allowed_triggers = { + StepFunctionTrigger = { + principal = "states.amazonaws.com" + source_arn = module.treasury_generation_step_function.state_machine_arn + } + + } +} + +module "lambda_function-treasuryProjectFileGeneration" { + source = "terraform-aws-modules/lambda/aws" + version = "6.5.0" + + // Metadata + function_name = "${var.namespace}-treasuryProjectFileGeneration" + description = "Creates the Treasury Report for Projects." + + // Networking + vpc_subnet_ids = null + vpc_security_group_ids = null + attach_network_policy = false + + // Permissions + role_permissions_boundary = local.permissions_boundary_arn + attach_cloudwatch_logs_policy = true + cloudwatch_logs_retention_in_days = var.log_retention_in_days + attach_policy_jsons = length(local.lambda_default_execution_policies) > 0 + number_of_policy_jsons = length(local.lambda_default_execution_policies) + policy_jsons = local.lambda_default_execution_policies + attach_policy_statements = true + policy_statements = { + AllowDownloadExcelObjects = { + effect = "Allow" + actions = [ + "s3:GetObject", + "s3:HeadObject", + ] + resources = [ + # Path: treasuryreports/{organization_id}/{reporting_period_id}/{filename}.xlsm + "${module.reporting_data_bucket.bucket_arn}/treasuryreports/*/*/*.xlsm", + ] + } + AllowUploadXlsxObjects = { + effect = "Allow" + actions = [ + "s3:PutObject" + ] + resources = [ + # Path: uploads/{organization_id}/{reporting_period_id}/{filename}.xlsx + "${module.reporting_data_bucket.bucket_arn}/uploads/*/*/*.xlsx", + ] + } + AllowUploadCsvObjects = { + effect = "Allow" + actions = [ + "s3:PutObject" + ] + resources = [ + # Path: uploads/{organization_id}/{reporting_period_id}/{filename}.csv + "${module.reporting_data_bucket.bucket_arn}/uploads/*/*/*.csv", + ] + } + AllowUploadJsonObjects = { + effect = "Allow" + actions = [ + "s3:PutObject" + ] + resources = [ + # Path: uploads/{organization_id}/{reporting_period_id}/{filename}.json + "${module.reporting_data_bucket.bucket_arn}/uploads/*/*/*.json", + ] + } + } + + // Artifacts + publish = true + create_package = false + s3_existing_package = { + bucket = aws_s3_object.lambda_artifact-python.bucket + key = aws_s3_object.lambda_artifact-python.key + } + + // Runtime + handler = var.datadog_enabled ? local.datadog_lambda_py_handler : "src.functions.generate_treasury_report.handle" + runtime = var.lambda_py_runtime + architectures = [var.lambda_arch] + layers = local.lambda_py_layer_arns + timeout = 60 # 1 minute, in seconds + memory_size = 512 + environment_variables = merge(local.lambda_default_environment_variables, { + DD_LAMBDA_HANDLER = "src.functions.generate_treasury_report.handle" + DD_LOGS_INJECTION = "true" + }) + + // Triggers + allowed_triggers = { + S3BucketNotification = { + principal = "s3.amazonaws.com" + source_arn = module.reporting_data_bucket.bucket_arn + } + StepFunctionTrigger = { + principal = "states.amazonaws.com" + source_arn = module.treasury_generation_step_function.state_machine_arn + } + } +} diff --git a/terraform/treasury_generation_step_function.tf b/terraform/treasury_generation_step_function.tf new file mode 100644 index 00000000..6539a2ed --- /dev/null +++ b/terraform/treasury_generation_step_function.tf @@ -0,0 +1,140 @@ +module "treasury_generation_step_function" { + source = "terraform-aws-modules/step-functions/aws" + + name = "generate-treasury-report" + definition = jsonencode({ + "Comment" : "Generate all the files for a treasury report", + "StartAt" : "Parallel", + "States" : { + "Parallel" : { + "Type" : "Parallel", + "End" : true, + "Branches" : [ + { + "StartAt" : "Generate Project 1A File", + "States" : { + "Generate Project 1A File" : { + "Type" : "Task", + "Resource" : "arn:aws:states:::lambda:invoke", + "OutputPath" : "$.Payload", + "Parameters" : { + "Payload.$" : "$", + "ProjectType" : "1A", + "FunctionName" : module.lambda_function-treasuryProjectFileGeneration.lambda_function_arn + }, + "Retry" : [ + { + "ErrorEquals" : [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds" : 1, + "MaxAttempts" : 2, + "BackoffRate" : 2 + } + ], + "End" : true + } + } + }, + { + "StartAt" : "Generate Project 1B File", + "States" : { + "Generate Project 1B File" : { + "Type" : "Task", + "Resource" : "arn:aws:states:::lambda:invoke", + "OutputPath" : "$.Payload", + "Parameters" : { + "Payload.$" : "$", + "ProjectType" : "1B", + "FunctionName" : module.lambda_function-treasuryProjectFileGeneration.lambda_function_arn + }, + "Retry" : [ + { + "ErrorEquals" : [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds" : 1, + "MaxAttempts" : 2, + "BackoffRate" : 2 + } + ], + "End" : true + } + } + }, + { + "StartAt" : "Generate Project 1C File", + "States" : { + "Generate Project 1C File" : { + "Type" : "Task", + "Resource" : "arn:aws:states:::lambda:invoke", + "OutputPath" : "$.Payload", + "Parameters" : { + "Payload.$" : "$", + "ProjectType" : "1C", + "FunctionName" : module.lambda_function-treasuryProjectFileGeneration.lambda_function_arn + }, + "Retry" : [ + { + "ErrorEquals" : [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds" : 1, + "MaxAttempts" : 2, + "BackoffRate" : 2 + } + ], + "End" : true + } + } + }, + { + "StartAt" : "Generate Subrecipient File", + "States" : { + "Generate Subrecipient File" : { + "Type" : "Task", + "Resource" : "arn:aws:states:::lambda:invoke", + "OutputPath" : "$.Payload", + "Parameters" : { + "Payload.$" : "$", + "FunctionName" : module.lambda_function-subrecipientTreasuryReportGen.lambda_function_arn + }, + "Retry" : [ + { + "ErrorEquals" : [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds" : 1, + "MaxAttempts" : 2, + "BackoffRate" : 2 + } + ], + "End" : true + } + } + } + ] + } + } + }) + + service_integrations = { + lambda = { + lambda = [module.lambda_function-treasuryProjectFileGeneration.lambda_function_arn, module.lambda_function-subrecipientTreasuryReportGen.lambda_function_arn] + } + } + + type = "STANDARD" +}