From 356a8a7f97dacf507f56120a9e83f4d86fd70bdd Mon Sep 17 00:00:00 2001 From: Petros Kalos Date: Tue, 16 Apr 2024 14:13:50 +0300 Subject: [PATCH] run db migrations in a custom resource (#1177) ### Feature or Bugfix - Bugfix ### Detail Currently the DB is being initialised in the DBMigrations pipeline stage (using CodeBuild) which runs after the BackendStage. The SavePermissions TriggerFunction runs during the deployment of Backend, just after deployment of the DBCluster. As a result on clean deployments the SavePermissions step will fail because the DB is uninitialized. To resolve that in this PR we do the following * remove DBMigrations stage based on CodeBuild * run DBMigrations as part of a TriggerFunction/CustomResource after the DB deployment * run SavePermissions TriggerFunction after the DBMigrations TriggerFunction ### Security Please answer the questions below briefly where applicable, or write `N/A`. Based on [OWASP 10](https://owasp.org/Top10/en/). - Does this PR introduce or modify any input fields or queries - this includes fetching data from storage outside the application (e.g. a database, an S3 bucket)? - Is the input sanitized? - What precautions are you taking before deserializing the data you consume? - Is injection prevented by parametrizing queries? - Have you ensured no `eval` or similar functions are used? - Does this PR introduce any functionality or component that requires authorization? - How have you ensured it respects the existing AuthN/AuthZ mechanisms? - Are you logging failed auth attempts? - Are you using or adding any cryptographic features? - Do you use a standard proven implementations? - Are the used keys controlled by the customer? Where are they stored? - Are you introducing any new policies/roles/users? - Have you used the least-privilege principle? How? By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. --- backend/dbmigrations_handler.py | 18 ++ backend/requirements.txt | 3 +- ...rigger_handler.py => saveperms_handler.py} | 3 +- deploy/stacks/backend_stack.py | 35 ++- deploy/stacks/dbmigration.py | 239 ------------------ deploy/stacks/pipeline.py | 32 --- deploy/stacks/trigger_function_stack.py | 3 +- 7 files changed, 40 insertions(+), 293 deletions(-) create mode 100644 backend/dbmigrations_handler.py rename backend/{trigger_handler.py => saveperms_handler.py} (88%) delete mode 100644 deploy/stacks/dbmigration.py diff --git a/backend/dbmigrations_handler.py b/backend/dbmigrations_handler.py new file mode 100644 index 000000000..00d7c5c98 --- /dev/null +++ b/backend/dbmigrations_handler.py @@ -0,0 +1,18 @@ +""" +The handler of this module will be called once upon every deployment +""" + +import logging +import os + +from alembic import command +from alembic.config import Config + +logger = logging.getLogger() +logger.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) + + +def handler(event, context) -> None: + alembic_cfg = Config('alembic.ini') + alembic_cfg.set_main_option('script_location', './migrations') + command.upgrade(alembic_cfg, 'head') # logging breaks after this command diff --git a/backend/requirements.txt b/backend/requirements.txt index bf8128150..9e4c133d6 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -14,4 +14,5 @@ PyYAML==6.0 requests==2.31.0 requests_aws4auth==1.1.1 sqlalchemy==1.3.24 -starlette==0.36.3 \ No newline at end of file +starlette==0.36.3 +alembic==1.13.1 \ No newline at end of file diff --git a/backend/trigger_handler.py b/backend/saveperms_handler.py similarity index 88% rename from backend/trigger_handler.py rename to backend/saveperms_handler.py index c857faf5d..8f8d601cd 100644 --- a/backend/trigger_handler.py +++ b/backend/saveperms_handler.py @@ -11,10 +11,9 @@ logger = logging.getLogger() logger.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) -log = logging.getLogger(__name__) -def save_permissions(event, context): +def handler(event, context) -> None: load_modules(modes={ImportMode.API}) envname = os.getenv('envname', 'local') engine = get_engine(envname=envname) diff --git a/deploy/stacks/backend_stack.py b/deploy/stacks/backend_stack.py index 535e0639e..61c5a1ee8 100644 --- a/deploy/stacks/backend_stack.py +++ b/deploy/stacks/backend_stack.py @@ -11,7 +11,6 @@ from .container import ContainerStack from .cw_canaries import CloudWatchCanariesStack from .cw_rum import CloudWatchRumStack -from .dbmigration import DBMigrationStack from .lambda_api import LambdaApiStack from .monitoring import MonitoringStack from .opensearch import OpenSearchStack @@ -220,21 +219,6 @@ def __init__( **kwargs, ) - dbmigration_stack = DBMigrationStack( - self, - 'DbMigration', - envname=envname, - resource_prefix=resource_prefix, - vpc=vpc, - s3_prefix_list=self.s3_prefix_list, - tooling_account_id=tooling_account_id, - pipeline_bucket=pipeline_bucket, - vpce_connection=vpce_connection, - codeartifact_domain_name=codeartifact_domain_name, - codeartifact_pip_repo_name=codeartifact_pip_repo_name, - **kwargs, - ) - if quicksight_enabled: pivot_role_in_account = iam.Role( self, @@ -321,22 +305,37 @@ def __init__( self.lambda_api_stack.api_handler, ], ecs_security_groups=self.ecs_stack.ecs_security_groups, - codebuild_dbmigration_sg=dbmigration_stack.codebuild_sg, prod_sizing=prod_sizing, quicksight_monitoring_sg=quicksight_monitoring_sg, **kwargs, ) + db_migrations = TriggerFunctionStack( + self, + 'DbMigrations', + handler='dbmigrations_handler.handler', + envname=envname, + resource_prefix=resource_prefix, + vpc=vpc, + vpce_connection=vpce_connection, + image_tag=image_tag, + ecr_repository=repo, + execute_after=[aurora_stack.cluster], + connectables=[aurora_stack.cluster], + **kwargs, + ) + TriggerFunctionStack( self, 'SavePerms', - handler='trigger_handler.save_permissions', + handler='saveperms_handler.handler', envname=envname, resource_prefix=resource_prefix, vpc=vpc, vpce_connection=vpce_connection, image_tag=image_tag, ecr_repository=repo, + execute_after=[db_migrations.trigger_function], connectables=[aurora_stack.cluster], **kwargs, ) diff --git a/deploy/stacks/dbmigration.py b/deploy/stacks/dbmigration.py deleted file mode 100644 index c8edd9e74..000000000 --- a/deploy/stacks/dbmigration.py +++ /dev/null @@ -1,239 +0,0 @@ -from aws_cdk import aws_codebuild as codebuild -from aws_cdk import aws_ec2 as ec2 -from aws_cdk import aws_iam as iam - -from .pyNestedStack import pyNestedClass - - -class DBMigrationStack(pyNestedClass): - def __init__( - self, - scope, - id, - vpc, - s3_prefix_list=None, - envname='dev', - resource_prefix='dataall', - pipeline_bucket: str = None, - tooling_account_id=None, - codeartifact_domain_name=None, - codeartifact_pip_repo_name=None, - vpce_connection=None, - **kwargs, - ): - super().__init__(scope, id, **kwargs) - - self.build_project_role = iam.Role( - self, - id=f'DBMigrationCBRole{envname}', - role_name=f'{resource_prefix}-{envname}-cb-dbmigration-role', - assumed_by=iam.CompositePrincipal( - iam.ServicePrincipal('codebuild.amazonaws.com'), - iam.AccountPrincipal(tooling_account_id), - ), - ) - private_subnets = vpc.select_subnets(subnet_type=ec2.SubnetType.PRIVATE_WITH_NAT) - - subnet_resources = [] - - for subnet in private_subnets.subnets: - subnet_resources.append(f'arn:aws:ec2:{self.region}:{self.account}:subnet/{subnet.subnet_id}') - - subnet_iam_condition = {'ec2:Subnet': subnet_resources, 'ec2:AuthorizedService': 'codebuild.amazonaws.com'} - - self.build_project_role.attach_inline_policy( - iam.Policy( - self, - f'DBMigrationCBProject{envname}PolicyDocument', - statements=[ - iam.PolicyStatement( - effect=iam.Effect.ALLOW, - actions=[ - 'ec2:CreateNetworkInterface', - 'ec2:DeleteNetworkInterface', - ], - resources=[ - f'arn:aws:ec2:{self.region}:{self.account}:*/*', - ], - ), - iam.PolicyStatement( - actions=[ - 'ec2:CreateNetworkInterfacePermission', - ], - resources=[ - f'arn:aws:ec2:{self.region}:{self.account}:network-interface/*', - ], - conditions={ - 'StringEquals': subnet_iam_condition, - }, - ), - iam.PolicyStatement( - actions=[ - 'ec2:DescribeNetworkInterfaces', - 'ec2:DescribeSubnets', - 'ec2:DescribeSecurityGroups', - 'ec2:DescribeDhcpOptions', - 'ec2:DescribeVpcs', - ], - resources=['*'], - ), - ], - ) - ) - - self.build_project_role.add_to_policy( - iam.PolicyStatement( - actions=[ - 'secretsmanager:GetSecretValue', - 'kms:Decrypt', - 'secretsmanager:DescribeSecret', - 'kms:Encrypt', - 'kms:GenerateDataKey', - 'ssm:GetParametersByPath', - 'ssm:GetParameters', - 'ssm:GetParameter', - ], - resources=[ - f'arn:aws:secretsmanager:{self.region}:{self.account}:secret:*{resource_prefix}*', - f'arn:aws:secretsmanager:{self.region}:{self.account}:secret:*dataall*', - f'arn:aws:kms:{self.region}:{self.account}:key/*', - f'arn:aws:ssm:*:{self.account}:parameter/*dataall*', - f'arn:aws:ssm:*:{self.account}:parameter/*{resource_prefix}*', - ], - ) - ) - self.build_project_role.add_to_policy( - iam.PolicyStatement( - actions=[ - 's3:GetObject', - 's3:ListBucketVersions', - 's3:ListBucket', - 's3:GetBucketLocation', - 's3:GetObjectVersion', - 'codebuild:StartBuild', - 'codebuild:BatchGetBuilds', - ], - resources=[ - f'arn:aws:s3:::{pipeline_bucket}/*', - f'arn:aws:s3:::{pipeline_bucket}', - f'arn:aws:codebuild:{self.region}:{self.account}:project/*{resource_prefix}*', - ], - ), - ) - self.build_project_role.add_to_policy( - iam.PolicyStatement( - actions=[ - 's3:GetObject', - 's3:ListBucketVersions', - 's3:ListBucket', - 's3:GetBucketLocation', - 's3:GetObjectVersion', - ], - resources=[ - f'arn:aws:s3:::{resource_prefix}-{envname}-{self.account}-{self.region}-resources/*', - f'arn:aws:s3:::{resource_prefix}-{envname}-{self.account}-{self.region}-resources', - ], - ), - ) - self.build_project_role.add_to_policy( - iam.PolicyStatement( - actions=[ - 'codeartifact:GetAuthorizationToken', - 'codeartifact:ReadFromRepository', - 'codeartifact:GetRepositoryEndpoint', - 'codeartifact:GetRepositoryPermissionsPolicy', - ], - resources=[ - f'arn:aws:codeartifact:*:{tooling_account_id}:repository/{codeartifact_domain_name}/{codeartifact_pip_repo_name}', - f'arn:aws:codeartifact:*:{tooling_account_id}:domain/{codeartifact_domain_name}', - ], - ), - ) - self.build_project_role.add_to_policy( - iam.PolicyStatement( - actions=['sts:GetServiceBearerToken'], - resources=['*'], - conditions={'StringEquals': {'sts:AWSServiceName': 'codeartifact.amazonaws.com'}}, - ), - ) - self.build_project_role.add_to_policy( - iam.PolicyStatement( - actions=['logs:CreateLogGroup', 'logs:CreateLogStream', 'logs:PutLogEvents'], - resources=[ - f'arn:aws:logs:{self.region}:{self.account}:log-group:/aws/codebuild/{resource_prefix}-{envname}-dbmigration*', - ], - ) - ) - - self.build_project_role.add_to_policy( - iam.PolicyStatement( - actions=[ - 'codebuild:CreateReportGroup', - 'codebuild:CreateReport', - 'codebuild:UpdateReport', - 'codebuild:BatchPutTestCases', - 'codebuild:BatchPutCodeCoverages', - ], - resources=[ - f'arn:aws:codebuild:{self.region}:{self.account}:report-group:{resource_prefix}-{envname}-dbmigration-*', - ], - ) - ) - self.codebuild_sg = ec2.SecurityGroup( - self, - f'DBMigrationCBSG{envname}', - security_group_name=f'{resource_prefix}-{envname}-cb-dbmigration-sg', - vpc=vpc, - allow_all_outbound=False, - disable_inline_rules=True, - ) - sg_connection = ec2.Connections(security_groups=[self.codebuild_sg]) - sg_connection.allow_to(vpce_connection, ec2.Port.tcp(443), 'Allow DB Migration CodeBuild to VPC Endpoint SG') - sg_connection.allow_from( - vpce_connection, - ec2.Port.tcp_range(start_port=1024, end_port=65535), - 'Allow DB Migration CodeBuild from VPC Endpoint', - ) - sg_connection.allow_to( - ec2.Connections(peer=ec2.Peer.prefix_list(s3_prefix_list)), - ec2.Port.tcp(443), - 'Allow DB Migration CodeBuild to S3 Prefix List', - ) - - self.db_migration_project = codebuild.Project( - scope=self, - id=f'DBMigrationCBProject{envname}', - project_name=f'{resource_prefix}-{envname}-dbmigration', - environment=codebuild.BuildEnvironment( - build_image=codebuild.LinuxBuildImage.AMAZON_LINUX_2_5, - ), - role=self.build_project_role.without_policy_updates(), - build_spec=codebuild.BuildSpec.from_object( - dict( - version='0.2', - phases={ - 'build': { - 'commands': [ - f'aws s3api get-object --bucket {pipeline_bucket} --key source_build.zip source_build.zip', - 'unzip source_build.zip', - 'python -m venv env', - '. env/bin/activate', - f'aws codeartifact login --tool pip --domain {codeartifact_domain_name} --domain-owner {tooling_account_id} --repository {codeartifact_pip_repo_name}', - 'pip install -r backend/requirements.txt', - 'pip install alembic', - 'export PYTHONPATH=backend', - f'export envname={envname}', - 'alembic -c backend/alembic.ini upgrade head', - ] - }, - }, - ) - ), - vpc=vpc, - subnet_selection=ec2.SubnetSelection( - subnets=vpc.select_subnets(subnet_type=ec2.SubnetType.PRIVATE_WITH_NAT).subnets - ), - security_groups=[self.codebuild_sg], - ) - - self.db_migration_project.node.add_dependency(self.build_project_role) diff --git a/deploy/stacks/pipeline.py b/deploy/stacks/pipeline.py index 459aad4ed..0afea9b06 100644 --- a/deploy/stacks/pipeline.py +++ b/deploy/stacks/pipeline.py @@ -182,10 +182,6 @@ def __init__( ) ) - self.set_db_migration_stage( - target_env, - ) - if target_env.get('enable_update_dataall_stacks_in_cicd_pipeline', False): self.set_stacks_updater_stage(target_env) @@ -653,34 +649,6 @@ def set_backend_stage(self, target_env, repository_name): ) return backend_stage - def set_db_migration_stage( - self, - target_env, - ): - migration_wave = self.pipeline.add_wave(f"{self.resource_prefix}-{target_env['envname']}-dbmigration-stage") - migration_wave.add_post( - pipelines.CodeBuildStep( - id='MigrateDB', - build_environment=codebuild.BuildEnvironment( - build_image=codebuild.LinuxBuildImage.AMAZON_LINUX_2_5, - ), - commands=[ - 'mkdir ~/.aws/ && touch ~/.aws/config', - 'echo "[profile buildprofile]" > ~/.aws/config', - f'echo "role_arn = arn:aws:iam::{target_env["account"]}:role/{self.resource_prefix}-{target_env["envname"]}-cb-dbmigration-role" >> ~/.aws/config', - 'echo "credential_source = EcsContainer" >> ~/.aws/config', - 'aws sts get-caller-identity --profile buildprofile', - f'aws codebuild start-build --project-name {self.resource_prefix}-{target_env["envname"]}-dbmigration --profile buildprofile --region {target_env.get("region", self.region)} > codebuild-id.json', - f'aws codebuild batch-get-builds --ids $(jq -r .build.id codebuild-id.json) --profile buildprofile --region {target_env.get("region", self.region)} > codebuild-output.json', - f'while [ "$(jq -r .builds[0].buildStatus codebuild-output.json)" != "SUCCEEDED" ] && [ "$(jq -r .builds[0].buildStatus codebuild-output.json)" != "FAILED" ]; do echo "running migration"; aws codebuild batch-get-builds --ids $(jq -r .build.id codebuild-id.json) --profile buildprofile --region {target_env.get("region", self.region)} > codebuild-output.json; echo "$(jq -r .builds[0].buildStatus codebuild-output.json)"; sleep 5; done', - 'if [ "$(jq -r .builds[0].buildStatus codebuild-output.json)" = "FAILED" ]; then echo "Failed"; cat codebuild-output.json; exit -1; fi', - 'cat codebuild-output.json ', - ], - role=self.expanded_codebuild_role.without_policy_updates(), - vpc=self.vpc, - ), - ) - def set_stacks_updater_stage( self, target_env, diff --git a/deploy/stacks/trigger_function_stack.py b/deploy/stacks/trigger_function_stack.py index 23195e6fc..70924a474 100644 --- a/deploy/stacks/trigger_function_stack.py +++ b/deploy/stacks/trigger_function_stack.py @@ -26,6 +26,7 @@ def __init__( vpc: ec2.IVpc = None, vpce_connection: ec2.IConnectable = None, connectables: List[ec2.IConnectable] = [], + execute_after: List[Construct] = [], **kwargs, ): super().__init__(scope, id, **kwargs) @@ -54,7 +55,7 @@ def __init__( retry_attempts=0, runtime=_lambda.Runtime.FROM_IMAGE, handler=_lambda.Handler.FROM_IMAGE, - execute_after=connectables, + execute_after=execute_after, execute_on_handler_change=True, )