diff --git a/backend/dataall/core/environment/api/resolvers.py b/backend/dataall/core/environment/api/resolvers.py index fc1575e99..2c6af3f63 100644 --- a/backend/dataall/core/environment/api/resolvers.py +++ b/backend/dataall/core/environment/api/resolvers.py @@ -119,7 +119,6 @@ def create_environment(context: Context, source, input={}): environment_uri=env.environmentUri, target_type='environment', target_uri=env.environmentUri, - target_label=env.label, ) StackService.deploy_stack(targetUri=env.environmentUri) env.userRoleInEnvironment = EnvironmentPermission.Owner.value @@ -365,8 +364,7 @@ def resolve_environment_networks(context: Context, source, **kwargs): def get_environment(context: Context, source, environmentUri: str = None): - with context.engine.scoped_session() as session: - return EnvironmentService.find_environment_by_uri(session, uri=environmentUri) + return EnvironmentService.find_environment_by_uri(uri=environmentUri) def resolve_user_role(context: Context, source: Environment): @@ -481,7 +479,7 @@ def generate_environment_access_token(context, source, environmentUri: str = Non def get_environment_stack(context: Context, source: Environment, **kwargs): return StackService.get_stack_with_cfn_resources( targetUri=source.environmentUri, - environmentUri=source.environmentUri, + env=source, ) diff --git a/backend/dataall/core/environment/services/environment_service.py b/backend/dataall/core/environment/services/environment_service.py index b3976a57d..ec4cb36e3 100644 --- a/backend/dataall/core/environment/services/environment_service.py +++ b/backend/dataall/core/environment/services/environment_service.py @@ -23,8 +23,7 @@ from dataall.core.environment.db.environment_models import Environment, EnvironmentGroup from dataall.core.environment.api.enums import EnvironmentPermission, EnvironmentType -from dataall.core.stacks.db.keyvaluetag_repositories import KeyValueTag -from dataall.core.stacks.db.stack_models import Stack +from dataall.core.stacks.db.keyvaluetag_repositories import KeyValueTagRepository from dataall.core.stacks.api.enums import StackStatus from dataall.core.environment.services.managed_iam_policies import PolicyManager @@ -685,8 +684,9 @@ def get_environment_by_uri(session, uri) -> Environment: @staticmethod @ResourcePolicyService.has_resource_permission(environment_permissions.GET_ENVIRONMENT) - def find_environment_by_uri(session, uri) -> Environment: - return EnvironmentService.get_environment_by_uri(session, uri) + def find_environment_by_uri(uri) -> Environment: + with get_context().db_engine.scoped_session() as session: + return EnvironmentService.get_environment_by_uri(session, uri) @staticmethod def list_all_active_environments(session) -> [Environment]: @@ -699,11 +699,6 @@ def list_all_active_environments(session) -> [Environment]: log.info(f'Retrieved all active dataall environments {[e.AwsAccountId for e in environments]}') return environments - @staticmethod - @ResourcePolicyService.has_resource_permission(environment_permissions.GET_ENVIRONMENT) - def get_stack(session, uri, stack_uri) -> Stack: - return session.query(Stack).get(stack_uri) - @staticmethod @ResourcePolicyService.has_resource_permission(environment_permissions.DELETE_ENVIRONMENT) def delete_environment(session, uri, environment): @@ -733,7 +728,7 @@ def delete_environment(session, uri, environment): resource_prefix=environment.resourcePrefix, ).delete_all_policies() - KeyValueTag.delete_key_value_tags(session, environment.environmentUri, 'environment') + KeyValueTagRepository.delete_key_value_tags(session, environment.environmentUri, 'environment') EnvironmentResourceManager.delete_env(session, environment) EnvironmentParameterRepository(session).delete_params(environment.environmentUri) diff --git a/backend/dataall/core/stacks/api/queries.py b/backend/dataall/core/stacks/api/queries.py index f4eac4413..818bcadfc 100644 --- a/backend/dataall/core/stacks/api/queries.py +++ b/backend/dataall/core/stacks/api/queries.py @@ -15,8 +15,8 @@ name='getStackLogs', type=gql.ArrayType(gql.Ref('StackLog')), args=[ - gql.Argument(name='environmentUri', type=gql.NonNullableType(gql.String)), - gql.Argument(name='stackUri', type=gql.NonNullableType(gql.String)), + gql.Argument(name='targetUri', type=gql.NonNullableType(gql.String)), + gql.Argument(name='targetType', type=gql.NonNullableType(gql.String)), ], resolver=get_stack_logs, ) diff --git a/backend/dataall/core/stacks/api/resolvers.py b/backend/dataall/core/stacks/api/resolvers.py index 47aefe7cc..e4ea10119 100644 --- a/backend/dataall/core/stacks/api/resolvers.py +++ b/backend/dataall/core/stacks/api/resolvers.py @@ -3,31 +3,20 @@ import os from dataall.base.api.context import Context -from dataall.core.environment.db.environment_models import Environment from dataall.core.environment.services.environment_service import EnvironmentService +from dataall.core.stacks.services.keyvaluetag_service import KeyValueTagService from dataall.core.stacks.services.stack_service import StackService -from dataall.core.stacks.aws.cloudformation import CloudFormation -from dataall.core.stacks.aws.cloudwatch import CloudWatch from dataall.core.stacks.db.stack_models import Stack -from dataall.core.stacks.db.keyvaluetag_repositories import KeyValueTag -from dataall.core.stacks.db.stack_repositories import StackRepository -from dataall.base.db import exceptions +from dataall.core.stacks.aws.cloudwatch import CloudWatch from dataall.base.utils import Parameter + log = logging.getLogger(__name__) def get_stack(context: Context, source, environmentUri: str = None, stackUri: str = None): - with context.engine.scoped_session() as session: - env: Environment = EnvironmentService.get_environment_by_uri(session, environmentUri) - stack: Stack = StackRepository.get_stack_by_uri(session, stackUri) - cfn_task = StackService.save_describe_stack_task(session, env, stack, None) - CloudFormation.describe_stack_resources(engine=context.engine, task=cfn_task) - return EnvironmentService.get_stack( - session=session, - uri=environmentUri, - stack_uri=stackUri, - ) + env = EnvironmentService.find_environment_by_uri(uri=environmentUri) + return StackService.get_and_describe_stack_in_env(env, stackUri) def resolve_link(context, source, **kwargs): @@ -67,51 +56,26 @@ def resolve_task_id(context, source: Stack, **kwargs): return source.EcsTaskArn.split('/')[-1] -def get_stack_logs(context: Context, source, environmentUri: str = None, stackUri: str = None): - with context.engine.scoped_session() as session: - stack = EnvironmentService.get_stack(session=session, uri=environmentUri, stack_uri=stackUri) - if not stack.EcsTaskArn: - raise exceptions.AWSResourceNotFound( - action='GET_STACK_LOGS', - message='Logs could not be found for this stack', - ) - - query = f"""fields @timestamp, @message, @logStream, @log as @logGroup - | sort @timestamp asc - | filter @logStream like "{stack.EcsTaskArn.split('/')[-1]}" - """ - envname = os.getenv('envname', 'local') - results = CloudWatch.run_query( - query=query, - log_group_name=f"/{Parameter().get_parameter(env=envname, path='resourcePrefix')}/{envname}/ecs/cdkproxy", - days=1, - ) - log.info(f'Running Logs query {query}') - return results +def get_stack_logs(context: Context, source, targetUri: str = None, targetType: str = None): + query = StackService.get_stack_logs(target_uri=targetUri, target_type=targetType) + envname = os.getenv('envname', 'local') + log_group_name = f"/{Parameter().get_parameter(env=envname, path='resourcePrefix')}/{envname}/ecs/cdkproxy" + results = CloudWatch.run_query( + query=query, + log_group_name=log_group_name, + days=1, + ) + log.info(f'Running Logs query {query} for log_group_name={log_group_name}') + return results def update_stack(context: Context, source, targetUri: str = None, targetType: str = None): - with context.engine.scoped_session() as session: - stack = StackRepository.update_stack(session=session, uri=targetUri, target_type=targetType) - StackService.deploy_stack(stack.targetUri) - return stack + return StackService.update_stack_by_target_uri(targetUri, targetType) def list_key_value_tags(context: Context, source, targetUri: str = None, targetType: str = None): - with context.engine.scoped_session() as session: - return KeyValueTag.list_key_value_tags( - session=session, - uri=targetUri, - target_type=targetType, - ) + return KeyValueTagService.list_key_value_tags(targetUri, targetType) def update_key_value_tags(context: Context, source, input=None): - with context.engine.scoped_session() as session: - kv_tags = KeyValueTag.update_key_value_tags( - session=session, - uri=input['targetUri'], - data=input, - ) - StackService.deploy_stack(targetUri=input['targetUri']) - return kv_tags + return StackService.update_stack_tags(input) diff --git a/backend/dataall/core/stacks/db/keyvaluetag_repositories.py b/backend/dataall/core/stacks/db/keyvaluetag_repositories.py index ecf518472..13bca32c3 100644 --- a/backend/dataall/core/stacks/db/keyvaluetag_repositories.py +++ b/backend/dataall/core/stacks/db/keyvaluetag_repositories.py @@ -1,85 +1,31 @@ import logging -from dataall.base.context import get_context -from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService -from dataall.core.stacks.db import stack_models as models -from dataall.core.stacks.db.target_type_repositories import TargetType -from dataall.base.db import exceptions +from dataall.core.stacks.db.stack_models import KeyValueTag +from typing import List logger = logging.getLogger(__name__) -class KeyValueTag: +class KeyValueTagRepository: @staticmethod - def update_key_value_tags(session, uri: str, data: dict = None) -> [models.KeyValueTag]: - if not uri: - raise exceptions.RequiredParameter('targetUri') - if not data: - raise exceptions.RequiredParameter('data') - if not data.get('targetType'): - raise exceptions.RequiredParameter('targetType') - - context = get_context() - ResourcePolicyService.check_user_resource_permission( - session=session, - username=context.username, - groups=context.groups, - resource_uri=uri, - permission_name=TargetType.get_resource_update_permission_name(data['targetType']), - ) - - tag_keys = [tag['key'].lower() for tag in data.get('tags', [])] - if tag_keys and len(tag_keys) != len(set(tag_keys)): - raise exceptions.UnauthorizedOperation( - action='SAVE_KEY_VALUE_TAGS', - message='Duplicate tag keys found. Please note that Tag keys are case insensitive', - ) - - tags = [] - session.query(models.KeyValueTag).filter( - models.KeyValueTag.targetUri == uri, - models.KeyValueTag.targetType == data['targetType'], - ).delete() - for tag in data.get('tags'): - kv_tag: models.KeyValueTag = models.KeyValueTag( - targetUri=uri, targetType=data['targetType'], key=tag['key'], value=tag['value'], cascade=tag['cascade'] - ) - tags.append(kv_tag) - session.add(kv_tag) - - return tags - - @staticmethod - def list_key_value_tags(session, uri, target_type) -> dict: - context = get_context() - ResourcePolicyService.check_user_resource_permission( - session=session, - username=context.username, - groups=context.groups, - resource_uri=uri, - permission_name=TargetType.get_resource_read_permission_name(target_type), - ) - return KeyValueTag.find_key_value_tags(session, uri, target_type) - - @staticmethod - def find_key_value_tags(session, target_uri, target_type) -> [models.KeyValueTag]: + def find_key_value_tags(session, target_uri, target_type) -> List[KeyValueTag]: return ( - session.query(models.KeyValueTag) + session.query(KeyValueTag) .filter( - models.KeyValueTag.targetUri == target_uri, - models.KeyValueTag.targetType == target_type, + KeyValueTag.targetUri == target_uri, + KeyValueTag.targetType == target_type, ) .all() ) @staticmethod - def find_environment_cascade_key_value_tags(session, target_uri) -> [models.KeyValueTag]: + def find_environment_cascade_key_value_tags(session, target_uri) -> List[KeyValueTag]: return ( - session.query(models.KeyValueTag) + session.query(KeyValueTag) .filter( - models.KeyValueTag.targetUri == target_uri, - models.KeyValueTag.targetType == 'environment', - models.KeyValueTag.cascade.is_(True), + KeyValueTag.targetUri == target_uri, + KeyValueTag.targetType == 'environment', + KeyValueTag.cascade.is_(True), ) .all() ) @@ -87,10 +33,10 @@ def find_environment_cascade_key_value_tags(session, target_uri) -> [models.KeyV @staticmethod def delete_key_value_tags(session, target_uri, target_type): return ( - session.query(models.KeyValueTag) + session.query(KeyValueTag) .filter( - models.KeyValueTag.targetUri == target_uri, - models.KeyValueTag.targetType == target_type, + KeyValueTag.targetUri == target_uri, + KeyValueTag.targetType == target_type, ) .delete() ) diff --git a/backend/dataall/core/stacks/db/stack_repositories.py b/backend/dataall/core/stacks/db/stack_repositories.py index 0c0e6a43b..203aa06ce 100644 --- a/backend/dataall/core/stacks/db/stack_repositories.py +++ b/backend/dataall/core/stacks/db/stack_repositories.py @@ -42,7 +42,7 @@ def find_stack_by_uri(session, stack_uri): return stack @staticmethod - def create_stack(session, environment_uri, target_label, target_uri, target_type, payload=None) -> models.Stack: + def create_stack(session, environment_uri, target_uri, target_type, payload=None) -> models.Stack: environment: Environment = session.query(Environment).get(environment_uri) if not environment: raise exceptions.ObjectNotFound('Environment', environment_uri) @@ -63,21 +63,3 @@ def create_stack(session, environment_uri, target_label, target_uri, target_type session.add(stack) session.commit() return stack - - @staticmethod - def update_stack(session, uri: str, target_type: str) -> [models.Stack]: - if not uri: - raise exceptions.RequiredParameter('targetUri') - if not target_type: - raise exceptions.RequiredParameter('targetType') - - context = get_context() - ResourcePolicyService.check_user_resource_permission( - session=session, - username=context.username, - groups=context.groups, - resource_uri=uri, - permission_name=TargetType.get_resource_update_permission_name(target_type), - ) - stack = StackRepository.get_stack_by_target_uri(session, target_uri=uri) - return stack diff --git a/backend/dataall/core/stacks/services/keyvaluetag_service.py b/backend/dataall/core/stacks/services/keyvaluetag_service.py new file mode 100644 index 000000000..cfe4035af --- /dev/null +++ b/backend/dataall/core/stacks/services/keyvaluetag_service.py @@ -0,0 +1,78 @@ +from typing import List + + +from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService +from dataall.core.stacks.db.keyvaluetag_repositories import KeyValueTagRepository +from dataall.core.stacks.db.stack_models import KeyValueTag +from dataall.base.db.exceptions import RequiredParameter, UnauthorizedOperation +from dataall.base.context import get_context +from dataall.core.stacks.db.target_type_repositories import TargetType + + +class KeyValueTagParamValidationService: + @staticmethod + def validate_update_param(uri: str, data: dict): + if not uri: + raise RequiredParameter('targetUri') + if not data: + raise RequiredParameter('data') + if not data.get('targetType'): + raise RequiredParameter('targetType') + + @staticmethod + def verify_target_type_and_uri(target_type, target_uri): + if not target_uri: + raise RequiredParameter('targetUri') + if not target_type: + raise RequiredParameter('targetType') + + +class KeyValueTagService: + @staticmethod + def update_key_value_tags(uri: str, data: dict = None) -> List[KeyValueTag]: + KeyValueTagParamValidationService.validate_update_param(uri, data) + context = get_context() + with context.db_engine.scoped_session() as session: + ResourcePolicyService.check_user_resource_permission( + session=session, + username=context.username, + groups=context.groups, + resource_uri=uri, + permission_name=TargetType.get_resource_update_permission_name(data['targetType']), + ) + + tag_keys = [tag['key'].lower() for tag in data.get('tags', [])] + if tag_keys and len(tag_keys) != len(set(tag_keys)): + raise UnauthorizedOperation( + action='SAVE_KEY_VALUE_TAGS', + message='Duplicate tag keys found. Please note that Tag keys are case insensitive', + ) + + tags = [] + KeyValueTagRepository.delete_key_value_tags(session, uri, data['targetType']) + for tag in data.get('tags'): + kv_tag: KeyValueTag = KeyValueTag( + targetUri=uri, + targetType=data['targetType'], + key=tag['key'], + value=tag['value'], + cascade=tag['cascade'], + ) + tags.append(kv_tag) + session.add(kv_tag) + + return tags + + @staticmethod + def list_key_value_tags(target_uri, target_type) -> dict: + KeyValueTagParamValidationService.verify_target_type_and_uri(target_uri, target_type) + context = get_context() + with context.db_engine.scoped_session() as session: + ResourcePolicyService.check_user_resource_permission( + session=session, + username=context.username, + groups=context.groups, + resource_uri=target_uri, + permission_name=TargetType.get_resource_read_permission_name(target_type), + ) + return KeyValueTagRepository.find_key_value_tags(session, target_uri, target_type) diff --git a/backend/dataall/core/stacks/services/runtime_stacks_tagging.py b/backend/dataall/core/stacks/services/runtime_stacks_tagging.py index 19b5d66d7..6d7c3b1ee 100644 --- a/backend/dataall/core/stacks/services/runtime_stacks_tagging.py +++ b/backend/dataall/core/stacks/services/runtime_stacks_tagging.py @@ -8,7 +8,7 @@ from dataall.core.environment.db.environment_models import Environment from dataall.core.environment.services.environment_service import EnvironmentService from dataall.core.organizations.db.organization_repositories import OrganizationRepository -from dataall.core.stacks.db.keyvaluetag_repositories import KeyValueTag +from dataall.core.stacks.db.keyvaluetag_repositories import KeyValueTagRepository from dataall.core.stacks.db.stack_models import KeyValueTag as KeyValueTagModel @@ -133,7 +133,7 @@ def get_environment(cls, session, target_stack): def get_model_key_value_tags(cls, session, stack, target_type): return [ (kv.key, kv.value) - for kv in KeyValueTag.find_key_value_tags( + for kv in KeyValueTagRepository.find_key_value_tags( session, stack.target_uri, target_type, @@ -144,7 +144,7 @@ def get_model_key_value_tags(cls, session, stack, target_type): def get_environment_cascade_key_value_tags(cls, session, environmentUri): return [ (kv.key, kv.value) - for kv in KeyValueTag.find_environment_cascade_key_value_tags( + for kv in KeyValueTagRepository.find_environment_cascade_key_value_tags( session, environmentUri, ) diff --git a/backend/dataall/core/stacks/services/stack_service.py b/backend/dataall/core/stacks/services/stack_service.py index 419302495..0f2b88a67 100644 --- a/backend/dataall/core/stacks/services/stack_service.py +++ b/backend/dataall/core/stacks/services/stack_service.py @@ -1,25 +1,53 @@ import os import requests +import logging -from dataall.core.environment.db.environment_repositories import EnvironmentRepository +from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService +from dataall.core.stacks.aws.cloudformation import CloudFormation +from dataall.core.stacks.services.keyvaluetag_service import KeyValueTagService from dataall.core.tasks.service_handlers import Worker from dataall.base.config import config from dataall.base.context import get_context -from dataall.core.environment.db.environment_models import Environment from dataall.core.stacks.aws.ecs import Ecs from dataall.core.stacks.db.stack_repositories import StackRepository from dataall.core.stacks.db.stack_models import Stack from dataall.core.tasks.db.task_models import Task from dataall.base.utils import Parameter +from dataall.base.db.exceptions import AWSResourceNotFound +from dataall.base.db.exceptions import RequiredParameter +from dataall.core.stacks.db.target_type_repositories import TargetType +from dataall.core.environment.db.environment_models import Environment + +log = logging.getLogger(__name__) + + +class StackRequestVerifier: + @staticmethod + def verify_get_and_describe_params(env_uri, stack_uri): + if not env_uri: + raise RequiredParameter('Environment URI is required') + if not stack_uri: + raise RequiredParameter('Stack URI is required') + + @staticmethod + def validate_update_tag_input(data): + if not data.get('targetUri'): + raise ValueError('targetUri is required') + + @staticmethod + def verify_target_type_and_uri(target_type, target_uri): + if not target_uri: + raise RequiredParameter('targetUri') + if not target_type: + raise RequiredParameter('targetType') class StackService: @staticmethod - def get_stack_with_cfn_resources(targetUri: str, environmentUri: str): + def get_stack_with_cfn_resources(targetUri: str, env: Environment): context = get_context() with context.db_engine.scoped_session() as session: - env: Environment = EnvironmentRepository.get_environment_by_uri(session, environmentUri) stack: Stack = StackRepository.find_stack_by_target_uri(session, target_uri=targetUri) if not stack: stack = Stack( @@ -99,3 +127,73 @@ def delete_stack(target_uri, accountid, cdk_role_arn, region): Worker.queue(context.db_engine, [task.taskUri]) return True + + @staticmethod + def get_stack_by_uri(stack_uri): + with get_context().db_engine.scoped_session() as session: + return StackRepository.get_stack_by_uri(session, stack_uri) + + @staticmethod + def get_and_describe_stack_in_env(env: Environment, stack_uri): + StackRequestVerifier.verify_get_and_describe_params(env.environmentUri, stack_uri) + stack: Stack = StackService.get_stack_by_uri(stack_uri) + with get_context().db_engine.scoped_session() as session: + cfn_task = StackService.save_describe_stack_task(session, env, stack, None) + CloudFormation.describe_stack_resources(engine=get_context().db_engine, task=cfn_task) + return stack + + @staticmethod + def update_stack_by_target_uri(target_uri, target_type): + StackRequestVerifier.verify_target_type_and_uri(target_uri, target_type) + context = get_context() + with context.db_engine.scoped_session() as session: + ResourcePolicyService.check_user_resource_permission( + session=session, + username=context.username, + groups=context.groups, + resource_uri=target_uri, + permission_name=TargetType.get_resource_update_permission_name(target_type), + ) + stack = StackRepository.get_stack_by_target_uri(session, target_uri=target_uri) + StackService.deploy_stack(stack.targetUri) + return stack + + @staticmethod + def update_stack_tags(input): + StackRequestVerifier.validate_update_tag_input(input) + target_uri = input.get('targetUri') + kv_tags = KeyValueTagService.update_key_value_tags( + uri=target_uri, + data=input, + ) + StackService.deploy_stack(targetUri=target_uri) + return kv_tags + + @staticmethod + def get_stack_logs(target_uri, target_type): + context = get_context() + StackRequestVerifier.verify_target_type_and_uri(target_uri, target_type) + + with context.db_engine.scoped_session() as session: + ResourcePolicyService.check_user_resource_permission( + session=session, + username=context.username, + groups=context.groups, + resource_uri=target_uri, + permission_name=TargetType.get_resource_read_permission_name(target_type), + ) + stack = StackRepository.get_stack_by_target_uri(session, target_uri) + + if not stack.EcsTaskArn: + raise AWSResourceNotFound( + action='GET_STACK_LOGS', + message='Logs could not be found for this stack', + ) + + log.info(f'Get stack Logs for stack {stack.name}') + + query = f"""fields @timestamp, @message, @logStream, @log as @logGroup + | sort @timestamp asc + | filter @logStream like "{stack.EcsTaskArn.split('/')[-1]}" + """ + return query diff --git a/backend/dataall/modules/datapipelines/api/resolvers.py b/backend/dataall/modules/datapipelines/api/resolvers.py index f3da3e619..e242a7a47 100644 --- a/backend/dataall/modules/datapipelines/api/resolvers.py +++ b/backend/dataall/modules/datapipelines/api/resolvers.py @@ -2,6 +2,7 @@ from dataall.base.api.context import Context from dataall.base.db import exceptions +from dataall.core.environment.services.environment_service import EnvironmentService from dataall.core.stacks.services.stack_service import StackService from dataall.modules.datapipelines.api.enums import DataPipelineRole from dataall.modules.datapipelines.db.datapipelines_models import DataPipeline @@ -102,7 +103,8 @@ def resolve_clone_url_http(context: Context, source: DataPipeline, **kwargs): def resolve_stack(context, source: DataPipeline, **kwargs): if not source: return None + env = EnvironmentService.find_environment_by_uri(uri=source.environmentUri) return StackService.get_stack_with_cfn_resources( targetUri=source.DataPipelineUri, - environmentUri=source.environmentUri, + env=env, ) diff --git a/backend/dataall/modules/datapipelines/services/datapipelines_service.py b/backend/dataall/modules/datapipelines/services/datapipelines_service.py index efba37926..8da72dd16 100644 --- a/backend/dataall/modules/datapipelines/services/datapipelines_service.py +++ b/backend/dataall/modules/datapipelines/services/datapipelines_service.py @@ -7,7 +7,7 @@ from dataall.core.environment.services.environment_service import EnvironmentService from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService from dataall.core.permissions.services.tenant_policy_service import TenantPolicyService -from dataall.core.stacks.db.keyvaluetag_repositories import KeyValueTag +from dataall.core.stacks.db.keyvaluetag_repositories import KeyValueTagRepository from dataall.core.stacks.db.stack_repositories import StackRepository from dataall.core.stacks.services.stack_service import StackService from dataall.core.tasks.db.task_models import Task @@ -84,7 +84,6 @@ def create_pipeline( environment_uri=pipeline.environmentUri, target_type='cdkpipeline', target_uri=pipeline.DataPipelineUri, - target_label=pipeline.label, payload={'account': pipeline.AwsAccountId, 'region': pipeline.region}, ) else: @@ -93,7 +92,6 @@ def create_pipeline( environment_uri=pipeline.environmentUri, target_type='pipeline', target_uri=pipeline.DataPipelineUri, - target_label=pipeline.label, payload={'account': pipeline.AwsAccountId, 'region': pipeline.region}, ) @@ -226,7 +224,7 @@ def delete_pipeline(uri: str, deleteFromAWS: bool): ) DatapipelinesRepository.delete_pipeline_environments(session, uri) - KeyValueTag.delete_key_value_tags(session, pipeline.DataPipelineUri, 'pipeline') + KeyValueTagRepository.delete_key_value_tags(session, pipeline.DataPipelineUri, 'pipeline') session.delete(pipeline) diff --git a/backend/dataall/modules/datasets/api/dataset/resolvers.py b/backend/dataall/modules/datasets/api/dataset/resolvers.py index 1eba7cd8c..526c3a0f7 100644 --- a/backend/dataall/modules/datasets/api/dataset/resolvers.py +++ b/backend/dataall/modules/datasets/api/dataset/resolvers.py @@ -147,9 +147,10 @@ def generate_dataset_access_token(context, source, datasetUri: str = None): def get_dataset_stack(context: Context, source: Dataset, **kwargs): if not source: return None + env = EnvironmentService.find_environment_by_uri(uri=source.environmentUri) return StackService.get_stack_with_cfn_resources( targetUri=source.datasetUri, - environmentUri=source.environmentUri, + env=env, ) diff --git a/backend/dataall/modules/datasets/services/dataset_service.py b/backend/dataall/modules/datasets/services/dataset_service.py index 74421c807..dffbbc10c 100644 --- a/backend/dataall/modules/datasets/services/dataset_service.py +++ b/backend/dataall/modules/datasets/services/dataset_service.py @@ -14,7 +14,7 @@ from dataall.base.context import get_context from dataall.core.permissions.services.group_policy_service import GroupPolicyService from dataall.core.environment.services.environment_service import EnvironmentService -from dataall.core.stacks.db.keyvaluetag_repositories import KeyValueTag +from dataall.core.stacks.db.keyvaluetag_repositories import KeyValueTagRepository from dataall.core.stacks.db.stack_repositories import StackRepository from dataall.core.stacks.db.stack_models import Stack from dataall.core.tasks.db.task_models import Task @@ -365,9 +365,11 @@ def generate_dataset_access_token(uri): @staticmethod def get_dataset_stack(dataset: Dataset): + env = EnvironmentService.find_environment_by_uri(uri=dataset.environmentUri) + return StackService.get_stack_with_cfn_resources( targetUri=dataset.datasetUri, - environmentUri=dataset.environmentUri, + env=env, ) @staticmethod @@ -402,7 +404,7 @@ def delete_dataset(uri: str, delete_from_aws: bool = False): DatasetTableRepository.delete_dataset_tables(session, dataset.datasetUri) DatasetLocationRepository.delete_dataset_locations(session, dataset.datasetUri) DatasetBucketRepository.delete_dataset_buckets(session, dataset.datasetUri) - KeyValueTag.delete_key_value_tags(session, dataset.datasetUri, 'dataset') + KeyValueTagRepository.delete_key_value_tags(session, dataset.datasetUri, 'dataset') VoteRepository.delete_votes(session, dataset.datasetUri, 'dataset') ResourcePolicyService.delete_resource_policy( @@ -441,7 +443,6 @@ def _create_dataset_stack(session, dataset: Dataset) -> Stack: session=session, environment_uri=dataset.environmentUri, target_uri=dataset.datasetUri, - target_label=dataset.label, target_type='dataset', payload={ 'bucket_name': dataset.S3BucketName, diff --git a/backend/dataall/modules/mlstudio/api/resolvers.py b/backend/dataall/modules/mlstudio/api/resolvers.py index ef14d8f28..3c7c3e71c 100644 --- a/backend/dataall/modules/mlstudio/api/resolvers.py +++ b/backend/dataall/modules/mlstudio/api/resolvers.py @@ -2,6 +2,7 @@ from dataall.base.api.context import Context from dataall.base.db import exceptions +from dataall.core.environment.services.environment_service import EnvironmentService from dataall.core.stacks.services.stack_service import StackService from dataall.modules.mlstudio.api.enums import SagemakerStudioRole from dataall.modules.mlstudio.db.mlstudio_models import SagemakerStudioUser @@ -119,9 +120,10 @@ def resolve_sagemaker_studio_user_stack(context: Context, source: SagemakerStudi """ if not source: return None + env = EnvironmentService.find_environment_by_uri(uri=source.environmentUri) return StackService.get_stack_with_cfn_resources( targetUri=source.sagemakerStudioUserUri, - environmentUri=source.environmentUri, + env=env, ) diff --git a/backend/dataall/modules/mlstudio/services/mlstudio_service.py b/backend/dataall/modules/mlstudio/services/mlstudio_service.py index bfd2f75d1..c47df205b 100644 --- a/backend/dataall/modules/mlstudio/services/mlstudio_service.py +++ b/backend/dataall/modules/mlstudio/services/mlstudio_service.py @@ -170,7 +170,6 @@ def create_sagemaker_studio_user(*, uri: str, admin_group: str, request: Sagemak environment_uri=sagemaker_studio_user.environmentUri, target_type='mlstudio', target_uri=sagemaker_studio_user.sagemakerStudioUserUri, - target_label=sagemaker_studio_user.label, ) StackService.deploy_stack(targetUri=sagemaker_studio_user.sagemakerStudioUserUri) diff --git a/backend/dataall/modules/notebooks/api/resolvers.py b/backend/dataall/modules/notebooks/api/resolvers.py index c5270d08f..84e119cd4 100644 --- a/backend/dataall/modules/notebooks/api/resolvers.py +++ b/backend/dataall/modules/notebooks/api/resolvers.py @@ -1,5 +1,6 @@ from dataall.base.api.context import Context from dataall.base.db import exceptions +from dataall.core.environment.services.environment_service import EnvironmentService from dataall.core.stacks.services.stack_service import StackService from dataall.modules.notebooks.api.enums import SagemakerNotebookRole from dataall.modules.notebooks.db.notebook_models import SagemakerNotebook @@ -87,9 +88,10 @@ def resolve_user_role(context: Context, source: SagemakerNotebook): def resolve_notebook_stack(context: Context, source: SagemakerNotebook, **kwargs): if not source: return None + env = EnvironmentService.find_environment_by_uri(uri=source.environmentUri) return StackService.get_stack_with_cfn_resources( targetUri=source.notebookUri, - environmentUri=source.environmentUri, + env=env, ) diff --git a/backend/dataall/modules/notebooks/services/notebook_service.py b/backend/dataall/modules/notebooks/services/notebook_service.py index 36ba8ce2e..3a00e0984 100644 --- a/backend/dataall/modules/notebooks/services/notebook_service.py +++ b/backend/dataall/modules/notebooks/services/notebook_service.py @@ -14,7 +14,7 @@ from dataall.core.environment.services.environment_service import EnvironmentService from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService from dataall.core.permissions.services.tenant_policy_service import TenantPolicyService -from dataall.core.stacks.db.keyvaluetag_repositories import KeyValueTag +from dataall.core.stacks.db.keyvaluetag_repositories import KeyValueTagRepository from dataall.core.stacks.db.stack_repositories import StackRepository from dataall.base.db import exceptions from dataall.core.stacks.services.stack_service import StackService @@ -143,7 +143,6 @@ def create_notebook(*, uri: str, admin_group: str, request: NotebookCreationRequ environment_uri=notebook.environmentUri, target_type='notebook', target_uri=notebook.notebookUri, - target_label=notebook.label, ) StackService.deploy_stack(targetUri=notebook.notebookUri) @@ -199,7 +198,7 @@ def delete_notebook(*, uri: str, delete_from_aws: bool): """Deletes notebook from the database and if delete_from_aws is True from AWS as well""" with _session() as session: notebook = NotebookService._get_notebook(session, uri) - KeyValueTag.delete_key_value_tags(session, notebook.notebookUri, 'notebook') + KeyValueTagRepository.delete_key_value_tags(session, notebook.notebookUri, 'notebook') session.delete(notebook) ResourcePolicyService.delete_resource_policy( diff --git a/frontend/src/modules/Shared/Stack/Stack.js b/frontend/src/modules/Shared/Stack/Stack.js index fc75b6d96..bad40102b 100644 --- a/frontend/src/modules/Shared/Stack/Stack.js +++ b/frontend/src/modules/Shared/Stack/Stack.js @@ -257,6 +257,7 @@ export const Stack = (props) => { diff --git a/frontend/src/modules/Shared/Stack/StackLogs.js b/frontend/src/modules/Shared/Stack/StackLogs.js index fcd8037b9..be9bd2976 100644 --- a/frontend/src/modules/Shared/Stack/StackLogs.js +++ b/frontend/src/modules/Shared/Stack/StackLogs.js @@ -15,7 +15,7 @@ import { SET_ERROR, useDispatch } from 'globalErrors'; import { getStackLogs, useClient } from 'services'; export const StackLogs = (props) => { - const { environmentUri, stack, onClose, open } = props; + const { environmentUri, stack, targetType, onClose, open } = props; const { settings } = useSettings(); const client = useClient(); const dispatch = useDispatch(); @@ -26,7 +26,7 @@ export const StackLogs = (props) => { setLoading(true); try { const response = await client.query( - getStackLogs(environmentUri, stack.stackUri) + getStackLogs(stack.targetUri, targetType) ); if (response && !response.errors) { setLogs(response.data.getStackLogs.map((l) => l.message)); diff --git a/frontend/src/services/graphql/Stack/getStackLogs.js b/frontend/src/services/graphql/Stack/getStackLogs.js index 01ee69dce..970abaae9 100644 --- a/frontend/src/services/graphql/Stack/getStackLogs.js +++ b/frontend/src/services/graphql/Stack/getStackLogs.js @@ -1,13 +1,13 @@ import { gql } from 'apollo-boost'; -export const getStackLogs = (environmentUri, stackUri) => ({ +export const getStackLogs = (targetUri, targetType) => ({ variables: { - environmentUri, - stackUri + targetUri, + targetType }, query: gql` - query getStackLogs($environmentUri: String!, $stackUri: String!) { - getStackLogs(environmentUri: $environmentUri, stackUri: $stackUri) { + query getStackLogs($targetUri: String!, $targetType: String!) { + getStackLogs(targetUri: $targetUri, targetType: $targetType) { message timestamp } diff --git a/tests/modules/conftest.py b/tests/modules/conftest.py index 738b4666c..499c7bd26 100644 --- a/tests/modules/conftest.py +++ b/tests/modules/conftest.py @@ -70,7 +70,6 @@ def _create_env_stack(session, env): environment_uri=env.environmentUri, target_type='environment', target_uri=env.environmentUri, - target_label=env.label, )