From a1769490e3334d2b07388e4b1d32eb9c287a4c6d Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Tue, 9 Apr 2024 23:53:31 +0530 Subject: [PATCH 01/36] Maintenance Window Frontend Changes - 1 --- .../design/components/layout/DefaultNavbar.js | 9 +- .../components/MaintenanceViewer.js | 148 ++++++++++++++++++ .../Administration/components/index.js | 1 + .../views/AdministrationView.js | 6 +- .../graphql/MaintenanceWindow/index.js | 1 + .../MaintenanceWindow/isMaintenanceMode.js | 12 ++ 6 files changed, 174 insertions(+), 3 deletions(-) create mode 100644 frontend/src/modules/Administration/components/MaintenanceViewer.js create mode 100644 frontend/src/services/graphql/MaintenanceWindow/index.js create mode 100644 frontend/src/services/graphql/MaintenanceWindow/isMaintenanceMode.js diff --git a/frontend/src/design/components/layout/DefaultNavbar.js b/frontend/src/design/components/layout/DefaultNavbar.js index b851d267e..761beb45a 100644 --- a/frontend/src/design/components/layout/DefaultNavbar.js +++ b/frontend/src/design/components/layout/DefaultNavbar.js @@ -1,5 +1,5 @@ import React from 'react'; -import { AppBar, Box, IconButton, Toolbar } from '@mui/material'; +import {AppBar, Box, IconButton, Toolbar, Typography} from '@mui/material'; import { makeStyles } from '@mui/styles'; import { Menu } from '@mui/icons-material'; import PropTypes from 'prop-types'; @@ -7,6 +7,7 @@ import { AccountPopover, NotificationsPopover } from '../popovers'; import { Logo } from '../Logo'; import { SettingsDrawer } from '../SettingsDrawer'; import { ModuleNames, isModuleEnabled } from 'utils'; +import {isMaintenanceMode} from "../../../services/graphql/MaintenanceWindow"; const useStyles = makeStyles((theme) => ({ appBar: { @@ -20,6 +21,12 @@ export const DefaultNavbar = ({ openDrawer, onOpenDrawerChange }) => { return ( + {isMaintenanceMode() ? + + data.all is in maintenance mode. You can still navigate inside data.all but during this period, please do not make any modifications to any data.all assets ( datasets, environment, etc ). + + : <>} + {!openDrawer && ( { + + return ( + + + + + Are you sure ? + + }/> + + + + + + + ) +} + +export const MaintenanceViewer = () => { + + const [updating, setUpdating] = useState(false); + const [mode, setMode] = useState('') + const [popUp, setPopUp] = useState(false) + const [confirmedMode, setConfirmedMode] = useState('') + + const refreshMaintenanceView = () =>{ + console.log("Refreshing the maintenance view now!!!") + return true + } + + const startMaintenanceWindow = () => { + setConfirmedMode(mode) + setPopUp(true) + return true; + } + + const refreshWindow = () =>{ + setUpdating(true) + console.log("Refreshing the page") + setTimeout(() =>{ + setUpdating(false) + }, 2000) + + } + + return ( + + + + Create a Maintenance Window + + } /> + + + + + {setMode(event.target.value)}} + select + value={mode} + variant="outlined" + > + {maintenanceModes.map((group) => ( + + {group.label} + + ))} + + + + {/**/} + + + } + sx={{ m: 1 }} + variant="contained" + > + Refresh + + + + + + + Status : + + + | + + + Maintenance Mode : {confirmedMode} + + + + + + + + + ) +} \ No newline at end of file diff --git a/frontend/src/modules/Administration/components/index.js b/frontend/src/modules/Administration/components/index.js index 06ef79939..27cd64355 100644 --- a/frontend/src/modules/Administration/components/index.js +++ b/frontend/src/modules/Administration/components/index.js @@ -1,3 +1,4 @@ export * from './AdministrationTeams'; export * from './AdministratorDashboardViewer'; export * from './TeamPermissionsEditForm'; +export * from './MaintenanceViewer' diff --git a/frontend/src/modules/Administration/views/AdministrationView.js b/frontend/src/modules/Administration/views/AdministrationView.js index cc0235d07..15931713f 100644 --- a/frontend/src/modules/Administration/views/AdministrationView.js +++ b/frontend/src/modules/Administration/views/AdministrationView.js @@ -13,11 +13,12 @@ import { useState } from 'react'; import { Helmet } from 'react-helmet-async'; import { Link as RouterLink } from 'react-router-dom'; import { ChevronRightIcon, useSettings } from 'design'; -import { AdministrationTeams, DashboardViewer } from '../components'; +import { AdministrationTeams, DashboardViewer, MaintenanceViewer } from '../components'; const tabs = [ { label: 'Teams', value: 'teams' }, - { label: 'Monitoring', value: 'dashboard' } + { label: 'Monitoring', value: 'dashboard' }, + { label: 'Maintenance', value: 'maintenance' } ]; const AdministrationView = () => { @@ -90,6 +91,7 @@ const AdministrationView = () => { {currentTab === 'teams' && } {currentTab === 'dashboard' && } + {currentTab === 'maintenance' && } diff --git a/frontend/src/services/graphql/MaintenanceWindow/index.js b/frontend/src/services/graphql/MaintenanceWindow/index.js new file mode 100644 index 000000000..dd431b117 --- /dev/null +++ b/frontend/src/services/graphql/MaintenanceWindow/index.js @@ -0,0 +1 @@ +export * from './isMaintenanceMode' \ No newline at end of file diff --git a/frontend/src/services/graphql/MaintenanceWindow/isMaintenanceMode.js b/frontend/src/services/graphql/MaintenanceWindow/isMaintenanceMode.js new file mode 100644 index 000000000..2eeddf25a --- /dev/null +++ b/frontend/src/services/graphql/MaintenanceWindow/isMaintenanceMode.js @@ -0,0 +1,12 @@ +import { gql } from 'apollo-boost'; +export const isMaintenanceMode = () =>{ + return true; +} +// export const isMaintenanceMode = () => ({ +// query: gql` +// query isMaintenanceMode { +// isMaintenanceMode{ +// inMaintenance +// } +// }` +// }) \ No newline at end of file From dc1eb1a9ad52be1b43f6574c27a0e4f6081b85a6 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Tue, 16 Apr 2024 19:12:26 +0530 Subject: [PATCH 02/36] Frontend views + few backend files comments for code --- backend/api_handler.py | 5 + backend/search_handler.py | 5 + frontend/src/App.js | 6 +- .../authentication/components/AuthGuard.js | 14 ++ .../components/NoAccessMaintenanceWindow.js | 27 +++ frontend/src/design/components/index.js | 1 + .../components/MaintenanceViewer.js | 195 +++++++++++++++--- .../MaintenanceWindow/isMaintenanceMode.js | 2 +- 8 files changed, 225 insertions(+), 30 deletions(-) create mode 100644 frontend/src/design/components/NoAccessMaintenanceWindow.js diff --git a/backend/api_handler.py b/backend/api_handler.py index 9889e1bcf..53b005a2a 100644 --- a/backend/api_handler.py +++ b/backend/api_handler.py @@ -165,6 +165,11 @@ def handler(event, context): 'schema': SCHEMA, } + # If maintenance mode is enabled -> Check Status by using the graphQL Endpoint + # If groups doesn't contain data.all administrator group + # Check what is the access mode + # Return response with error "Maintenance Window is ON" + # Determine if there are any Operations that Require ReAuth From SSM Parameter try: reauth_apis = ParameterStoreManager.get_parameter_value( diff --git a/backend/search_handler.py b/backend/search_handler.py index 5a3a1318c..d5bf0d7d0 100644 --- a/backend/search_handler.py +++ b/backend/search_handler.py @@ -21,6 +21,11 @@ def handler(event, context): }, } elif event['httpMethod'] == 'POST': + # If maintenance mode is enabled -> Check Status by using the graphQL Endpoint + # If groups doesn't contain data.all administrator group + # Check what is the access mode + # Return response with error "Maintenance Window is ON" + body = event.get('body') print(body) success = True diff --git a/frontend/src/App.js b/frontend/src/App.js index d8d9c2374..1329c8b46 100644 --- a/frontend/src/App.js +++ b/frontend/src/App.js @@ -5,10 +5,11 @@ import { createMaterialTheme, useScrollReset, useSettings, - LoadingScreen + LoadingScreen, SplashScreen } from './design'; import routes from './routes'; import { useAuth } from './authentication'; +import {isMaintenanceMode} from "./services/graphql/MaintenanceWindow"; export const App = () => { const content = useRoutes(routes); @@ -26,7 +27,8 @@ export const App = () => { return ( - {auth.isInitialized ? content : } + {/*{auth.isInitialized ? isMaintenanceMode() ? content : : }*/} + {auth.isInitialized ? content : } ); }; diff --git a/frontend/src/authentication/components/AuthGuard.js b/frontend/src/authentication/components/AuthGuard.js index 8410407ab..b1e74b0ff 100644 --- a/frontend/src/authentication/components/AuthGuard.js +++ b/frontend/src/authentication/components/AuthGuard.js @@ -7,12 +7,16 @@ import { RegexToValidateWindowPathName, WindowPathLengthThreshold } from '../../utils'; +import {isMaintenanceMode} from "../../services/graphql/MaintenanceWindow"; +import {useClient} from "../../services"; +import {NoAccessMaintenanceWindow} from "../../design"; export const AuthGuard = (props) => { const { children } = props; const auth = useAuth(); const location = useLocation(); const [requestedLocation, setRequestedLocation] = useState(null); + const client = useClient(); if (!auth.isAuthenticated) { if (location.pathname !== requestedLocation) { @@ -55,6 +59,16 @@ export const AuthGuard = (props) => { sessionStorage.removeItem('window-location'); } + // Check if the maintenance window is enabled and has NO-ACCESS Status + // If yes then display a blank screen with a message that data.all is in maintenance mode + if (client){ + // Replace this call with query call getting mode + if (isMaintenanceMode()){ + return + } + } + + return <>{children}; }; diff --git a/frontend/src/design/components/NoAccessMaintenanceWindow.js b/frontend/src/design/components/NoAccessMaintenanceWindow.js new file mode 100644 index 000000000..e71145ba6 --- /dev/null +++ b/frontend/src/design/components/NoAccessMaintenanceWindow.js @@ -0,0 +1,27 @@ +import {Box, Typography} from '@mui/material'; +import { Logo } from './Logo'; +import React from "react"; + +export const NoAccessMaintenanceWindow = () => ( + + + data.all is in maintenance mode. Please contact data.all administrators for any assistance. + + + +); diff --git a/frontend/src/design/components/index.js b/frontend/src/design/components/index.js index b9c460dd7..064ad7229 100644 --- a/frontend/src/design/components/index.js +++ b/frontend/src/design/components/index.js @@ -29,3 +29,4 @@ export * from './defaults'; export * from './layout'; export * from './popovers'; export * from './SanitizedHTML'; +export * from './NoAccessMaintenanceWindow' diff --git a/frontend/src/modules/Administration/components/MaintenanceViewer.js b/frontend/src/modules/Administration/components/MaintenanceViewer.js index b8d5540a2..392736697 100644 --- a/frontend/src/modules/Administration/components/MaintenanceViewer.js +++ b/frontend/src/modules/Administration/components/MaintenanceViewer.js @@ -1,15 +1,52 @@ -import {Box, Button, Card, CardHeader, Dialog, Divider, MenuItem, TextField, Typography} from "@mui/material"; -import React, {useState} from "react"; -import {Article, SystemUpdate} from "@mui/icons-material"; +import { + Box, + Button, + Card, + CardHeader, + CircularProgress, + Dialog, + Divider, + Grid, IconButton, + MenuItem, + TextField, + Typography +} from "@mui/material"; +import React, {useCallback, useEffect, useState} from "react"; +import {Article, CancelRounded, SystemUpdate} from "@mui/icons-material"; import {LoadingButton} from "@mui/lab"; import {Label} from "../../../design"; +import {isMaintenanceMode} from "../../../services/graphql/MaintenanceWindow"; +import {useClient} from "../../../services"; +import {SET_ERROR, useDispatch} from "../../../globalErrors"; +import {useSnackbar} from "notistack"; const maintenanceModes = [ {value: "READ-ONLY", label: "Read-Only"}, {value: "NO-ACCESS", label: "No-Access"} ] -export const MaintenanceConfirmationPopUp = ({popUp, setPopUp}) => { +export const MaintenanceConfirmationPopUp = ({popUp, setPopUp, mode, confirmedMode, setConfirmedMode, maintenanceButtonText, setMaintenanceButtonText, setDropDownStatus, refreshingTimer, startRefreshPolling}) => { + + const handlePopUpModal = () => { + // Call the GrapQL API and then after the success is received change the UI + if (maintenanceButtonText === 'Start Maintenance') { + // Call the GraphQL to enable maintenance window + setMaintenanceButtonText('End Maintenance') + // Freeze the dropdown menu + setDropDownStatus(true) + // Start the Timer + startRefreshPolling() + }else if (maintenanceButtonText === 'End Maintenance'){ + // Call the GraphQL to disable maintenance window + setMaintenanceButtonText('Start Maintenance') + // Unfreeze the dropdown menu + setDropDownStatus(false) + // End the running timer as well + clearInterval(refreshingTimer) + } + setConfirmedMode(mode) + setPopUp(false) + } return ( @@ -17,20 +54,21 @@ export const MaintenanceConfirmationPopUp = ({popUp, setPopUp}) => { - Are you sure ? + Are you sure you want to {maintenanceButtonText.toLowerCase()}? }/> - - + @@ -46,37 +85,136 @@ export const MaintenanceConfirmationPopUp = ({popUp, setPopUp}) => { } export const MaintenanceViewer = () => { - + const client = useClient(); const [updating, setUpdating] = useState(false); const [mode, setMode] = useState('') const [popUp, setPopUp] = useState(false) const [confirmedMode, setConfirmedMode] = useState('') + const [maintenanceButtonText, setMaintenanceButtonText] = useState('Start Maintenance') + const [maintenanceWindowStatus, setMaintenanceWindowStatus] = useState('NOT-IN-MAINTENANCE') + const [dropDownStatus, setDropDownStatus] = useState(false) + const [refreshingTimer, setRefreshingTimer] = useState('') + const { enqueueSnackbar, closeSnackbar } = useSnackbar(); + const dispatch = useDispatch(); - const refreshMaintenanceView = () =>{ + const refreshMaintenanceView = async () =>{ console.log("Refreshing the maintenance view now!!!") + // Call the que + setUpdating(true) + setTimeout(() =>{ + setUpdating(false) + }, 2000) + + refreshStatus().catch((e) => dispatch({ type: SET_ERROR, error: e.message })) return true } const startMaintenanceWindow = () => { - setConfirmedMode(mode) + // Check if proper maintenance mode is selected + // Use Formik forms for this in the future + console.log(`value of the mode is ${mode}`) + if (!['READ-ONLY', 'NO-ACCESS'].includes(mode) && maintenanceButtonText === 'Start Maintenance'){ + dispatch({ type: SET_ERROR, error: 'Please select correct maintenance mode' }) + return false; + } setPopUp(true) return true; } - const refreshWindow = () =>{ - setUpdating(true) - console.log("Refreshing the page") - setTimeout(() =>{ - setUpdating(false) - }, 2000) + const startRefreshPolling = useCallback( + async () => { + console.log("I am here in the refresh polling ") + if (client){ + const setTimer = setInterval(() => { + refreshStatus().catch((e) => dispatch({ type: SET_ERROR, error: e.message }))} + , [10000]) + setRefreshingTimer(setTimer) + } + } + , [client]) + const refreshStatus = async () => { + closeSnackbar(); + await console.log("gsdlfjslf") + console.log("Refreshing the status of the maintenance window") + // Call the query to get the status of the maintenance window + // Update the status of the maintenance window + // Enqueue Snack bar to show that the maintenance window status is being polled + enqueueSnackbar( + + + + + + + + Maintenance Window Status is being updated !! + + + + , + { + key: new Date().getTime() + Math.random(), + anchorOrigin: { + horizontal: 'right', + vertical: 'top' + }, + variant: 'info', + persist: true, + action: (key) => ( + { + closeSnackbar(key); + }} + > + + + ) + } + ); } + useEffect(() => { + if (client) { + // For the first time + // Check if the maintenance mode is ON + if (isMaintenanceMode()){ + // If ON, then + // Fetch the value of the maintenance mode and paste it on the text field, disable the text field + // Make the button say "End Maintenance" mode + // Fetch the Status of the maintenance mode + // Also, edit the Maintenance mode value + const maintenanceMode = 'READ-ONLY' // GET THIS FROM GRAPHQL ENDPOINT + setMaintenanceButtonText('End Maintenance') + setMaintenanceWindowStatus('IN-PROGRESS') // GET THIS FROM GRAPHQL ENDPOINT + setConfirmedMode(maintenanceMode) + setDropDownStatus(true) + + const setTimer = setInterval(() => { + refreshStatus().catch((e) => dispatch({ type: SET_ERROR, error: e.message }))} + , [10000]) + setRefreshingTimer(setTimer) + return () => clearInterval(setTimer) + + }else{ + // If OFF, then + // Make the button say "Start Maintenance" + // Clear the status and maintenance mode values + setMaintenanceButtonText('Start Maintenance') + setConfirmedMode('') + } + } + }, [client]); + + return ( Create a Maintenance Window @@ -94,30 +232,30 @@ export const MaintenanceViewer = () => { select value={mode} variant="outlined" + disabled={dropDownStatus} > {maintenanceModes.map((group) => ( {group.label} ))} - - - - {/**/} + + } sx={{ m: 1 }} variant="contained" @@ -129,20 +267,23 @@ export const MaintenanceViewer = () => { - Status : + Status : {maintenanceWindowStatus === 'READY-FOR-DEPLOYMENT' ? () : maintenanceWindowStatus === 'IN-PROGRESS' ? () : maintenanceWindowStatus === 'NOT-IN-MAINTENANCE' ? () : <> - } | - Maintenance Mode : {confirmedMode} + Current Maintenance Mode : {confirmedMode} + + Note - To get updates on the maintenance window status please be on this tab or keep visiting maintenance tab. For safe deployments, please deploy when the status is READY-FOR-DEPLOYMENT + - + ) } \ No newline at end of file diff --git a/frontend/src/services/graphql/MaintenanceWindow/isMaintenanceMode.js b/frontend/src/services/graphql/MaintenanceWindow/isMaintenanceMode.js index 2eeddf25a..66ea88077 100644 --- a/frontend/src/services/graphql/MaintenanceWindow/isMaintenanceMode.js +++ b/frontend/src/services/graphql/MaintenanceWindow/isMaintenanceMode.js @@ -1,6 +1,6 @@ import { gql } from 'apollo-boost'; export const isMaintenanceMode = () =>{ - return true; + return false; } // export const isMaintenanceMode = () => ({ // query: gql` From 7febedfe703585b7b256be57f3d72473d8950c3b Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Wed, 17 Apr 2024 19:32:01 +0530 Subject: [PATCH 03/36] Alembic Script to upgrade db --- .../b833ad41db68_maintenance_window_schema.py | 73 +++++++++++++++++++ .../components/MaintenanceViewer.js | 10 +-- 2 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 backend/migrations/versions/b833ad41db68_maintenance_window_schema.py diff --git a/backend/migrations/versions/b833ad41db68_maintenance_window_schema.py b/backend/migrations/versions/b833ad41db68_maintenance_window_schema.py new file mode 100644 index 000000000..20f8d0518 --- /dev/null +++ b/backend/migrations/versions/b833ad41db68_maintenance_window_schema.py @@ -0,0 +1,73 @@ +"""maintenance_window_schema + +Revision ID: b833ad41db68 +Revises: 194608b1ff7f +Create Date: 2024-04-16 19:30:05.226603 + +""" +import os + +from alembic import op +import sqlalchemy as sa +from sqlalchemy import Column, String, orm + +from dataall.base.db import get_engine, has_table, Base + +# revision identifiers, used by Alembic. +revision = 'b833ad41db68' +down_revision = '194608b1ff7f' +branch_labels = None +depends_on = None + + +class Maintenance(Base): + __tablename__ = 'maintenance' + status = Column(String, nullable=False, primary_key=True) + mode = Column(String, default='', nullable=True) + + +def upgrade(): + # Upgrade scripts does the following : + # 1. Creates the maintenance table with two columns : status and mode + # 2. Creates a single record in maintenance table with status : INACTIVE and mode: '' ( Blank ) + try: + envname = os.getenv('envname', 'local') + print('ENVNAME', envname) + engine = get_engine(envname=envname).engine + + bind = op.get_bind() + session = orm.Session(bind=bind) + + # Create the maintenance table + if not has_table('maintenance', engine): + print('Creating maintenance table') + + op.create_table( + 'maintenance', + sa.Column('status', sa.String(), nullable=False, primary_key=True), + sa.Column('mode', sa.String(), nullable=True, default='') + ) + + maintenance_record: [Maintenance] = Maintenance(status='INACTIVE', mode='') + session.add(maintenance_record) + print('Commiting single row to the maintenance table') + session.commit() + + except Exception as e: + print(f'Failed to create migration for maintenance table') + raise e + + +def downgrade(): + # Script for deleting the maintenance table + try: + envname = os.getenv('envname', 'local') + print('ENVNAME', envname) + engine = get_engine(envname=envname).engine + print('Starting downgrade of maintenance') + if has_table('maintenance', engine=engine): + print('Dropping table maintenance') + op.drop_table('maintenance') + except Exception as e: + print('Failed to downgrade maintenance table') + raise e diff --git a/frontend/src/modules/Administration/components/MaintenanceViewer.js b/frontend/src/modules/Administration/components/MaintenanceViewer.js index 392736697..18a384ed6 100644 --- a/frontend/src/modules/Administration/components/MaintenanceViewer.js +++ b/frontend/src/modules/Administration/components/MaintenanceViewer.js @@ -91,7 +91,7 @@ export const MaintenanceViewer = () => { const [popUp, setPopUp] = useState(false) const [confirmedMode, setConfirmedMode] = useState('') const [maintenanceButtonText, setMaintenanceButtonText] = useState('Start Maintenance') - const [maintenanceWindowStatus, setMaintenanceWindowStatus] = useState('NOT-IN-MAINTENANCE') + const [maintenanceWindowStatus, setMaintenanceWindowStatus] = useState('INACTIVE') const [dropDownStatus, setDropDownStatus] = useState(false) const [refreshingTimer, setRefreshingTimer] = useState('') const { enqueueSnackbar, closeSnackbar } = useSnackbar(); @@ -190,7 +190,7 @@ export const MaintenanceViewer = () => { // Also, edit the Maintenance mode value const maintenanceMode = 'READ-ONLY' // GET THIS FROM GRAPHQL ENDPOINT setMaintenanceButtonText('End Maintenance') - setMaintenanceWindowStatus('IN-PROGRESS') // GET THIS FROM GRAPHQL ENDPOINT + setMaintenanceWindowStatus('PENDING') // GET THIS FROM GRAPHQL ENDPOINT setConfirmedMode(maintenanceMode) setDropDownStatus(true) @@ -267,19 +267,19 @@ export const MaintenanceViewer = () => { - Status : {maintenanceWindowStatus === 'READY-FOR-DEPLOYMENT' ? () : maintenanceWindowStatus === 'IN-PROGRESS' ? () : maintenanceWindowStatus === 'NOT-IN-MAINTENANCE' ? () : <> - } + Maintenance window status : {maintenanceWindowStatus === 'ACTIVE' ? () : maintenanceWindowStatus === 'PENDING' ? () : maintenanceWindowStatus === 'INACTIVE' ? () : <> - } | - Current Maintenance Mode : {confirmedMode} + Current maintenance mode : {confirmedMode} - Note - To get updates on the maintenance window status please be on this tab or keep visiting maintenance tab. For safe deployments, please deploy when the status is READY-FOR-DEPLOYMENT + Note - For safe deployments, please deploy when the status is ACTIVE From c2eedda434796db5d566fb42fc08ed46d24dd916 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Thu, 18 Apr 2024 21:51:43 +0530 Subject: [PATCH 04/36] Event-bridge base functions and maintenance modules --- backend/dataall/base/aws/event_bridge.py | 30 +++ .../dataall/modules/maintenance/__init__.py | 27 ++ .../modules/maintenance/api/__init__.py | 5 + .../dataall/modules/maintenance/api/enums.py | 11 + .../modules/maintenance/api/input_types.py | 11 + .../modules/maintenance/api/mutations.py | 18 ++ .../modules/maintenance/api/queries.py | 17 ++ .../modules/maintenance/api/resolvers.py | 62 +++++ .../dataall/modules/maintenance/api/types.py | 22 ++ .../modules/maintenance/db/__init__.py | 1 + .../maintenance/db/maintenance_models.py | 12 + .../maintenance/db/maintenance_repository.py | 25 ++ .../modules/maintenance/services/__init__.py | 8 + .../services/maintenance_permissions.py | 49 ++++ .../services/maintenance_service.py | 255 ++++++++++++++++++ config.json | 3 + 16 files changed, 556 insertions(+) create mode 100644 backend/dataall/base/aws/event_bridge.py create mode 100644 backend/dataall/modules/maintenance/__init__.py create mode 100644 backend/dataall/modules/maintenance/api/__init__.py create mode 100644 backend/dataall/modules/maintenance/api/enums.py create mode 100644 backend/dataall/modules/maintenance/api/input_types.py create mode 100644 backend/dataall/modules/maintenance/api/mutations.py create mode 100644 backend/dataall/modules/maintenance/api/queries.py create mode 100644 backend/dataall/modules/maintenance/api/resolvers.py create mode 100644 backend/dataall/modules/maintenance/api/types.py create mode 100644 backend/dataall/modules/maintenance/db/__init__.py create mode 100644 backend/dataall/modules/maintenance/db/maintenance_models.py create mode 100644 backend/dataall/modules/maintenance/db/maintenance_repository.py create mode 100644 backend/dataall/modules/maintenance/services/__init__.py create mode 100644 backend/dataall/modules/maintenance/services/maintenance_permissions.py create mode 100644 backend/dataall/modules/maintenance/services/maintenance_service.py diff --git a/backend/dataall/base/aws/event_bridge.py b/backend/dataall/base/aws/event_bridge.py new file mode 100644 index 000000000..0df443c2a --- /dev/null +++ b/backend/dataall/base/aws/event_bridge.py @@ -0,0 +1,30 @@ +import logging +import os + +import boto3 + +logger = logging.getLogger(__name__) + + +class EventBridge: + + def __init__(self): + self.client = boto3.client('events', region_name=os.getenv('AWS_REGION', 'eu-west-1')) + + def enable_scheduled_ecs_tasks(self, list_of_tasks): + logger.info("Enabling ecs tasks") + try: + for ecs_task in list_of_tasks: + self.client.enable_rule(Name=ecs_task) + except Exception as e: + logger.error(f'Error while re-enabling scheduled ecs tasks due to {e}') + raise e + + def disable_scheduled_ecs_tasks(self, list_of_tasks): + logger.info("Disabling ecs tasks") + try: + for ecs_task in list_of_tasks: + self.client.disable_rule(Name=ecs_task) + except Exception as e: + logger.error(f'Error while re-enabling scheduled ecs tasks due to {e}') + raise e \ No newline at end of file diff --git a/backend/dataall/modules/maintenance/__init__.py b/backend/dataall/modules/maintenance/__init__.py new file mode 100644 index 000000000..d31075912 --- /dev/null +++ b/backend/dataall/modules/maintenance/__init__.py @@ -0,0 +1,27 @@ +"""Contains the code related to Maintenance Window Activity""" + +import logging +from typing import Set + +from dataall.base.loader import ImportMode, ModuleInterface +from dataall.core.stacks.db.target_type_repositories import TargetType + +log = logging.getLogger(__name__) + + +class MaintenanceApiModuleInterface(ModuleInterface): + """Implements ModuleInterface for Maintenance GraphQl lambda""" + + @staticmethod + def is_supported(modes: Set[ImportMode]) -> bool: + return ImportMode.API in modes + + def __init__(self): + import dataall.modules.maintenance.api + + # from dataall.modules.notebooks.services.notebook_permissions import GET_NOTEBOOK, UPDATE_NOTEBOOK + # + # TargetType('notebook', GET_NOTEBOOK, UPDATE_NOTEBOOK) + print('API of maintenance window activity has been imported') + log.info('API of maintenance window activity has been imported') + diff --git a/backend/dataall/modules/maintenance/api/__init__.py b/backend/dataall/modules/maintenance/api/__init__.py new file mode 100644 index 000000000..5bcc7b839 --- /dev/null +++ b/backend/dataall/modules/maintenance/api/__init__.py @@ -0,0 +1,5 @@ +"""The package defines the schema for Maintenance Module""" + +from dataall.modules.maintenance.api import input_types, mutations, queries, types, resolvers + +__all__ = ['types', 'input_types', 'queries', 'mutations', 'resolvers'] diff --git a/backend/dataall/modules/maintenance/api/enums.py b/backend/dataall/modules/maintenance/api/enums.py new file mode 100644 index 000000000..5f15574d5 --- /dev/null +++ b/backend/dataall/modules/maintenance/api/enums.py @@ -0,0 +1,11 @@ +"""Contains the enums GraphQL mapping for SageMaker notebooks""" + +from dataall.base.api.constants import GraphQLEnumMapper + + +class MaintenanceModes(GraphQLEnumMapper): + """Describes the Maintenance Modes""" + + READONLY = 'READ-ONLY' + NOACCESS = 'NO-ACCESS' + diff --git a/backend/dataall/modules/maintenance/api/input_types.py b/backend/dataall/modules/maintenance/api/input_types.py new file mode 100644 index 000000000..7a0a8c517 --- /dev/null +++ b/backend/dataall/modules/maintenance/api/input_types.py @@ -0,0 +1,11 @@ +"""The module defines GraphQL input types for the Maintenance Window Activity""" + +from dataall.base.api import gql + +MaintenanceWindowInput = gql.InputType( + name='MaintenanceWindowInput', + arguments=[ + gql.Argument('status', gql.NonNullableType(gql.String)), + gql.Argument('mode', gql.String), + ], +) \ No newline at end of file diff --git a/backend/dataall/modules/maintenance/api/mutations.py b/backend/dataall/modules/maintenance/api/mutations.py new file mode 100644 index 000000000..da83e9e50 --- /dev/null +++ b/backend/dataall/modules/maintenance/api/mutations.py @@ -0,0 +1,18 @@ +"""The module defines GraphQL mutations for the Maintenance Window Activity""" + +from dataall.base.api import gql +from dataall.modules.maintenance.api.resolvers import start_maintenance_window, stop_maintenance_window + + +startMaintenanceWindow = gql.MutationField( + name='startMaintenanceWindow', + args=[gql.Argument(name='mode', type=gql.String)], + type=gql.Boolean, + resolver=start_maintenance_window, +) + +stopMaintenanceWindow = gql.MutationField( + name='stopMaintenanceWindow', + type=gql.Boolean, + resolver=stop_maintenance_window, +) diff --git a/backend/dataall/modules/maintenance/api/queries.py b/backend/dataall/modules/maintenance/api/queries.py new file mode 100644 index 000000000..826678758 --- /dev/null +++ b/backend/dataall/modules/maintenance/api/queries.py @@ -0,0 +1,17 @@ +"""The module defines GraphQL queries for the SageMaker notebooks""" + +from dataall.base.api import gql +from dataall.modules.maintenance.api.resolvers import get_maintenance_window_status, get_maintenance_window_mode + + +getMaintenanceWindowStatus = gql.QueryField( + name='getMaintenanceWindowStatus', + type=gql.Ref('Maintenance'), + resolver=get_maintenance_window_status +) + +getMaintenanceWindowMode = gql.QueryField( + name='getMaintenanceWindowMode', + type=gql.String, + resolver=get_maintenance_window_mode +) diff --git a/backend/dataall/modules/maintenance/api/resolvers.py b/backend/dataall/modules/maintenance/api/resolvers.py new file mode 100644 index 000000000..358f36a40 --- /dev/null +++ b/backend/dataall/modules/maintenance/api/resolvers.py @@ -0,0 +1,62 @@ +from dataall.base.api.context import Context +from dataall.core.stacks.api import stack_helper +from dataall.base.db import exceptions +from dataall.modules.maintenance.api.enums import MaintenanceModes +from dataall.modules.maintenance.api.types import Maintenance +from dataall.modules.maintenance.services.maintenance_service import MaintenanceService +from dataall.modules.notebooks.api.enums import SagemakerNotebookRole +from dataall.modules.notebooks.db.notebook_models import SagemakerNotebook +from dataall.modules.notebooks.services.notebook_service import NotebookService, NotebookCreationRequest + + +def create_notebook(context: Context, source: SagemakerNotebook, input: dict = None): + """Creates a SageMaker notebook. Deploys the notebooks stack into AWS""" + RequestValidator.validate_creation_request(input) + request = NotebookCreationRequest.from_dict(input) + return NotebookService.create_notebook( + uri=input['environmentUri'], admin_group=input['SamlAdminGroupName'], request=request + ) + + +def start_maintenance_window(context: Context, source: Maintenance, mode: str): + """Starts the maintenance window""" + if mode not in [item.value for item in list(MaintenanceModes)]: + raise Exception('Mode is not conforming to the MaintenanceModes enums') + # Check from the context if the groups contains the DataAdminstrators group + return MaintenanceService.start_maintenance_window(mode=mode) + + +def stop_maintenance_window(context: Context, source: Maintenance): + # Check from the context if the groups contains the DataAdminstrators group + return MaintenanceService.stop_maintenance_window() + +def get_maintenance_window_status(context: Context, source: Maintenance): + return MaintenanceService.get_maintenance_window_status(engine=context.engine) + +def get_maintenance_window_mode(context: Context, source: Maintenance): + return MaintenanceService.get_maintenance_window_mode() + + +class RequestValidator: + """Aggregates all validation logic for operating with notebooks""" + + @staticmethod + def required_uri(uri): + if not uri: + raise exceptions.RequiredParameter('URI') + + @staticmethod + def validate_creation_request(data): + required = RequestValidator._required + if not data: + raise exceptions.RequiredParameter('data') + if not data.get('label'): + raise exceptions.RequiredParameter('name') + + required(data, 'environmentUri') + required(data, 'SamlAdminGroupName') + + @staticmethod + def _required(data: dict, name: str): + if not data.get(name): + raise exceptions.RequiredParameter(name) diff --git a/backend/dataall/modules/maintenance/api/types.py b/backend/dataall/modules/maintenance/api/types.py new file mode 100644 index 000000000..8ea279337 --- /dev/null +++ b/backend/dataall/modules/maintenance/api/types.py @@ -0,0 +1,22 @@ +"""Defines the object types of the SageMaker notebooks""" + +from dataall.base.api import gql +from dataall.modules.notebooks.api.resolvers import ( + resolve_notebook_stack, + resolve_notebook_status, + resolve_user_role, +) + +from dataall.core.environment.api.resolvers import resolve_environment +from dataall.core.organizations.api.resolvers import resolve_organization_by_env + +from dataall.modules.notebooks.api.enums import SagemakerNotebookRole + + +Maintenance = gql.ObjectType( + name='Maintenance', + fields=[ + gql.Field(name='status', type=gql.String), + gql.Field(name='mode', type=gql.String) + ] +) diff --git a/backend/dataall/modules/maintenance/db/__init__.py b/backend/dataall/modules/maintenance/db/__init__.py new file mode 100644 index 000000000..86631d191 --- /dev/null +++ b/backend/dataall/modules/maintenance/db/__init__.py @@ -0,0 +1 @@ +"""Contains a code to that interacts with the database""" diff --git a/backend/dataall/modules/maintenance/db/maintenance_models.py b/backend/dataall/modules/maintenance/db/maintenance_models.py new file mode 100644 index 000000000..97b40f0df --- /dev/null +++ b/backend/dataall/modules/maintenance/db/maintenance_models.py @@ -0,0 +1,12 @@ +"""ORM models for maintenance windo""" + +from sqlalchemy import Column, String + +from dataall.base.db import Base + + +class Maintenance(Base): + """ORM Model for maintenance window""" + __tablename__ = 'maintenance' + status = Column(String, nullable=False, primary_key=True) + mode = Column(String, default='', nullable=True) \ No newline at end of file diff --git a/backend/dataall/modules/maintenance/db/maintenance_repository.py b/backend/dataall/modules/maintenance/db/maintenance_repository.py new file mode 100644 index 000000000..d8fa3394a --- /dev/null +++ b/backend/dataall/modules/maintenance/db/maintenance_repository.py @@ -0,0 +1,25 @@ +""" +DAO layer that encapsulates the logic and interaction with the database for maintenance +""" + +from dataall.modules.maintenance.db.maintenance_models import Maintenance + + +class MaintenanceRepository: + + def __init__(self, session): + self._session = session + + def save_maintenance_status_and_mode(self, maintenance_status: str, maintenance_mode: str): + maintenance_record = self._session.query(Maintenance) + maintenance_record.status = maintenance_status + maintenance_record.mode = maintenance_mode + self._session.commit() + + def get_maintenance_record(self): + return self._session.query(Maintenance) + + def get_maintenance_mode(self): + maintenance_record = self._session.query(Maintenance) + return maintenance_record.mode + diff --git a/backend/dataall/modules/maintenance/services/__init__.py b/backend/dataall/modules/maintenance/services/__init__.py new file mode 100644 index 000000000..dbb93f498 --- /dev/null +++ b/backend/dataall/modules/maintenance/services/__init__.py @@ -0,0 +1,8 @@ +""" +Contains the code needed for service layer. +The service layer is a layer where all business logic is aggregated +""" + +from dataall.modules.maintenance.services import maintenance_service + +__all__ = ['maintenance_service'] diff --git a/backend/dataall/modules/maintenance/services/maintenance_permissions.py b/backend/dataall/modules/maintenance/services/maintenance_permissions.py new file mode 100644 index 000000000..b6274b5b7 --- /dev/null +++ b/backend/dataall/modules/maintenance/services/maintenance_permissions.py @@ -0,0 +1,49 @@ +""" +Add module's permissions to the global permissions. +Contains permissions for sagemaker notebooks +""" + +from dataall.core.permissions.services.resources_permissions import ( + RESOURCES_ALL_WITH_DESC, + RESOURCES_ALL, +) + +from dataall.core.permissions.services.environment_permissions import ( + ENVIRONMENT_INVITED, + ENVIRONMENT_INVITATION_REQUEST, + ENVIRONMENT_ALL, +) + +from dataall.core.permissions.services.tenant_permissions import TENANT_ALL, TENANT_ALL_WITH_DESC + +GET_MAINTENANCE_STATUS +UPDATE_MAINTENANCE_STATUS +MANAGE_MAINTENANCE_STATUS + +GET_NOTEBOOK = 'GET_NOTEBOOK' +UPDATE_NOTEBOOK = 'UPDATE_NOTEBOOK' +DELETE_NOTEBOOK = 'DELETE_NOTEBOOK' +CREATE_NOTEBOOK = 'CREATE_NOTEBOOK' +MANAGE_NOTEBOOKS = 'MANAGE_NOTEBOOKS' + +NOTEBOOK_ALL = [ + GET_NOTEBOOK, + DELETE_NOTEBOOK, + UPDATE_NOTEBOOK, +] + +ENVIRONMENT_ALL.append(CREATE_NOTEBOOK) +ENVIRONMENT_INVITED.append(CREATE_NOTEBOOK) +ENVIRONMENT_INVITATION_REQUEST.append(CREATE_NOTEBOOK) + +TENANT_ALL.append(MANAGE_NOTEBOOKS) +TENANT_ALL_WITH_DESC[MANAGE_NOTEBOOKS] = 'Manage notebooks' + + +RESOURCES_ALL.append(CREATE_NOTEBOOK) +RESOURCES_ALL.extend(NOTEBOOK_ALL) + +RESOURCES_ALL_WITH_DESC[CREATE_NOTEBOOK] = 'Create notebooks on this environment' +RESOURCES_ALL_WITH_DESC[GET_NOTEBOOK] = 'General permission to get a notebook' +RESOURCES_ALL_WITH_DESC[DELETE_NOTEBOOK] = 'Permission to delete a notebook' +RESOURCES_ALL_WITH_DESC[UPDATE_NOTEBOOK] = 'Permission to edit a notebook' diff --git a/backend/dataall/modules/maintenance/services/maintenance_service.py b/backend/dataall/modules/maintenance/services/maintenance_service.py new file mode 100644 index 000000000..aa1396b40 --- /dev/null +++ b/backend/dataall/modules/maintenance/services/maintenance_service.py @@ -0,0 +1,255 @@ +""" +A service layer for sagemaker notebooks +Central part for working with notebooks +""" + +import dataclasses +import logging +from dataclasses import dataclass, field +from typing import List, Dict + +from dataall.base.context import get_context as context +from dataall.core.environment.db.environment_models import Environment +from dataall.core.environment.env_permission_checker import has_group_permission +from dataall.core.environment.services.environment_service import EnvironmentService +from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService +from dataall.core.permissions.services.tenant_policy_service import TenantPolicyService +from dataall.core.stacks.api import stack_helper +from dataall.core.stacks.db.keyvaluetag_repositories import KeyValueTag +from dataall.core.stacks.db.stack_repositories import Stack +from dataall.base.db import exceptions +from dataall.modules.maintenance.db.maintenance_repository import MaintenanceRepository +from dataall.modules.notebooks.aws.sagemaker_notebook_client import client +from dataall.modules.notebooks.db.notebook_models import SagemakerNotebook +from dataall.modules.notebooks.db.notebook_repository import NotebookRepository +from dataall.modules.notebooks.services.notebook_permissions import ( + MANAGE_NOTEBOOKS, + CREATE_NOTEBOOK, + NOTEBOOK_ALL, + GET_NOTEBOOK, + UPDATE_NOTEBOOK, + DELETE_NOTEBOOK, +) +from dataall.base.utils.naming_convention import ( + NamingConventionService, + NamingConventionPattern, +) +from dataall.base.utils import slugify + +logger = logging.getLogger(__name__) + + +class MaintenanceService: + + # Todo : Check what permissions you need to give here + @staticmethod + def start_maintenance_window(mode: str = None): + # Update the RDS table with the mode and status to PENDING + logger.info("Putting data.all into maintenance window") + + @staticmethod + def stop_maintenance_window(): + # Update the RDS table by changing mode to - '' + # Update the RDS table by changing the status to INACTIVE + logger.info("Stopping maintenance window") + + @staticmethod + def get_maintenance_window_status(engine): + logger.info("Checking maintenance window status") + return MaintenanceRepository(engine).get_maintenance_record() + + + @staticmethod + def get_maintenance_window_mode(): + logger.info("Gettting the maintenance window mode") + return "READ-ONLY" + + +@dataclass +class NotebookCreationRequest: + """A request dataclass for notebook creation. Adds default values for missed parameters""" + + label: str + VpcId: str + SubnetId: str + SamlAdminGroupName: str + environment: Dict = field(default_factory=dict) + description: str = 'No description provided' + VolumeSizeInGB: int = 32 + InstanceType: str = 'ml.t3.medium' + tags: List[str] = field(default_factory=list) + + @classmethod + def from_dict(cls, env): + """Copies only required fields from the dictionary and creates an instance of class""" + fields = set([f.name for f in dataclasses.fields(cls)]) + return cls(**{k: v for k, v in env.items() if k in fields}) + + +class NotebookService: + """ + Encapsulate the logic of interactions with sagemaker notebooks. + """ + + _NOTEBOOK_RESOURCE_TYPE = 'notebook' + + @staticmethod + @TenantPolicyService.has_tenant_permission(MANAGE_NOTEBOOKS) + @ResourcePolicyService.has_resource_permission(CREATE_NOTEBOOK) + @has_group_permission(CREATE_NOTEBOOK) + def create_notebook(*, uri: str, admin_group: str, request: NotebookCreationRequest) -> SagemakerNotebook: + """ + Creates a notebook and attach policies to it + Throws an exception if notebook are not enabled for the environment + """ + + with _session() as session: + env = EnvironmentService.get_environment_by_uri(session, uri) + enabled = EnvironmentService.get_boolean_env_param(session, env, 'notebooksEnabled') + + if not enabled: + raise exceptions.UnauthorizedOperation( + action=CREATE_NOTEBOOK, + message=f'Notebooks feature is disabled for the environment {env.label}', + ) + + env_group = request.environment + if not env_group: + env_group = EnvironmentService.get_environment_group( + session, + group_uri=admin_group, + environment_uri=env.environmentUri, + ) + + notebook = SagemakerNotebook( + label=request.label, + environmentUri=env.environmentUri, + description=request.description, + NotebookInstanceName=slugify(request.label, separator=''), + NotebookInstanceStatus='NotStarted', + AWSAccountId=env.AwsAccountId, + region=env.region, + RoleArn=env_group.environmentIAMRoleArn, + owner=context().username, + SamlAdminGroupName=admin_group, + tags=request.tags, + VpcId=request.VpcId, + SubnetId=request.SubnetId, + VolumeSizeInGB=request.VolumeSizeInGB, + InstanceType=request.InstanceType, + ) + + NotebookRepository(session).save_notebook(notebook) + + notebook.NotebookInstanceName = NamingConventionService( + target_uri=notebook.notebookUri, + target_label=notebook.label, + pattern=NamingConventionPattern.NOTEBOOK, + resource_prefix=env.resourcePrefix, + ).build_compliant_name() + + ResourcePolicyService.attach_resource_policy( + session=session, + group=request.SamlAdminGroupName, + permissions=NOTEBOOK_ALL, + resource_uri=notebook.notebookUri, + resource_type=SagemakerNotebook.__name__, + ) + + if env.SamlGroupName != admin_group: + ResourcePolicyService.attach_resource_policy( + session=session, + group=env.SamlGroupName, + permissions=NOTEBOOK_ALL, + resource_uri=notebook.notebookUri, + resource_type=SagemakerNotebook.__name__, + ) + + Stack.create_stack( + session=session, + environment_uri=notebook.environmentUri, + target_type='notebook', + target_uri=notebook.notebookUri, + target_label=notebook.label, + ) + + stack_helper.deploy_stack(targetUri=notebook.notebookUri) + + return notebook + + @staticmethod + def list_user_notebooks(filter) -> dict: + """List existed user notebooks. Filters only required notebooks by the filter param""" + with _session() as session: + return NotebookRepository(session).paginated_user_notebooks( + username=context().username, groups=context().groups, filter=filter + ) + + @staticmethod + @ResourcePolicyService.has_resource_permission(GET_NOTEBOOK) + def get_notebook(*, uri) -> SagemakerNotebook: + """Gets a notebook by uri""" + with _session() as session: + return NotebookService._get_notebook(session, uri) + + @staticmethod + @ResourcePolicyService.has_resource_permission(UPDATE_NOTEBOOK) + def start_notebook(*, uri): + """Starts notebooks instance""" + notebook = NotebookService.get_notebook(uri=uri) + client(notebook).start_instance() + + @staticmethod + @ResourcePolicyService.has_resource_permission(UPDATE_NOTEBOOK) + def stop_notebook(*, uri: str) -> None: + """Stop notebook instance""" + notebook = NotebookService.get_notebook(uri=uri) + client(notebook).stop_instance() + + @staticmethod + @ResourcePolicyService.has_resource_permission(GET_NOTEBOOK) + def get_notebook_presigned_url(*, uri: str) -> str: + """Creates and returns a presigned url for a notebook""" + notebook = NotebookService.get_notebook(uri=uri) + return client(notebook).presigned_url() + + @staticmethod + @ResourcePolicyService.has_resource_permission(GET_NOTEBOOK) + def get_notebook_status(*, uri) -> str: + """Retrieves notebook status""" + notebook = NotebookService.get_notebook(uri=uri) + return client(notebook).get_notebook_instance_status() + + @staticmethod + @ResourcePolicyService.has_resource_permission(DELETE_NOTEBOOK) + def delete_notebook(*, uri: str, delete_from_aws: bool): + """Deletes notebook from the database and if delete_from_aws is True from AWS as well""" + with _session() as session: + notebook = NotebookService._get_notebook(session, uri) + KeyValueTag.delete_key_value_tags(session, notebook.notebookUri, 'notebook') + session.delete(notebook) + + ResourcePolicyService.delete_resource_policy( + session=session, + resource_uri=notebook.notebookUri, + group=notebook.SamlAdminGroupName, + ) + + env: Environment = EnvironmentService.get_environment_by_uri(session, notebook.environmentUri) + + if delete_from_aws: + stack_helper.delete_stack( + target_uri=uri, accountid=env.AwsAccountId, cdk_role_arn=env.CDKRoleArn, region=env.region + ) + + @staticmethod + def _get_notebook(session, uri) -> SagemakerNotebook: + notebook = NotebookRepository(session).find_notebook(uri) + + if not notebook: + raise exceptions.ObjectNotFound('SagemakerNotebook', uri) + return notebook + + +def _session(): + return context().db_engine.scoped_session() diff --git a/config.json b/config.json index a79a17d0f..e85529f9e 100644 --- a/config.json +++ b/config.json @@ -34,6 +34,9 @@ }, "dashboards": { "active": true + }, + "maintenance": { + "active" : true } }, "core": { From 482db02f447732ffa5a789bcf978395d7e634e56 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Thu, 18 Apr 2024 23:38:01 +0530 Subject: [PATCH 05/36] Changes for making graphQL Work --- .../dataall/modules/maintenance/db/maintenance_models.py | 2 +- .../modules/maintenance/db/maintenance_repository.py | 6 ++++-- .../modules/maintenance/services/maintenance_service.py | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/backend/dataall/modules/maintenance/db/maintenance_models.py b/backend/dataall/modules/maintenance/db/maintenance_models.py index 97b40f0df..910d98031 100644 --- a/backend/dataall/modules/maintenance/db/maintenance_models.py +++ b/backend/dataall/modules/maintenance/db/maintenance_models.py @@ -9,4 +9,4 @@ class Maintenance(Base): """ORM Model for maintenance window""" __tablename__ = 'maintenance' status = Column(String, nullable=False, primary_key=True) - mode = Column(String, default='', nullable=True) \ No newline at end of file + mode = Column(String, default='', nullable=True) diff --git a/backend/dataall/modules/maintenance/db/maintenance_repository.py b/backend/dataall/modules/maintenance/db/maintenance_repository.py index d8fa3394a..d898bdb68 100644 --- a/backend/dataall/modules/maintenance/db/maintenance_repository.py +++ b/backend/dataall/modules/maintenance/db/maintenance_repository.py @@ -2,8 +2,10 @@ DAO layer that encapsulates the logic and interaction with the database for maintenance """ +import logging from dataall.modules.maintenance.db.maintenance_models import Maintenance +log = logging.getLogger(__name__) class MaintenanceRepository: @@ -11,13 +13,13 @@ def __init__(self, session): self._session = session def save_maintenance_status_and_mode(self, maintenance_status: str, maintenance_mode: str): - maintenance_record = self._session.query(Maintenance) + maintenance_record = self._session.query(Maintenance).one() maintenance_record.status = maintenance_status maintenance_record.mode = maintenance_mode self._session.commit() def get_maintenance_record(self): - return self._session.query(Maintenance) + return self._session.query(Maintenance).one() def get_maintenance_mode(self): maintenance_record = self._session.query(Maintenance) diff --git a/backend/dataall/modules/maintenance/services/maintenance_service.py b/backend/dataall/modules/maintenance/services/maintenance_service.py index aa1396b40..628a16c17 100644 --- a/backend/dataall/modules/maintenance/services/maintenance_service.py +++ b/backend/dataall/modules/maintenance/services/maintenance_service.py @@ -56,7 +56,8 @@ def stop_maintenance_window(): @staticmethod def get_maintenance_window_status(engine): logger.info("Checking maintenance window status") - return MaintenanceRepository(engine).get_maintenance_record() + with engine.scoped_session() as session: + return MaintenanceRepository(session).get_maintenance_record() @staticmethod From 485129ea0d4ca7edd99c555675d4f0e89fc70534 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Fri, 19 Apr 2024 00:35:02 +0530 Subject: [PATCH 06/36] Mutation graphQl corrections --- backend/dataall/modules/maintenance/api/resolvers.py | 5 ++++- .../modules/maintenance/db/maintenance_repository.py | 4 ++-- .../modules/maintenance/services/maintenance_service.py | 9 ++++++++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/backend/dataall/modules/maintenance/api/resolvers.py b/backend/dataall/modules/maintenance/api/resolvers.py index 358f36a40..5cb70f9ad 100644 --- a/backend/dataall/modules/maintenance/api/resolvers.py +++ b/backend/dataall/modules/maintenance/api/resolvers.py @@ -1,3 +1,5 @@ +import logging + from dataall.base.api.context import Context from dataall.core.stacks.api import stack_helper from dataall.base.db import exceptions @@ -23,7 +25,8 @@ def start_maintenance_window(context: Context, source: Maintenance, mode: str): if mode not in [item.value for item in list(MaintenanceModes)]: raise Exception('Mode is not conforming to the MaintenanceModes enums') # Check from the context if the groups contains the DataAdminstrators group - return MaintenanceService.start_maintenance_window(mode=mode) + logging.info(context.groups) + return MaintenanceService.start_maintenance_window(engine=context.engine, mode=mode) def stop_maintenance_window(context: Context, source: Maintenance): diff --git a/backend/dataall/modules/maintenance/db/maintenance_repository.py b/backend/dataall/modules/maintenance/db/maintenance_repository.py index d898bdb68..65779ad86 100644 --- a/backend/dataall/modules/maintenance/db/maintenance_repository.py +++ b/backend/dataall/modules/maintenance/db/maintenance_repository.py @@ -12,9 +12,9 @@ class MaintenanceRepository: def __init__(self, session): self._session = session - def save_maintenance_status_and_mode(self, maintenance_status: str, maintenance_mode: str): + def save_maintenance_status_and_mode(self, maintenance_mode: str): maintenance_record = self._session.query(Maintenance).one() - maintenance_record.status = maintenance_status + maintenance_record.status = 'PENDING' maintenance_record.mode = maintenance_mode self._session.commit() diff --git a/backend/dataall/modules/maintenance/services/maintenance_service.py b/backend/dataall/modules/maintenance/services/maintenance_service.py index 628a16c17..bab32b634 100644 --- a/backend/dataall/modules/maintenance/services/maintenance_service.py +++ b/backend/dataall/modules/maintenance/services/maintenance_service.py @@ -43,9 +43,16 @@ class MaintenanceService: # Todo : Check what permissions you need to give here @staticmethod - def start_maintenance_window(mode: str = None): + def start_maintenance_window(engine, mode: str = None): # Update the RDS table with the mode and status to PENDING logger.info("Putting data.all into maintenance window") + try: + with engine.scoped_session() as session: + MaintenanceRepository(session).save_maintenance_status_and_mode(maintenance_mode=mode) + return True + except Exception as e: + logger.error(f"Error occurred while starting maintenance window due to {e}") + return False @staticmethod def stop_maintenance_window(): From 89e9944901b1fe903130852cec8e1514c9119d91 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Fri, 19 Apr 2024 19:53:26 +0530 Subject: [PATCH 07/36] Resolver changs --- backend/dataall/modules/maintenance/api/resolvers.py | 6 +++++- .../modules/maintenance/db/maintenance_repository.py | 4 ++-- .../maintenance/services/maintenance_service.py | 12 ++++++++++-- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/backend/dataall/modules/maintenance/api/resolvers.py b/backend/dataall/modules/maintenance/api/resolvers.py index 5cb70f9ad..777f3bd36 100644 --- a/backend/dataall/modules/maintenance/api/resolvers.py +++ b/backend/dataall/modules/maintenance/api/resolvers.py @@ -26,12 +26,16 @@ def start_maintenance_window(context: Context, source: Maintenance, mode: str): raise Exception('Mode is not conforming to the MaintenanceModes enums') # Check from the context if the groups contains the DataAdminstrators group logging.info(context.groups) + if "DAAdministrators" not in context.groups: + raise Exception('Only data.all admin group members can start maintenance window') return MaintenanceService.start_maintenance_window(engine=context.engine, mode=mode) def stop_maintenance_window(context: Context, source: Maintenance): # Check from the context if the groups contains the DataAdminstrators group - return MaintenanceService.stop_maintenance_window() + if "DAAdministrators" not in context.groups: + raise Exception('Only data.all admin group members can stop maintenance window') + return MaintenanceService.stop_maintenance_window(engine=context.engine) def get_maintenance_window_status(context: Context, source: Maintenance): return MaintenanceService.get_maintenance_window_status(engine=context.engine) diff --git a/backend/dataall/modules/maintenance/db/maintenance_repository.py b/backend/dataall/modules/maintenance/db/maintenance_repository.py index 65779ad86..ebbf114a4 100644 --- a/backend/dataall/modules/maintenance/db/maintenance_repository.py +++ b/backend/dataall/modules/maintenance/db/maintenance_repository.py @@ -12,9 +12,9 @@ class MaintenanceRepository: def __init__(self, session): self._session = session - def save_maintenance_status_and_mode(self, maintenance_mode: str): + def save_maintenance_status_and_mode(self, maintenance_status: str, maintenance_mode: str): maintenance_record = self._session.query(Maintenance).one() - maintenance_record.status = 'PENDING' + maintenance_record.status = maintenance_status maintenance_record.mode = maintenance_mode self._session.commit() diff --git a/backend/dataall/modules/maintenance/services/maintenance_service.py b/backend/dataall/modules/maintenance/services/maintenance_service.py index bab32b634..79a28ddfd 100644 --- a/backend/dataall/modules/maintenance/services/maintenance_service.py +++ b/backend/dataall/modules/maintenance/services/maintenance_service.py @@ -48,17 +48,25 @@ def start_maintenance_window(engine, mode: str = None): logger.info("Putting data.all into maintenance window") try: with engine.scoped_session() as session: - MaintenanceRepository(session).save_maintenance_status_and_mode(maintenance_mode=mode) + MaintenanceRepository(session).save_maintenance_status_and_mode(maintenance_status='PENDING' ,maintenance_mode=mode) return True except Exception as e: logger.error(f"Error occurred while starting maintenance window due to {e}") return False @staticmethod - def stop_maintenance_window(): + def stop_maintenance_window(engine): # Update the RDS table by changing mode to - '' # Update the RDS table by changing the status to INACTIVE logger.info("Stopping maintenance window") + try: + with engine.scoped_session() as session: + MaintenanceRepository(session).save_maintenance_status_and_mode(maintenance_status='INACTIVE', maintenance_mode='') + return True + except Exception as e: + logger.error(f"Error occurred while stopping maintenance window due to {e}") + return False + @staticmethod def get_maintenance_window_status(engine): From 2c400b73ff64314a41442130539fd1389afa9d87 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Mon, 22 Apr 2024 15:05:57 -0500 Subject: [PATCH 08/36] Struture for event bridge graphql calls --- backend/dataall/modules/maintenance/api/resolvers.py | 5 +++-- .../maintenance/services/maintenance_service.py | 10 +++++++++- deploy/stacks/container.py | 9 +++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/backend/dataall/modules/maintenance/api/resolvers.py b/backend/dataall/modules/maintenance/api/resolvers.py index 777f3bd36..cc1c9d8d6 100644 --- a/backend/dataall/modules/maintenance/api/resolvers.py +++ b/backend/dataall/modules/maintenance/api/resolvers.py @@ -1,6 +1,7 @@ import logging from dataall.base.api.context import Context +from dataall.core.permissions.services.tenant_policy_service import TenantPolicyValidationService from dataall.core.stacks.api import stack_helper from dataall.base.db import exceptions from dataall.modules.maintenance.api.enums import MaintenanceModes @@ -26,14 +27,14 @@ def start_maintenance_window(context: Context, source: Maintenance, mode: str): raise Exception('Mode is not conforming to the MaintenanceModes enums') # Check from the context if the groups contains the DataAdminstrators group logging.info(context.groups) - if "DAAdministrators" not in context.groups: + if not TenantPolicyValidationService.is_tenant_admin(context.groups): raise Exception('Only data.all admin group members can start maintenance window') return MaintenanceService.start_maintenance_window(engine=context.engine, mode=mode) def stop_maintenance_window(context: Context, source: Maintenance): # Check from the context if the groups contains the DataAdminstrators group - if "DAAdministrators" not in context.groups: + if not TenantPolicyValidationService.is_tenant_admin(context.groups): raise Exception('Only data.all admin group members can stop maintenance window') return MaintenanceService.stop_maintenance_window(engine=context.engine) diff --git a/backend/dataall/modules/maintenance/services/maintenance_service.py b/backend/dataall/modules/maintenance/services/maintenance_service.py index 79a28ddfd..78cd8e92f 100644 --- a/backend/dataall/modules/maintenance/services/maintenance_service.py +++ b/backend/dataall/modules/maintenance/services/maintenance_service.py @@ -8,6 +8,7 @@ from dataclasses import dataclass, field from typing import List, Dict +from dataall.base.aws.event_bridge import EventBridge from dataall.base.context import get_context as context from dataall.core.environment.db.environment_models import Environment from dataall.core.environment.env_permission_checker import has_group_permission @@ -49,6 +50,9 @@ def start_maintenance_window(engine, mode: str = None): try: with engine.scoped_session() as session: MaintenanceRepository(session).save_maintenance_status_and_mode(maintenance_status='PENDING' ,maintenance_mode=mode) + # Disable scheduled ECS tasks + event_bridge_session = EventBridge() + event_bridge_session.disable_scheduled_ecs_tasks(['dataall-staging-catalog-indexer-schedule']) return True except Exception as e: logger.error(f"Error occurred while starting maintenance window due to {e}") @@ -62,7 +66,11 @@ def stop_maintenance_window(engine): try: with engine.scoped_session() as session: MaintenanceRepository(session).save_maintenance_status_and_mode(maintenance_status='INACTIVE', maintenance_mode='') - return True + + # Enable scheduled ECS tasks + event_bridge_session = EventBridge() + event_bridge_session.enable_scheduled_ecs_tasks(['dataall-staging-catalog-indexer-schedule']) + return True except Exception as e: logger.error(f"Error occurred while stopping maintenance window due to {e}") return False diff --git a/deploy/stacks/container.py b/deploy/stacks/container.py index 7ded368c0..253f8ba53 100644 --- a/deploy/stacks/container.py +++ b/deploy/stacks/container.py @@ -617,6 +617,15 @@ def set_scheduled_task( rule_name=scheduled_task_id, security_groups=[security_group], ) + + # Add the rule of the scheduled task to parameter store + ssm.StringParameter( + self, + f'ECSTaskRule-{scheduled_task.event_rule.rule_name}', + parameter_name=f'/dataall/{self._envname}/ecs/ecs_scheduled_tasks/rule/{scheduled_task_id}', + string_value=scheduled_task.event_rule.rule_name, + ) + return scheduled_task, task @property From 3f5ace1c6b925a9047e83368b74d9ad395ef7253 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Tue, 23 Apr 2024 09:22:01 -0500 Subject: [PATCH 09/36] Adding few thigns --- backend/dataall/base/aws/event_bridge.py | 4 +- backend/dataall/base/aws/parameter_store.py | 13 + .../dataall/modules/maintenance/api/enums.py | 6 + .../modules/maintenance/api/resolvers.py | 34 --- .../services/maintenance_service.py | 240 +++--------------- 5 files changed, 62 insertions(+), 235 deletions(-) diff --git a/backend/dataall/base/aws/event_bridge.py b/backend/dataall/base/aws/event_bridge.py index 0df443c2a..d66fda27c 100644 --- a/backend/dataall/base/aws/event_bridge.py +++ b/backend/dataall/base/aws/event_bridge.py @@ -8,8 +8,8 @@ class EventBridge: - def __init__(self): - self.client = boto3.client('events', region_name=os.getenv('AWS_REGION', 'eu-west-1')) + def __init__(self, region=None): + self.client = boto3.client('events', region_name=region) def enable_scheduled_ecs_tasks(self, list_of_tasks): logger.info("Enabling ecs tasks") diff --git a/backend/dataall/base/aws/parameter_store.py b/backend/dataall/base/aws/parameter_store.py index a78b7cb16..85978377a 100644 --- a/backend/dataall/base/aws/parameter_store.py +++ b/backend/dataall/base/aws/parameter_store.py @@ -37,6 +37,19 @@ def get_parameter_value(AwsAccountId=None, region=None, parameter_path=None): raise Exception(e) return parameter_value + @staticmethod + def get_parameters_by_path(AwsAccountId=None, region=None, parameter_path=None): + if not parameter_path: + raise Exception('Parameter name is None') + try: + parameter_value = ParameterStoreManager.client(AwsAccountId, region).get_parameters_by_path(Path=parameter_path)[ + 'Parameters' + ] + log.info(ParameterStoreManager.client(AwsAccountId, region).get_parameters_by_path(Path=parameter_path)) + except ClientError as e: + raise Exception(e) + return parameter_value + @staticmethod def update_parameter(AwsAccountId, region, parameter_name, parameter_value): if not parameter_name: diff --git a/backend/dataall/modules/maintenance/api/enums.py b/backend/dataall/modules/maintenance/api/enums.py index 5f15574d5..c3e364da7 100644 --- a/backend/dataall/modules/maintenance/api/enums.py +++ b/backend/dataall/modules/maintenance/api/enums.py @@ -9,3 +9,9 @@ class MaintenanceModes(GraphQLEnumMapper): READONLY = 'READ-ONLY' NOACCESS = 'NO-ACCESS' +class MaintenanceStatus(): + """Describe the various statuses for maintenance""" + + PENDING = 'PENDING' + INACTIVE = 'INACTIVE' + ACTIVE = 'ACTIVE' diff --git a/backend/dataall/modules/maintenance/api/resolvers.py b/backend/dataall/modules/maintenance/api/resolvers.py index cc1c9d8d6..85fd5874c 100644 --- a/backend/dataall/modules/maintenance/api/resolvers.py +++ b/backend/dataall/modules/maintenance/api/resolvers.py @@ -12,15 +12,6 @@ from dataall.modules.notebooks.services.notebook_service import NotebookService, NotebookCreationRequest -def create_notebook(context: Context, source: SagemakerNotebook, input: dict = None): - """Creates a SageMaker notebook. Deploys the notebooks stack into AWS""" - RequestValidator.validate_creation_request(input) - request = NotebookCreationRequest.from_dict(input) - return NotebookService.create_notebook( - uri=input['environmentUri'], admin_group=input['SamlAdminGroupName'], request=request - ) - - def start_maintenance_window(context: Context, source: Maintenance, mode: str): """Starts the maintenance window""" if mode not in [item.value for item in list(MaintenanceModes)]: @@ -43,28 +34,3 @@ def get_maintenance_window_status(context: Context, source: Maintenance): def get_maintenance_window_mode(context: Context, source: Maintenance): return MaintenanceService.get_maintenance_window_mode() - - -class RequestValidator: - """Aggregates all validation logic for operating with notebooks""" - - @staticmethod - def required_uri(uri): - if not uri: - raise exceptions.RequiredParameter('URI') - - @staticmethod - def validate_creation_request(data): - required = RequestValidator._required - if not data: - raise exceptions.RequiredParameter('data') - if not data.get('label'): - raise exceptions.RequiredParameter('name') - - required(data, 'environmentUri') - required(data, 'SamlAdminGroupName') - - @staticmethod - def _required(data: dict, name: str): - if not data.get(name): - raise exceptions.RequiredParameter(name) diff --git a/backend/dataall/modules/maintenance/services/maintenance_service.py b/backend/dataall/modules/maintenance/services/maintenance_service.py index 78cd8e92f..cbdfbf84c 100644 --- a/backend/dataall/modules/maintenance/services/maintenance_service.py +++ b/backend/dataall/modules/maintenance/services/maintenance_service.py @@ -5,10 +5,12 @@ import dataclasses import logging +import os from dataclasses import dataclass, field from typing import List, Dict from dataall.base.aws.event_bridge import EventBridge +from dataall.base.aws.parameter_store import ParameterStoreManager from dataall.base.context import get_context as context from dataall.core.environment.db.environment_models import Environment from dataall.core.environment.env_permission_checker import has_group_permission @@ -19,6 +21,7 @@ from dataall.core.stacks.db.keyvaluetag_repositories import KeyValueTag from dataall.core.stacks.db.stack_repositories import Stack from dataall.base.db import exceptions +from dataall.modules.maintenance.api.enums import MaintenanceStatus from dataall.modules.maintenance.db.maintenance_repository import MaintenanceRepository from dataall.modules.notebooks.aws.sagemaker_notebook_client import client from dataall.modules.notebooks.db.notebook_models import SagemakerNotebook @@ -42,17 +45,30 @@ class MaintenanceService: - # Todo : Check what permissions you need to give here @staticmethod def start_maintenance_window(engine, mode: str = None): # Update the RDS table with the mode and status to PENDING - logger.info("Putting data.all into maintenance window") + logger.info("Putting data.all into maintenance") try: with engine.scoped_session() as session: - MaintenanceRepository(session).save_maintenance_status_and_mode(maintenance_status='PENDING' ,maintenance_mode=mode) + # Todo: Check the current status of maintenance mode and then execute the update query + maintenance_record = MaintenanceRepository(session).get_maintenance_record() + if maintenance_record.status == MaintenanceStatus.PENDING or maintenance_record.status == MaintenanceStatus.ACTIVE: + logger.error("Maintenance window already in PENDING or ACTIVE state. Cannot start maintenance window. Stop the maintenance window and start again") + return False + MaintenanceRepository(session).save_maintenance_status_and_mode(maintenance_status=MaintenanceStatus.PENDING ,maintenance_mode=mode) # Disable scheduled ECS tasks - event_bridge_session = EventBridge() - event_bridge_session.disable_scheduled_ecs_tasks(['dataall-staging-catalog-indexer-schedule']) + # Get all the SSMs related to the scheduled tasks + ecs_scheduled_rules = ParameterStoreManager.get_parameters_by_path( + region=os.getenv('AWS_REGION', 'eu-west-1'), + parameter_path=f"/dataall/{os.getenv('envname', 'local')}/ecs/ecs_scheduled_tasks/rule" + ) + logger.info(ecs_scheduled_rules) + ecs_scheduled_rules_list = [item['Value'] for item in ecs_scheduled_rules] + logger.info("Value of ecs scheduled tasks") + logger.info(ecs_scheduled_rules_list) + event_bridge_session = EventBridge(region=os.getenv('AWS_REGION', 'eu-west-1')) + event_bridge_session.disable_scheduled_ecs_tasks(ecs_scheduled_rules_list) return True except Exception as e: logger.error(f"Error occurred while starting maintenance window due to {e}") @@ -62,14 +78,25 @@ def start_maintenance_window(engine, mode: str = None): def stop_maintenance_window(engine): # Update the RDS table by changing mode to - '' # Update the RDS table by changing the status to INACTIVE - logger.info("Stopping maintenance window") + logger.info("Stopping maintenance") try: with engine.scoped_session() as session: + maintenance_record = MaintenanceRepository(session).get_maintenance_record() + if maintenance_record.status == MaintenanceStatus.PENDING or maintenance_record.status == MaintenanceStatus.INACTIVE: + logger.error("Maintenance window already in PENDING or INACTIVE state. Cannot start maintenance window. Stop the maintenance window and start again") + return False MaintenanceRepository(session).save_maintenance_status_and_mode(maintenance_status='INACTIVE', maintenance_mode='') - # Enable scheduled ECS tasks + ecs_scheduled_rules = ParameterStoreManager.get_parameters_by_path( + region=os.getenv('AWS_REGION', 'eu-west-1'), + parameter_path=f"/dataall/{os.getenv('envname', 'local')}/ecs/ecs_scheduled_tasks/rule" + ) + logger.info(ecs_scheduled_rules) + ecs_scheduled_rules_list = [item['Value'] for item in ecs_scheduled_rules] + logger.info("Value of ecs scheduled tasks") + logger.info(ecs_scheduled_rules_list) event_bridge_session = EventBridge() - event_bridge_session.enable_scheduled_ecs_tasks(['dataall-staging-catalog-indexer-schedule']) + event_bridge_session.enable_scheduled_ecs_tasks(ecs_scheduled_rules_list) return True except Exception as e: logger.error(f"Error occurred while stopping maintenance window due to {e}") @@ -80,200 +107,15 @@ def stop_maintenance_window(engine): def get_maintenance_window_status(engine): logger.info("Checking maintenance window status") with engine.scoped_session() as session: - return MaintenanceRepository(session).get_maintenance_record() + maintenance_record = MaintenanceRepository(session).get_maintenance_record() + if maintenance_record.status == MaintenanceStatus.PENDING: + # Check all the ECS tasks + return False + # Fetch the name of ECS services + @staticmethod def get_maintenance_window_mode(): logger.info("Gettting the maintenance window mode") return "READ-ONLY" - - -@dataclass -class NotebookCreationRequest: - """A request dataclass for notebook creation. Adds default values for missed parameters""" - - label: str - VpcId: str - SubnetId: str - SamlAdminGroupName: str - environment: Dict = field(default_factory=dict) - description: str = 'No description provided' - VolumeSizeInGB: int = 32 - InstanceType: str = 'ml.t3.medium' - tags: List[str] = field(default_factory=list) - - @classmethod - def from_dict(cls, env): - """Copies only required fields from the dictionary and creates an instance of class""" - fields = set([f.name for f in dataclasses.fields(cls)]) - return cls(**{k: v for k, v in env.items() if k in fields}) - - -class NotebookService: - """ - Encapsulate the logic of interactions with sagemaker notebooks. - """ - - _NOTEBOOK_RESOURCE_TYPE = 'notebook' - - @staticmethod - @TenantPolicyService.has_tenant_permission(MANAGE_NOTEBOOKS) - @ResourcePolicyService.has_resource_permission(CREATE_NOTEBOOK) - @has_group_permission(CREATE_NOTEBOOK) - def create_notebook(*, uri: str, admin_group: str, request: NotebookCreationRequest) -> SagemakerNotebook: - """ - Creates a notebook and attach policies to it - Throws an exception if notebook are not enabled for the environment - """ - - with _session() as session: - env = EnvironmentService.get_environment_by_uri(session, uri) - enabled = EnvironmentService.get_boolean_env_param(session, env, 'notebooksEnabled') - - if not enabled: - raise exceptions.UnauthorizedOperation( - action=CREATE_NOTEBOOK, - message=f'Notebooks feature is disabled for the environment {env.label}', - ) - - env_group = request.environment - if not env_group: - env_group = EnvironmentService.get_environment_group( - session, - group_uri=admin_group, - environment_uri=env.environmentUri, - ) - - notebook = SagemakerNotebook( - label=request.label, - environmentUri=env.environmentUri, - description=request.description, - NotebookInstanceName=slugify(request.label, separator=''), - NotebookInstanceStatus='NotStarted', - AWSAccountId=env.AwsAccountId, - region=env.region, - RoleArn=env_group.environmentIAMRoleArn, - owner=context().username, - SamlAdminGroupName=admin_group, - tags=request.tags, - VpcId=request.VpcId, - SubnetId=request.SubnetId, - VolumeSizeInGB=request.VolumeSizeInGB, - InstanceType=request.InstanceType, - ) - - NotebookRepository(session).save_notebook(notebook) - - notebook.NotebookInstanceName = NamingConventionService( - target_uri=notebook.notebookUri, - target_label=notebook.label, - pattern=NamingConventionPattern.NOTEBOOK, - resource_prefix=env.resourcePrefix, - ).build_compliant_name() - - ResourcePolicyService.attach_resource_policy( - session=session, - group=request.SamlAdminGroupName, - permissions=NOTEBOOK_ALL, - resource_uri=notebook.notebookUri, - resource_type=SagemakerNotebook.__name__, - ) - - if env.SamlGroupName != admin_group: - ResourcePolicyService.attach_resource_policy( - session=session, - group=env.SamlGroupName, - permissions=NOTEBOOK_ALL, - resource_uri=notebook.notebookUri, - resource_type=SagemakerNotebook.__name__, - ) - - Stack.create_stack( - session=session, - environment_uri=notebook.environmentUri, - target_type='notebook', - target_uri=notebook.notebookUri, - target_label=notebook.label, - ) - - stack_helper.deploy_stack(targetUri=notebook.notebookUri) - - return notebook - - @staticmethod - def list_user_notebooks(filter) -> dict: - """List existed user notebooks. Filters only required notebooks by the filter param""" - with _session() as session: - return NotebookRepository(session).paginated_user_notebooks( - username=context().username, groups=context().groups, filter=filter - ) - - @staticmethod - @ResourcePolicyService.has_resource_permission(GET_NOTEBOOK) - def get_notebook(*, uri) -> SagemakerNotebook: - """Gets a notebook by uri""" - with _session() as session: - return NotebookService._get_notebook(session, uri) - - @staticmethod - @ResourcePolicyService.has_resource_permission(UPDATE_NOTEBOOK) - def start_notebook(*, uri): - """Starts notebooks instance""" - notebook = NotebookService.get_notebook(uri=uri) - client(notebook).start_instance() - - @staticmethod - @ResourcePolicyService.has_resource_permission(UPDATE_NOTEBOOK) - def stop_notebook(*, uri: str) -> None: - """Stop notebook instance""" - notebook = NotebookService.get_notebook(uri=uri) - client(notebook).stop_instance() - - @staticmethod - @ResourcePolicyService.has_resource_permission(GET_NOTEBOOK) - def get_notebook_presigned_url(*, uri: str) -> str: - """Creates and returns a presigned url for a notebook""" - notebook = NotebookService.get_notebook(uri=uri) - return client(notebook).presigned_url() - - @staticmethod - @ResourcePolicyService.has_resource_permission(GET_NOTEBOOK) - def get_notebook_status(*, uri) -> str: - """Retrieves notebook status""" - notebook = NotebookService.get_notebook(uri=uri) - return client(notebook).get_notebook_instance_status() - - @staticmethod - @ResourcePolicyService.has_resource_permission(DELETE_NOTEBOOK) - def delete_notebook(*, uri: str, delete_from_aws: bool): - """Deletes notebook from the database and if delete_from_aws is True from AWS as well""" - with _session() as session: - notebook = NotebookService._get_notebook(session, uri) - KeyValueTag.delete_key_value_tags(session, notebook.notebookUri, 'notebook') - session.delete(notebook) - - ResourcePolicyService.delete_resource_policy( - session=session, - resource_uri=notebook.notebookUri, - group=notebook.SamlAdminGroupName, - ) - - env: Environment = EnvironmentService.get_environment_by_uri(session, notebook.environmentUri) - - if delete_from_aws: - stack_helper.delete_stack( - target_uri=uri, accountid=env.AwsAccountId, cdk_role_arn=env.CDKRoleArn, region=env.region - ) - - @staticmethod - def _get_notebook(session, uri) -> SagemakerNotebook: - notebook = NotebookRepository(session).find_notebook(uri) - - if not notebook: - raise exceptions.ObjectNotFound('SagemakerNotebook', uri) - return notebook - - -def _session(): - return context().db_engine.scoped_session() From 9dd0890e71915d7771413a55ce8c5535bfbb8a85 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Tue, 23 Apr 2024 13:44:41 -0500 Subject: [PATCH 10/36] All GraphQL Endpoint working --- backend/dataall/core/stacks/aws/ecs.py | 7 ++- .../modules/maintenance/api/resolvers.py | 3 +- .../services/maintenance_service.py | 55 +++++++++---------- 3 files changed, 33 insertions(+), 32 deletions(-) diff --git a/backend/dataall/core/stacks/aws/ecs.py b/backend/dataall/core/stacks/aws/ecs.py index 8da1e1118..866d21db5 100644 --- a/backend/dataall/core/stacks/aws/ecs.py +++ b/backend/dataall/core/stacks/aws/ecs.py @@ -86,10 +86,13 @@ def run_ecs_task( raise e @staticmethod - def is_task_running(cluster_name, started_by): + def is_task_running(cluster_name, started_by=None): try: client = boto3.client('ecs') - running_tasks = client.list_tasks(cluster=cluster_name, startedBy=started_by, desiredStatus='RUNNING') + if started_by is None: + running_tasks = client.list_tasks(cluster=cluster_name, desiredStatus='RUNNING') + else: + running_tasks = client.list_tasks(cluster=cluster_name, startedBy=started_by, desiredStatus='RUNNING') if running_tasks and running_tasks.get('taskArns'): return True return False diff --git a/backend/dataall/modules/maintenance/api/resolvers.py b/backend/dataall/modules/maintenance/api/resolvers.py index 85fd5874c..6eb0c9921 100644 --- a/backend/dataall/modules/maintenance/api/resolvers.py +++ b/backend/dataall/modules/maintenance/api/resolvers.py @@ -17,7 +17,6 @@ def start_maintenance_window(context: Context, source: Maintenance, mode: str): if mode not in [item.value for item in list(MaintenanceModes)]: raise Exception('Mode is not conforming to the MaintenanceModes enums') # Check from the context if the groups contains the DataAdminstrators group - logging.info(context.groups) if not TenantPolicyValidationService.is_tenant_admin(context.groups): raise Exception('Only data.all admin group members can start maintenance window') return MaintenanceService.start_maintenance_window(engine=context.engine, mode=mode) @@ -33,4 +32,4 @@ def get_maintenance_window_status(context: Context, source: Maintenance): return MaintenanceService.get_maintenance_window_status(engine=context.engine) def get_maintenance_window_mode(context: Context, source: Maintenance): - return MaintenanceService.get_maintenance_window_mode() + return MaintenanceService.get_maintenance_window_mode(engine=context.engine) diff --git a/backend/dataall/modules/maintenance/services/maintenance_service.py b/backend/dataall/modules/maintenance/services/maintenance_service.py index cbdfbf84c..23d5d4cab 100644 --- a/backend/dataall/modules/maintenance/services/maintenance_service.py +++ b/backend/dataall/modules/maintenance/services/maintenance_service.py @@ -23,22 +23,7 @@ from dataall.base.db import exceptions from dataall.modules.maintenance.api.enums import MaintenanceStatus from dataall.modules.maintenance.db.maintenance_repository import MaintenanceRepository -from dataall.modules.notebooks.aws.sagemaker_notebook_client import client -from dataall.modules.notebooks.db.notebook_models import SagemakerNotebook -from dataall.modules.notebooks.db.notebook_repository import NotebookRepository -from dataall.modules.notebooks.services.notebook_permissions import ( - MANAGE_NOTEBOOKS, - CREATE_NOTEBOOK, - NOTEBOOK_ALL, - GET_NOTEBOOK, - UPDATE_NOTEBOOK, - DELETE_NOTEBOOK, -) -from dataall.base.utils.naming_convention import ( - NamingConventionService, - NamingConventionPattern, -) -from dataall.base.utils import slugify +from dataall.core.stacks.aws.ecs import Ecs logger = logging.getLogger(__name__) @@ -51,7 +36,6 @@ def start_maintenance_window(engine, mode: str = None): logger.info("Putting data.all into maintenance") try: with engine.scoped_session() as session: - # Todo: Check the current status of maintenance mode and then execute the update query maintenance_record = MaintenanceRepository(session).get_maintenance_record() if maintenance_record.status == MaintenanceStatus.PENDING or maintenance_record.status == MaintenanceStatus.ACTIVE: logger.error("Maintenance window already in PENDING or ACTIVE state. Cannot start maintenance window. Stop the maintenance window and start again") @@ -82,7 +66,7 @@ def stop_maintenance_window(engine): try: with engine.scoped_session() as session: maintenance_record = MaintenanceRepository(session).get_maintenance_record() - if maintenance_record.status == MaintenanceStatus.PENDING or maintenance_record.status == MaintenanceStatus.INACTIVE: + if maintenance_record.status == MaintenanceStatus.INACTIVE: logger.error("Maintenance window already in PENDING or INACTIVE state. Cannot start maintenance window. Stop the maintenance window and start again") return False MaintenanceRepository(session).save_maintenance_status_and_mode(maintenance_status='INACTIVE', maintenance_mode='') @@ -107,15 +91,30 @@ def stop_maintenance_window(engine): def get_maintenance_window_status(engine): logger.info("Checking maintenance window status") with engine.scoped_session() as session: - maintenance_record = MaintenanceRepository(session).get_maintenance_record() - if maintenance_record.status == MaintenanceStatus.PENDING: - # Check all the ECS tasks - return False - # Fetch the name of ECS services - - + try: + maintenance_record = MaintenanceRepository(session).get_maintenance_record() + if maintenance_record.status == MaintenanceStatus.PENDING: + # Check all the ECS tasks + ecs_cluster_name = ParameterStoreManager.get_parameter_value( + region=os.getenv('AWS_REGION', 'eu-west-1'), + parameter_path=f"/dataall/{os.getenv('envname', 'local')}/ecs/cluster/name" + ) + if Ecs.is_task_running(cluster_name=ecs_cluster_name): + return maintenance_record + else: + maintenance_record.status = MaintenanceStatus.ACTIVE + session.commit() + return maintenance_record + else: + logger.info("Maintenance window is not in PENDING state") + return maintenance_record + except Exception as e: + logger.error(f'Error while getting maintenance window status due to {e}') + raise e @staticmethod - def get_maintenance_window_mode(): - logger.info("Gettting the maintenance window mode") - return "READ-ONLY" + def get_maintenance_window_mode(engine): + logger.info("Fetching status of maintenance window") + with engine.scoped_session() as session: + maintenance_record = MaintenanceRepository(session).get_maintenance_record() + return maintenance_record.mode From 24a24932931930616190c3241bc833bf30b50078 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Tue, 23 Apr 2024 17:17:15 -0500 Subject: [PATCH 11/36] Complete graphQL endpoint + api_handler logic to block calls --- backend/api_handler.py | 46 ++++++++++++++++--- .../dataall/modules/maintenance/api/enums.py | 6 +-- .../modules/maintenance/api/resolvers.py | 13 +++--- .../services/maintenance_service.py | 2 +- 4 files changed, 50 insertions(+), 17 deletions(-) diff --git a/backend/api_handler.py b/backend/api_handler.py index 53b005a2a..7e8dd603e 100644 --- a/backend/api_handler.py +++ b/backend/api_handler.py @@ -16,10 +16,12 @@ from dataall.base.aws.sqs import SqsQueue from dataall.base.aws.parameter_store import ParameterStoreManager from dataall.base.context import set_context, dispose_context, RequestContext -from dataall.core.permissions.services.tenant_policy_service import TenantPolicyService +from dataall.core.permissions.services.tenant_policy_service import TenantPolicyService, TenantPolicyValidationService from dataall.base.db import get_engine from dataall.core.permissions.services.tenant_permissions import TENANT_ALL from dataall.base.loader import load_modules, ImportMode +from dataall.modules.maintenance.api.enums import MaintenanceModes, MaintenanceStatus +from dataall.modules.maintenance.services.maintenance_service import MaintenanceService logger = logging.getLogger() logger.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) @@ -83,6 +85,29 @@ def get_custom_groups(user_id): return service_provider.get_groups_for_user(user_id) +def send_unauthorized_response(query, message=''): + response = { + 'data': {query.get('operationName', 'operation'): None}, + 'errors': [ + { + 'message': message, + 'locations': None, + 'path': [query.get('operationName', '')], + } + ], + } + return { + 'statusCode': 401, + 'headers': { + 'content-type': 'application/json', + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Headers': '*', + 'Access-Control-Allow-Methods': '*', + }, + 'body': json.dumps(response), + } + + def handler(event, context): """Sample pure Lambda function @@ -165,10 +190,19 @@ def handler(event, context): 'schema': SCHEMA, } - # If maintenance mode is enabled -> Check Status by using the graphQL Endpoint - # If groups doesn't contain data.all administrator group - # Check what is the access mode - # Return response with error "Maintenance Window is ON" + query = json.loads(event.get('body')) + + # Logic to block when in maintenance + # Check if in some maintenance mode + # Check if in maintenance status is not INACTIVE + # Check if the user belongs to a 'DAAdministrators' group + if (MaintenanceService.get_maintenance_window_mode(engine=ENGINE) == MaintenanceModes.NOACCESS.value) and (MaintenanceService.get_maintenance_window_status(engine=ENGINE).status is not MaintenanceStatus.INACTIVE) and not TenantPolicyValidationService.is_tenant_admin(groups): + send_unauthorized_response(query=query, message='Access Restricted: data.all is currently undergoing maintenance, and your actions are temporarily blocked.') + elif (MaintenanceService.get_maintenance_window_mode(engine=ENGINE) == MaintenanceModes.READONLY.value) and (MaintenanceService.get_maintenance_window_status(engine=ENGINE).status is not MaintenanceStatus.INACTIVE) and not TenantPolicyValidationService.is_tenant_admin(groups): + # If its mutation then block and return + if query.get('query', '').split(' ')[0] == 'mutation': + send_unauthorized_response(query=query, message='Access Restricted: data.all is currently undergoing maintenance, and your actions are temporarily blocked.') + # Determine if there are any Operations that Require ReAuth From SSM Parameter try: @@ -181,7 +215,7 @@ def handler(event, context): else: raise Exception(f'Could not initialize user context from event {event}') - query = json.loads(event.get('body')) + # If The Operation is a ReAuth Operation - Ensure A Non-Expired Session or Return Error if reauth_apis and query.get('operationName', None) in reauth_apis: diff --git a/backend/dataall/modules/maintenance/api/enums.py b/backend/dataall/modules/maintenance/api/enums.py index c3e364da7..976cbac3d 100644 --- a/backend/dataall/modules/maintenance/api/enums.py +++ b/backend/dataall/modules/maintenance/api/enums.py @@ -1,11 +1,9 @@ """Contains the enums GraphQL mapping for SageMaker notebooks""" +from enum import Enum -from dataall.base.api.constants import GraphQLEnumMapper - -class MaintenanceModes(GraphQLEnumMapper): +class MaintenanceModes(Enum): """Describes the Maintenance Modes""" - READONLY = 'READ-ONLY' NOACCESS = 'NO-ACCESS' diff --git a/backend/dataall/modules/maintenance/api/resolvers.py b/backend/dataall/modules/maintenance/api/resolvers.py index 6eb0c9921..38c738d3a 100644 --- a/backend/dataall/modules/maintenance/api/resolvers.py +++ b/backend/dataall/modules/maintenance/api/resolvers.py @@ -2,14 +2,9 @@ from dataall.base.api.context import Context from dataall.core.permissions.services.tenant_policy_service import TenantPolicyValidationService -from dataall.core.stacks.api import stack_helper -from dataall.base.db import exceptions from dataall.modules.maintenance.api.enums import MaintenanceModes from dataall.modules.maintenance.api.types import Maintenance from dataall.modules.maintenance.services.maintenance_service import MaintenanceService -from dataall.modules.notebooks.api.enums import SagemakerNotebookRole -from dataall.modules.notebooks.db.notebook_models import SagemakerNotebook -from dataall.modules.notebooks.services.notebook_service import NotebookService, NotebookCreationRequest def start_maintenance_window(context: Context, source: Maintenance, mode: str): @@ -28,8 +23,14 @@ def stop_maintenance_window(context: Context, source: Maintenance): raise Exception('Only data.all admin group members can stop maintenance window') return MaintenanceService.stop_maintenance_window(engine=context.engine) + def get_maintenance_window_status(context: Context, source: Maintenance): + if not TenantPolicyValidationService.is_tenant_admin(context.groups): + raise Exception('Only data.all admin group members can access maintenance endpoints') return MaintenanceService.get_maintenance_window_status(engine=context.engine) + def get_maintenance_window_mode(context: Context, source: Maintenance): - return MaintenanceService.get_maintenance_window_mode(engine=context.engine) + if not TenantPolicyValidationService.is_tenant_admin(context.groups): + raise Exception('Only data.all admin group members can access maintenance endpoints') + return MaintenanceService.get_maintenance_window_mode(engine=context) diff --git a/backend/dataall/modules/maintenance/services/maintenance_service.py b/backend/dataall/modules/maintenance/services/maintenance_service.py index 23d5d4cab..51f1b7023 100644 --- a/backend/dataall/modules/maintenance/services/maintenance_service.py +++ b/backend/dataall/modules/maintenance/services/maintenance_service.py @@ -67,7 +67,7 @@ def stop_maintenance_window(engine): with engine.scoped_session() as session: maintenance_record = MaintenanceRepository(session).get_maintenance_record() if maintenance_record.status == MaintenanceStatus.INACTIVE: - logger.error("Maintenance window already in PENDING or INACTIVE state. Cannot start maintenance window. Stop the maintenance window and start again") + logger.error("Maintenance window already in INACTIVE state. Cannot stop maintenance window") return False MaintenanceRepository(session).save_maintenance_status_and_mode(maintenance_status='INACTIVE', maintenance_mode='') # Enable scheduled ECS tasks From be31c18e1a7ec822e936cedf7287d9d84e11abcd Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Wed, 24 Apr 2024 09:39:51 -0500 Subject: [PATCH 12/36] Correcting enums for maintenance status --- backend/api_handler.py | 4 ++-- backend/dataall/modules/maintenance/api/enums.py | 2 +- .../maintenance/services/maintenance_service.py | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/backend/api_handler.py b/backend/api_handler.py index 7e8dd603e..9ed6b369d 100644 --- a/backend/api_handler.py +++ b/backend/api_handler.py @@ -196,9 +196,9 @@ def handler(event, context): # Check if in some maintenance mode # Check if in maintenance status is not INACTIVE # Check if the user belongs to a 'DAAdministrators' group - if (MaintenanceService.get_maintenance_window_mode(engine=ENGINE) == MaintenanceModes.NOACCESS.value) and (MaintenanceService.get_maintenance_window_status(engine=ENGINE).status is not MaintenanceStatus.INACTIVE) and not TenantPolicyValidationService.is_tenant_admin(groups): + if (MaintenanceService.get_maintenance_window_mode(engine=ENGINE) == MaintenanceModes.NOACCESS.value) and (MaintenanceService.get_maintenance_window_status(engine=ENGINE).status is not MaintenanceStatus.INACTIVE.value) and not TenantPolicyValidationService.is_tenant_admin(groups): send_unauthorized_response(query=query, message='Access Restricted: data.all is currently undergoing maintenance, and your actions are temporarily blocked.') - elif (MaintenanceService.get_maintenance_window_mode(engine=ENGINE) == MaintenanceModes.READONLY.value) and (MaintenanceService.get_maintenance_window_status(engine=ENGINE).status is not MaintenanceStatus.INACTIVE) and not TenantPolicyValidationService.is_tenant_admin(groups): + elif (MaintenanceService.get_maintenance_window_mode(engine=ENGINE) == MaintenanceModes.READONLY.value) and (MaintenanceService.get_maintenance_window_status(engine=ENGINE).status is not MaintenanceStatus.INACTIVE.value) and not TenantPolicyValidationService.is_tenant_admin(groups): # If its mutation then block and return if query.get('query', '').split(' ')[0] == 'mutation': send_unauthorized_response(query=query, message='Access Restricted: data.all is currently undergoing maintenance, and your actions are temporarily blocked.') diff --git a/backend/dataall/modules/maintenance/api/enums.py b/backend/dataall/modules/maintenance/api/enums.py index 976cbac3d..868de8d75 100644 --- a/backend/dataall/modules/maintenance/api/enums.py +++ b/backend/dataall/modules/maintenance/api/enums.py @@ -7,7 +7,7 @@ class MaintenanceModes(Enum): READONLY = 'READ-ONLY' NOACCESS = 'NO-ACCESS' -class MaintenanceStatus(): +class MaintenanceStatus(Enum): """Describe the various statuses for maintenance""" PENDING = 'PENDING' diff --git a/backend/dataall/modules/maintenance/services/maintenance_service.py b/backend/dataall/modules/maintenance/services/maintenance_service.py index 51f1b7023..f01f704a8 100644 --- a/backend/dataall/modules/maintenance/services/maintenance_service.py +++ b/backend/dataall/modules/maintenance/services/maintenance_service.py @@ -37,10 +37,10 @@ def start_maintenance_window(engine, mode: str = None): try: with engine.scoped_session() as session: maintenance_record = MaintenanceRepository(session).get_maintenance_record() - if maintenance_record.status == MaintenanceStatus.PENDING or maintenance_record.status == MaintenanceStatus.ACTIVE: + if maintenance_record.status == MaintenanceStatus.PENDING.value or maintenance_record.status == MaintenanceStatus.ACTIVE.value: logger.error("Maintenance window already in PENDING or ACTIVE state. Cannot start maintenance window. Stop the maintenance window and start again") return False - MaintenanceRepository(session).save_maintenance_status_and_mode(maintenance_status=MaintenanceStatus.PENDING ,maintenance_mode=mode) + MaintenanceRepository(session).save_maintenance_status_and_mode(maintenance_status=MaintenanceStatus.PENDING.value ,maintenance_mode=mode) # Disable scheduled ECS tasks # Get all the SSMs related to the scheduled tasks ecs_scheduled_rules = ParameterStoreManager.get_parameters_by_path( @@ -66,7 +66,7 @@ def stop_maintenance_window(engine): try: with engine.scoped_session() as session: maintenance_record = MaintenanceRepository(session).get_maintenance_record() - if maintenance_record.status == MaintenanceStatus.INACTIVE: + if maintenance_record.status == MaintenanceStatus.INACTIVE.value: logger.error("Maintenance window already in INACTIVE state. Cannot stop maintenance window") return False MaintenanceRepository(session).save_maintenance_status_and_mode(maintenance_status='INACTIVE', maintenance_mode='') @@ -93,7 +93,7 @@ def get_maintenance_window_status(engine): with engine.scoped_session() as session: try: maintenance_record = MaintenanceRepository(session).get_maintenance_record() - if maintenance_record.status == MaintenanceStatus.PENDING: + if maintenance_record.status == MaintenanceStatus.PENDING.value: # Check all the ECS tasks ecs_cluster_name = ParameterStoreManager.get_parameter_value( region=os.getenv('AWS_REGION', 'eu-west-1'), @@ -102,7 +102,7 @@ def get_maintenance_window_status(engine): if Ecs.is_task_running(cluster_name=ecs_cluster_name): return maintenance_record else: - maintenance_record.status = MaintenanceStatus.ACTIVE + maintenance_record.status = MaintenanceStatus.ACTIVE.value session.commit() return maintenance_record else: From 174ce6826a85d4995a17f809e17d62c6760d0d28 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Thu, 25 Apr 2024 14:18:48 -0500 Subject: [PATCH 13/36] Finalizing changes for maintenance window --- backend/api_handler.py | 83 +-- backend/dataall/base/aws/event_bridge.py | 7 +- backend/dataall/base/aws/parameter_store.py | 6 +- .../dataall/base/utils/api_handler_utils.py | 47 ++ .../core/organizations/api/resolvers.py | 2 + .../dataall/core/organizations/api/types.py | 2 +- .../services/share_notification_service.py | 26 +- .../dataall/modules/maintenance/__init__.py | 6 - .../modules/maintenance/api/__init__.py | 4 +- .../dataall/modules/maintenance/api/enums.py | 5 +- .../modules/maintenance/api/input_types.py | 11 - .../modules/maintenance/api/queries.py | 14 +- .../modules/maintenance/api/resolvers.py | 15 +- .../dataall/modules/maintenance/api/types.py | 18 +- .../maintenance/db/maintenance_models.py | 3 +- .../maintenance/db/maintenance_repository.py | 6 +- .../services/maintenance_permissions.py | 49 -- .../services/maintenance_service.py | 83 ++- .../b833ad41db68_maintenance_window_schema.py | 5 +- backend/search_handler.py | 80 ++- config.json | 3 +- deploy/stacks/lambda_api.py | 9 +- deploy/stacks/param_store_stack.py | 26 +- frontend/src/App.js | 6 +- .../authentication/components/AuthGuard.js | 45 +- .../contexts/LocalAuthContext.js | 6 +- .../components/NoAccessMaintenanceWindow.js | 13 +- frontend/src/design/components/index.js | 2 +- .../design/components/layout/DefaultNavbar.js | 56 +- .../components/MaintenanceViewer.js | 674 +++++++++++------- .../Administration/components/index.js | 2 +- .../views/AdministrationView.js | 14 +- .../MaintenanceWindow/getMaintenanceStatus.js | 12 + .../graphql/MaintenanceWindow/index.js | 4 +- .../MaintenanceWindow/isMaintenanceMode.js | 12 - .../startMaintenanceWindow.js | 10 + .../stopMaintenanceWindow.js | 9 + 37 files changed, 799 insertions(+), 576 deletions(-) create mode 100644 backend/dataall/base/utils/api_handler_utils.py delete mode 100644 backend/dataall/modules/maintenance/api/input_types.py delete mode 100644 backend/dataall/modules/maintenance/services/maintenance_permissions.py create mode 100644 frontend/src/services/graphql/MaintenanceWindow/getMaintenanceStatus.js delete mode 100644 frontend/src/services/graphql/MaintenanceWindow/isMaintenanceMode.js create mode 100644 frontend/src/services/graphql/MaintenanceWindow/startMaintenanceWindow.js create mode 100644 frontend/src/services/graphql/MaintenanceWindow/stopMaintenanceWindow.js diff --git a/backend/api_handler.py b/backend/api_handler.py index 9ed6b369d..8ee21bbf4 100644 --- a/backend/api_handler.py +++ b/backend/api_handler.py @@ -12,6 +12,7 @@ from dataall.base.api import bootstrap as bootstrap_schema, get_executable_schema from dataall.base.services.service_provider_factory import ServiceProviderFactory +from dataall.base.utils.api_handler_utils import get_custom_groups, get_cognito_groups, send_unauthorized_response from dataall.core.tasks.service_handlers import Worker from dataall.base.aws.sqs import SqsQueue from dataall.base.aws.parameter_store import ParameterStoreManager @@ -22,6 +23,7 @@ from dataall.base.loader import load_modules, ImportMode from dataall.modules.maintenance.api.enums import MaintenanceModes, MaintenanceStatus from dataall.modules.maintenance.services.maintenance_service import MaintenanceService +from dataall.base.config import config logger = logging.getLogger() logger.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) @@ -64,50 +66,6 @@ def adapted(obj, info, **kwargs): print(f'Lambda Context ' f'Initialization took: {end - start:.3f} sec') -def get_cognito_groups(claims): - if not claims: - raise ValueError( - 'Received empty claims. ' 'Please verify authorizer configuration', - claims, - ) - groups = list() - saml_groups = claims.get('custom:saml.groups', '') - if len(saml_groups): - groups: list = saml_groups.replace('[', '').replace(']', '').replace(', ', ',').split(',') - cognito_groups = claims.get('cognito:groups', '') - if len(cognito_groups): - groups.extend(cognito_groups.split(',')) - return groups - - -def get_custom_groups(user_id): - service_provider = ServiceProviderFactory.get_service_provider_instance() - return service_provider.get_groups_for_user(user_id) - - -def send_unauthorized_response(query, message=''): - response = { - 'data': {query.get('operationName', 'operation'): None}, - 'errors': [ - { - 'message': message, - 'locations': None, - 'path': [query.get('operationName', '')], - } - ], - } - return { - 'statusCode': 401, - 'headers': { - 'content-type': 'application/json', - 'Access-Control-Allow-Origin': '*', - 'Access-Control-Allow-Headers': '*', - 'Access-Control-Allow-Methods': '*', - }, - 'body': json.dumps(response), - } - - def handler(event, context): """Sample pure Lambda function @@ -196,13 +154,34 @@ def handler(event, context): # Check if in some maintenance mode # Check if in maintenance status is not INACTIVE # Check if the user belongs to a 'DAAdministrators' group - if (MaintenanceService.get_maintenance_window_mode(engine=ENGINE) == MaintenanceModes.NOACCESS.value) and (MaintenanceService.get_maintenance_window_status(engine=ENGINE).status is not MaintenanceStatus.INACTIVE.value) and not TenantPolicyValidationService.is_tenant_admin(groups): - send_unauthorized_response(query=query, message='Access Restricted: data.all is currently undergoing maintenance, and your actions are temporarily blocked.') - elif (MaintenanceService.get_maintenance_window_mode(engine=ENGINE) == MaintenanceModes.READONLY.value) and (MaintenanceService.get_maintenance_window_status(engine=ENGINE).status is not MaintenanceStatus.INACTIVE.value) and not TenantPolicyValidationService.is_tenant_admin(groups): - # If its mutation then block and return - if query.get('query', '').split(' ')[0] == 'mutation': - send_unauthorized_response(query=query, message='Access Restricted: data.all is currently undergoing maintenance, and your actions are temporarily blocked.') - + # Todo : Add check to see if maintenance module is enabled or not from the config + if config.get_property('modules.maintenance.active'): + if ( + (MaintenanceService._get_maintenance_window_mode(engine=ENGINE) == MaintenanceModes.NOACCESS.value) + and ( + MaintenanceService.get_maintenance_window_status(engine=ENGINE).status + is not MaintenanceStatus.INACTIVE.value + ) + and not TenantPolicyValidationService.is_tenant_admin(groups) + ): + send_unauthorized_response( + query=query, + message='Access Restricted: data.all is currently undergoing maintenance, and your actions are temporarily blocked.', + ) + elif ( + (MaintenanceService._get_maintenance_window_mode(engine=ENGINE) == MaintenanceModes.READONLY.value) + and ( + MaintenanceService.get_maintenance_window_status(engine=ENGINE).status + is not MaintenanceStatus.INACTIVE.value + ) + and not TenantPolicyValidationService.is_tenant_admin(groups) + ): + # If its mutation then block and return + if query.get('query', '').split(' ')[0] == 'mutation': + send_unauthorized_response( + query=query, + message='Access Restricted: data.all is currently undergoing maintenance, and your actions are temporarily blocked.', + ) # Determine if there are any Operations that Require ReAuth From SSM Parameter try: @@ -215,8 +194,6 @@ def handler(event, context): else: raise Exception(f'Could not initialize user context from event {event}') - - # If The Operation is a ReAuth Operation - Ensure A Non-Expired Session or Return Error if reauth_apis and query.get('operationName', None) in reauth_apis: now = datetime.datetime.now(datetime.timezone.utc) diff --git a/backend/dataall/base/aws/event_bridge.py b/backend/dataall/base/aws/event_bridge.py index d66fda27c..d4a980e5a 100644 --- a/backend/dataall/base/aws/event_bridge.py +++ b/backend/dataall/base/aws/event_bridge.py @@ -7,12 +7,11 @@ class EventBridge: - def __init__(self, region=None): self.client = boto3.client('events', region_name=region) def enable_scheduled_ecs_tasks(self, list_of_tasks): - logger.info("Enabling ecs tasks") + logger.info('Enabling ecs tasks') try: for ecs_task in list_of_tasks: self.client.enable_rule(Name=ecs_task) @@ -21,10 +20,10 @@ def enable_scheduled_ecs_tasks(self, list_of_tasks): raise e def disable_scheduled_ecs_tasks(self, list_of_tasks): - logger.info("Disabling ecs tasks") + logger.info('Disabling ecs tasks') try: for ecs_task in list_of_tasks: self.client.disable_rule(Name=ecs_task) except Exception as e: logger.error(f'Error while re-enabling scheduled ecs tasks due to {e}') - raise e \ No newline at end of file + raise e diff --git a/backend/dataall/base/aws/parameter_store.py b/backend/dataall/base/aws/parameter_store.py index 85978377a..7957e0ef0 100644 --- a/backend/dataall/base/aws/parameter_store.py +++ b/backend/dataall/base/aws/parameter_store.py @@ -42,9 +42,9 @@ def get_parameters_by_path(AwsAccountId=None, region=None, parameter_path=None): if not parameter_path: raise Exception('Parameter name is None') try: - parameter_value = ParameterStoreManager.client(AwsAccountId, region).get_parameters_by_path(Path=parameter_path)[ - 'Parameters' - ] + parameter_value = ParameterStoreManager.client(AwsAccountId, region).get_parameters_by_path( + Path=parameter_path + )['Parameters'] log.info(ParameterStoreManager.client(AwsAccountId, region).get_parameters_by_path(Path=parameter_path)) except ClientError as e: raise Exception(e) diff --git a/backend/dataall/base/utils/api_handler_utils.py b/backend/dataall/base/utils/api_handler_utils.py new file mode 100644 index 000000000..88e82f206 --- /dev/null +++ b/backend/dataall/base/utils/api_handler_utils.py @@ -0,0 +1,47 @@ +import json + +from dataall.base.services.service_provider_factory import ServiceProviderFactory + + +def get_cognito_groups(claims): + if not claims: + raise ValueError( + 'Received empty claims. ' 'Please verify authorizer configuration', + claims, + ) + groups = list() + saml_groups = claims.get('custom:saml.groups', '') + if len(saml_groups): + groups: list = saml_groups.replace('[', '').replace(']', '').replace(', ', ',').split(',') + cognito_groups = claims.get('cognito:groups', '') + if len(cognito_groups): + groups.extend(cognito_groups.split(',')) + return groups + + +def get_custom_groups(user_id): + service_provider = ServiceProviderFactory.get_service_provider_instance() + return service_provider.get_groups_for_user(user_id) + + +def send_unauthorized_response(query, message=''): + response = { + 'data': {query.get('operationName', 'operation'): None}, + 'errors': [ + { + 'message': message, + 'locations': None, + 'path': [query.get('operationName', '')], + } + ], + } + return { + 'statusCode': 401, + 'headers': { + 'content-type': 'application/json', + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Headers': '*', + 'Access-Control-Allow-Methods': '*', + }, + 'body': json.dumps(response), + } diff --git a/backend/dataall/core/organizations/api/resolvers.py b/backend/dataall/core/organizations/api/resolvers.py index 08d095e75..45d849d63 100644 --- a/backend/dataall/core/organizations/api/resolvers.py +++ b/backend/dataall/core/organizations/api/resolvers.py @@ -25,9 +25,11 @@ def update_organization(context, source, organizationUri=None, input=None): def get_organization(context: Context, source, organizationUri=None): return OrganizationService.get_organization(uri=organizationUri) + def get_organization_simplified(context: Context, source, organizationUri=None): return OrganizationService.get_organization_simplified(uri=organizationUri) + def list_organizations(context: Context, source, filter=None): if not filter: filter = {'page': 1, 'pageSize': 5} diff --git a/backend/dataall/core/organizations/api/types.py b/backend/dataall/core/organizations/api/types.py index 3d176ca54..24acbef1e 100644 --- a/backend/dataall/core/organizations/api/types.py +++ b/backend/dataall/core/organizations/api/types.py @@ -57,6 +57,6 @@ fields=[ gql.Field(name='organizationUri', type=gql.ID), gql.Field(name='label', type=gql.String), - gql.Field(name='name', type=gql.String) + gql.Field(name='name', type=gql.String), ], ) diff --git a/backend/dataall/modules/dataset_sharing/services/share_notification_service.py b/backend/dataall/modules/dataset_sharing/services/share_notification_service.py index faaf7d30f..33a3e8e18 100644 --- a/backend/dataall/modules/dataset_sharing/services/share_notification_service.py +++ b/backend/dataall/modules/dataset_sharing/services/share_notification_service.py @@ -43,35 +43,37 @@ def __init__(self, session, dataset: Dataset, share: ShareObject): def notify_share_object_submission(self, email_id: str): share_link_text = '' - if os.environ.get("frontend_domain_url"): + if os.environ.get('frontend_domain_url'): share_link_text = f'

Please visit data.all share link to take action or view more details' msg = f'User {email_id} SUBMITTED share request for dataset {self.dataset.label} for principal {self.share.principalId}' subject = f'Data.all | Share Request Submitted for {self.dataset.label}' email_notification_msg = msg + share_link_text notifications = self._register_notifications( - notification_type=DataSharingNotificationType.SHARE_OBJECT_SUBMITTED.value, msg=msg) + notification_type=DataSharingNotificationType.SHARE_OBJECT_SUBMITTED.value, msg=msg + ) self._create_notification_task(subject=subject, msg=email_notification_msg) return notifications def notify_share_object_approval(self, email_id: str): share_link_text = '' - if os.environ.get("frontend_domain_url"): + if os.environ.get('frontend_domain_url'): share_link_text = f'

Please visit data.all share link to take action or view more details' msg = f'User {email_id} APPROVED share request for dataset {self.dataset.label} for principal {self.share.principalId}' subject = f'Data.all | Share Request Approved for {self.dataset.label}' email_notification_msg = msg + share_link_text notifications = self._register_notifications( - notification_type=DataSharingNotificationType.SHARE_OBJECT_APPROVED.value, msg=msg) + notification_type=DataSharingNotificationType.SHARE_OBJECT_APPROVED.value, msg=msg + ) self._create_notification_task(subject=subject, msg=email_notification_msg) return notifications def notify_share_object_rejection(self, email_id: str): share_link_text = '' - if os.environ.get("frontend_domain_url"): + if os.environ.get('frontend_domain_url'): share_link_text = f'

Please visit data.all share link to take action or view more details' if self.share.status == ShareObjectStatus.Rejected.value: msg = f'User {email_id} REJECTED share request for dataset {self.dataset.label} for principal {self.share.principalId}' @@ -85,7 +87,8 @@ def notify_share_object_rejection(self, email_id: str): email_notification_msg = msg + share_link_text notifications = self._register_notifications( - notification_type=DataSharingNotificationType.SHARE_OBJECT_REJECTED.value, msg=msg) + notification_type=DataSharingNotificationType.SHARE_OBJECT_REJECTED.value, msg=msg + ) self._create_notification_task(subject=subject, msg=email_notification_msg) return notifications @@ -94,7 +97,8 @@ def notify_new_data_available_from_owners(self, s3_prefix): msg = f'New data (at {s3_prefix}) is available from dataset {self.dataset.datasetUri} shared by owner {self.dataset.owner}' notifications = self._register_notifications( - notification_type=DataSharingNotificationType.DATASET_VERSION.value, msg=msg) + notification_type=DataSharingNotificationType.DATASET_VERSION.value, msg=msg + ) return notifications def _get_share_object_targeted_users(self): @@ -114,7 +118,7 @@ def _register_notifications(self, notification_type, msg): """ notifications = [] for recipient in self.notification_target_users: - log.info(f"Creating notification for {recipient}, msg {msg}") + log.info(f'Creating notification for {recipient}, msg {msg}') notifications.append( NotificationRepository.create_notification( session=self.session, @@ -144,8 +148,8 @@ def _create_notification_task(self, subject, msg): notification_recipient_groups_list = [self.dataset.SamlAdminGroupName, self.dataset.stewards] notification_recipient_email_ids = [] - if share_notification_config_type == "email": - if params.get("group_notifications", False) == True: + if share_notification_config_type == 'email': + if params.get('group_notifications', False) == True: notification_recipient_groups_list.append(self.share.groupUri) else: notification_recipient_email_ids = [self.share.owner] @@ -158,7 +162,7 @@ def _create_notification_task(self, subject, msg): 'subject': subject, 'message': msg, 'recipientGroupsList': notification_recipient_groups_list, - 'recipientEmailList': notification_recipient_email_ids + 'recipientEmailList': notification_recipient_email_ids, }, ) self.session.add(notification_task) diff --git a/backend/dataall/modules/maintenance/__init__.py b/backend/dataall/modules/maintenance/__init__.py index d31075912..29596caf8 100644 --- a/backend/dataall/modules/maintenance/__init__.py +++ b/backend/dataall/modules/maintenance/__init__.py @@ -4,7 +4,6 @@ from typing import Set from dataall.base.loader import ImportMode, ModuleInterface -from dataall.core.stacks.db.target_type_repositories import TargetType log = logging.getLogger(__name__) @@ -19,9 +18,4 @@ def is_supported(modes: Set[ImportMode]) -> bool: def __init__(self): import dataall.modules.maintenance.api - # from dataall.modules.notebooks.services.notebook_permissions import GET_NOTEBOOK, UPDATE_NOTEBOOK - # - # TargetType('notebook', GET_NOTEBOOK, UPDATE_NOTEBOOK) - print('API of maintenance window activity has been imported') log.info('API of maintenance window activity has been imported') - diff --git a/backend/dataall/modules/maintenance/api/__init__.py b/backend/dataall/modules/maintenance/api/__init__.py index 5bcc7b839..a2690b057 100644 --- a/backend/dataall/modules/maintenance/api/__init__.py +++ b/backend/dataall/modules/maintenance/api/__init__.py @@ -1,5 +1,5 @@ """The package defines the schema for Maintenance Module""" -from dataall.modules.maintenance.api import input_types, mutations, queries, types, resolvers +from dataall.modules.maintenance.api import mutations, queries, types, resolvers, enums -__all__ = ['types', 'input_types', 'queries', 'mutations', 'resolvers'] +__all__ = ['types', 'queries', 'mutations', 'resolvers', 'enums'] diff --git a/backend/dataall/modules/maintenance/api/enums.py b/backend/dataall/modules/maintenance/api/enums.py index 868de8d75..8771188e7 100644 --- a/backend/dataall/modules/maintenance/api/enums.py +++ b/backend/dataall/modules/maintenance/api/enums.py @@ -1,12 +1,15 @@ -"""Contains the enums GraphQL mapping for SageMaker notebooks""" +"""Contains the enums used in maintenance module""" + from enum import Enum class MaintenanceModes(Enum): """Describes the Maintenance Modes""" + READONLY = 'READ-ONLY' NOACCESS = 'NO-ACCESS' + class MaintenanceStatus(Enum): """Describe the various statuses for maintenance""" diff --git a/backend/dataall/modules/maintenance/api/input_types.py b/backend/dataall/modules/maintenance/api/input_types.py deleted file mode 100644 index 7a0a8c517..000000000 --- a/backend/dataall/modules/maintenance/api/input_types.py +++ /dev/null @@ -1,11 +0,0 @@ -"""The module defines GraphQL input types for the Maintenance Window Activity""" - -from dataall.base.api import gql - -MaintenanceWindowInput = gql.InputType( - name='MaintenanceWindowInput', - arguments=[ - gql.Argument('status', gql.NonNullableType(gql.String)), - gql.Argument('mode', gql.String), - ], -) \ No newline at end of file diff --git a/backend/dataall/modules/maintenance/api/queries.py b/backend/dataall/modules/maintenance/api/queries.py index 826678758..11c623cf0 100644 --- a/backend/dataall/modules/maintenance/api/queries.py +++ b/backend/dataall/modules/maintenance/api/queries.py @@ -1,17 +1,9 @@ -"""The module defines GraphQL queries for the SageMaker notebooks""" +"""The module defines GraphQL queries for the Maintenance Activity>""" from dataall.base.api import gql -from dataall.modules.maintenance.api.resolvers import get_maintenance_window_status, get_maintenance_window_mode +from dataall.modules.maintenance.api.resolvers import get_maintenance_window_status getMaintenanceWindowStatus = gql.QueryField( - name='getMaintenanceWindowStatus', - type=gql.Ref('Maintenance'), - resolver=get_maintenance_window_status -) - -getMaintenanceWindowMode = gql.QueryField( - name='getMaintenanceWindowMode', - type=gql.String, - resolver=get_maintenance_window_mode + name='getMaintenanceWindowStatus', type=gql.Ref('Maintenance'), resolver=get_maintenance_window_status ) diff --git a/backend/dataall/modules/maintenance/api/resolvers.py b/backend/dataall/modules/maintenance/api/resolvers.py index 38c738d3a..132dcca97 100644 --- a/backend/dataall/modules/maintenance/api/resolvers.py +++ b/backend/dataall/modules/maintenance/api/resolvers.py @@ -1,5 +1,3 @@ -import logging - from dataall.base.api.context import Context from dataall.core.permissions.services.tenant_policy_service import TenantPolicyValidationService from dataall.modules.maintenance.api.enums import MaintenanceModes @@ -10,27 +8,20 @@ def start_maintenance_window(context: Context, source: Maintenance, mode: str): """Starts the maintenance window""" if mode not in [item.value for item in list(MaintenanceModes)]: - raise Exception('Mode is not conforming to the MaintenanceModes enums') - # Check from the context if the groups contains the DataAdminstrators group + raise Exception('Mode is not conforming to the MaintenanceModes enum') + # Check from the context if the groups contains the DAAAdminstrators group if not TenantPolicyValidationService.is_tenant_admin(context.groups): raise Exception('Only data.all admin group members can start maintenance window') return MaintenanceService.start_maintenance_window(engine=context.engine, mode=mode) def stop_maintenance_window(context: Context, source: Maintenance): - # Check from the context if the groups contains the DataAdminstrators group + # Check from the context if the groups contains the DAAAdminstrators group if not TenantPolicyValidationService.is_tenant_admin(context.groups): raise Exception('Only data.all admin group members can stop maintenance window') return MaintenanceService.stop_maintenance_window(engine=context.engine) def get_maintenance_window_status(context: Context, source: Maintenance): - if not TenantPolicyValidationService.is_tenant_admin(context.groups): - raise Exception('Only data.all admin group members can access maintenance endpoints') return MaintenanceService.get_maintenance_window_status(engine=context.engine) - -def get_maintenance_window_mode(context: Context, source: Maintenance): - if not TenantPolicyValidationService.is_tenant_admin(context.groups): - raise Exception('Only data.all admin group members can access maintenance endpoints') - return MaintenanceService.get_maintenance_window_mode(engine=context) diff --git a/backend/dataall/modules/maintenance/api/types.py b/backend/dataall/modules/maintenance/api/types.py index 8ea279337..f464f3b50 100644 --- a/backend/dataall/modules/maintenance/api/types.py +++ b/backend/dataall/modules/maintenance/api/types.py @@ -1,22 +1,8 @@ -"""Defines the object types of the SageMaker notebooks""" +"""Defines the object types of the Maintenance activity""" from dataall.base.api import gql -from dataall.modules.notebooks.api.resolvers import ( - resolve_notebook_stack, - resolve_notebook_status, - resolve_user_role, -) - -from dataall.core.environment.api.resolvers import resolve_environment -from dataall.core.organizations.api.resolvers import resolve_organization_by_env - -from dataall.modules.notebooks.api.enums import SagemakerNotebookRole Maintenance = gql.ObjectType( - name='Maintenance', - fields=[ - gql.Field(name='status', type=gql.String), - gql.Field(name='mode', type=gql.String) - ] + name='Maintenance', fields=[gql.Field(name='status', type=gql.String), gql.Field(name='mode', type=gql.String)] ) diff --git a/backend/dataall/modules/maintenance/db/maintenance_models.py b/backend/dataall/modules/maintenance/db/maintenance_models.py index 910d98031..d20870dda 100644 --- a/backend/dataall/modules/maintenance/db/maintenance_models.py +++ b/backend/dataall/modules/maintenance/db/maintenance_models.py @@ -1,4 +1,4 @@ -"""ORM models for maintenance windo""" +"""ORM models for maintenance activity""" from sqlalchemy import Column, String @@ -7,6 +7,7 @@ class Maintenance(Base): """ORM Model for maintenance window""" + __tablename__ = 'maintenance' status = Column(String, nullable=False, primary_key=True) mode = Column(String, default='', nullable=True) diff --git a/backend/dataall/modules/maintenance/db/maintenance_repository.py b/backend/dataall/modules/maintenance/db/maintenance_repository.py index ebbf114a4..f45614205 100644 --- a/backend/dataall/modules/maintenance/db/maintenance_repository.py +++ b/backend/dataall/modules/maintenance/db/maintenance_repository.py @@ -7,12 +7,13 @@ log = logging.getLogger(__name__) -class MaintenanceRepository: +class MaintenanceRepository: def __init__(self, session): self._session = session - def save_maintenance_status_and_mode(self, maintenance_status: str, maintenance_mode: str): + def save_maintenance_status_and_mode(self, maintenance_status: str, maintenance_mode: str): + log.debug(f'Saving maintenance status and mode as {maintenance_status} , {maintenance_mode} respectively') maintenance_record = self._session.query(Maintenance).one() maintenance_record.status = maintenance_status maintenance_record.mode = maintenance_mode @@ -24,4 +25,3 @@ def get_maintenance_record(self): def get_maintenance_mode(self): maintenance_record = self._session.query(Maintenance) return maintenance_record.mode - diff --git a/backend/dataall/modules/maintenance/services/maintenance_permissions.py b/backend/dataall/modules/maintenance/services/maintenance_permissions.py deleted file mode 100644 index b6274b5b7..000000000 --- a/backend/dataall/modules/maintenance/services/maintenance_permissions.py +++ /dev/null @@ -1,49 +0,0 @@ -""" -Add module's permissions to the global permissions. -Contains permissions for sagemaker notebooks -""" - -from dataall.core.permissions.services.resources_permissions import ( - RESOURCES_ALL_WITH_DESC, - RESOURCES_ALL, -) - -from dataall.core.permissions.services.environment_permissions import ( - ENVIRONMENT_INVITED, - ENVIRONMENT_INVITATION_REQUEST, - ENVIRONMENT_ALL, -) - -from dataall.core.permissions.services.tenant_permissions import TENANT_ALL, TENANT_ALL_WITH_DESC - -GET_MAINTENANCE_STATUS -UPDATE_MAINTENANCE_STATUS -MANAGE_MAINTENANCE_STATUS - -GET_NOTEBOOK = 'GET_NOTEBOOK' -UPDATE_NOTEBOOK = 'UPDATE_NOTEBOOK' -DELETE_NOTEBOOK = 'DELETE_NOTEBOOK' -CREATE_NOTEBOOK = 'CREATE_NOTEBOOK' -MANAGE_NOTEBOOKS = 'MANAGE_NOTEBOOKS' - -NOTEBOOK_ALL = [ - GET_NOTEBOOK, - DELETE_NOTEBOOK, - UPDATE_NOTEBOOK, -] - -ENVIRONMENT_ALL.append(CREATE_NOTEBOOK) -ENVIRONMENT_INVITED.append(CREATE_NOTEBOOK) -ENVIRONMENT_INVITATION_REQUEST.append(CREATE_NOTEBOOK) - -TENANT_ALL.append(MANAGE_NOTEBOOKS) -TENANT_ALL_WITH_DESC[MANAGE_NOTEBOOKS] = 'Manage notebooks' - - -RESOURCES_ALL.append(CREATE_NOTEBOOK) -RESOURCES_ALL.extend(NOTEBOOK_ALL) - -RESOURCES_ALL_WITH_DESC[CREATE_NOTEBOOK] = 'Create notebooks on this environment' -RESOURCES_ALL_WITH_DESC[GET_NOTEBOOK] = 'General permission to get a notebook' -RESOURCES_ALL_WITH_DESC[DELETE_NOTEBOOK] = 'Permission to delete a notebook' -RESOURCES_ALL_WITH_DESC[UPDATE_NOTEBOOK] = 'Permission to edit a notebook' diff --git a/backend/dataall/modules/maintenance/services/maintenance_service.py b/backend/dataall/modules/maintenance/services/maintenance_service.py index f01f704a8..b788f752f 100644 --- a/backend/dataall/modules/maintenance/services/maintenance_service.py +++ b/backend/dataall/modules/maintenance/services/maintenance_service.py @@ -1,6 +1,6 @@ """ -A service layer for sagemaker notebooks -Central part for working with notebooks +A service layer for maintenance activity +Defines functions and business logic to be performed for maintenance window """ import dataclasses @@ -29,75 +29,82 @@ class MaintenanceService: - @staticmethod def start_maintenance_window(engine, mode: str = None): # Update the RDS table with the mode and status to PENDING - logger.info("Putting data.all into maintenance") + # Disable all scheduled ECS tasks which are created by data.all + logger.info('Putting data.all into maintenance') try: with engine.scoped_session() as session: maintenance_record = MaintenanceRepository(session).get_maintenance_record() - if maintenance_record.status == MaintenanceStatus.PENDING.value or maintenance_record.status == MaintenanceStatus.ACTIVE.value: - logger.error("Maintenance window already in PENDING or ACTIVE state. Cannot start maintenance window. Stop the maintenance window and start again") + if ( + maintenance_record.status == MaintenanceStatus.PENDING.value + or maintenance_record.status == MaintenanceStatus.ACTIVE.value + ): + logger.error( + 'Maintenance window already in PENDING or ACTIVE state. Cannot start maintenance window. Stop the maintenance window and start again' + ) return False - MaintenanceRepository(session).save_maintenance_status_and_mode(maintenance_status=MaintenanceStatus.PENDING.value ,maintenance_mode=mode) + MaintenanceRepository(session).save_maintenance_status_and_mode( + maintenance_status=MaintenanceStatus.PENDING.value, maintenance_mode=mode + ) # Disable scheduled ECS tasks # Get all the SSMs related to the scheduled tasks ecs_scheduled_rules = ParameterStoreManager.get_parameters_by_path( region=os.getenv('AWS_REGION', 'eu-west-1'), - parameter_path=f"/dataall/{os.getenv('envname', 'local')}/ecs/ecs_scheduled_tasks/rule" + parameter_path=f"/dataall/{os.getenv('envname', 'local')}/ecs/ecs_scheduled_tasks/rule", ) - logger.info(ecs_scheduled_rules) + logger.debug(ecs_scheduled_rules) ecs_scheduled_rules_list = [item['Value'] for item in ecs_scheduled_rules] - logger.info("Value of ecs scheduled tasks") - logger.info(ecs_scheduled_rules_list) event_bridge_session = EventBridge(region=os.getenv('AWS_REGION', 'eu-west-1')) event_bridge_session.disable_scheduled_ecs_tasks(ecs_scheduled_rules_list) return True except Exception as e: - logger.error(f"Error occurred while starting maintenance window due to {e}") + logger.error(f'Error occurred while starting maintenance window due to {e}') return False @staticmethod def stop_maintenance_window(engine): # Update the RDS table by changing mode to - '' # Update the RDS table by changing the status to INACTIVE - logger.info("Stopping maintenance") + # Enabled all the ECS Scheduled task + logger.info('Stopping maintenance mode') try: with engine.scoped_session() as session: maintenance_record = MaintenanceRepository(session).get_maintenance_record() if maintenance_record.status == MaintenanceStatus.INACTIVE.value: - logger.error("Maintenance window already in INACTIVE state. Cannot stop maintenance window") + logger.error('Maintenance window already in INACTIVE state. Cannot stop maintenance window') return False - MaintenanceRepository(session).save_maintenance_status_and_mode(maintenance_status='INACTIVE', maintenance_mode='') + MaintenanceRepository(session).save_maintenance_status_and_mode( + maintenance_status='INACTIVE', maintenance_mode='' + ) # Enable scheduled ECS tasks ecs_scheduled_rules = ParameterStoreManager.get_parameters_by_path( region=os.getenv('AWS_REGION', 'eu-west-1'), - parameter_path=f"/dataall/{os.getenv('envname', 'local')}/ecs/ecs_scheduled_tasks/rule" + parameter_path=f"/dataall/{os.getenv('envname', 'local')}/ecs/ecs_scheduled_tasks/rule", ) - logger.info(ecs_scheduled_rules) + logger.debug(ecs_scheduled_rules) ecs_scheduled_rules_list = [item['Value'] for item in ecs_scheduled_rules] - logger.info("Value of ecs scheduled tasks") - logger.info(ecs_scheduled_rules_list) - event_bridge_session = EventBridge() + event_bridge_session = EventBridge(region=os.getenv('AWS_REGION', 'eu-west-1')) event_bridge_session.enable_scheduled_ecs_tasks(ecs_scheduled_rules_list) return True except Exception as e: - logger.error(f"Error occurred while stopping maintenance window due to {e}") + logger.error(f'Error occurred while stopping maintenance window due to {e}') return False - @staticmethod def get_maintenance_window_status(engine): - logger.info("Checking maintenance window status") - with engine.scoped_session() as session: - try: + logger.info('Checking maintenance window status') + # Checks if all ECS tasks in the data.all infra account have completed + # Updates the maintenance status and returns maintenance record + try: + with engine.scoped_session() as session: maintenance_record = MaintenanceRepository(session).get_maintenance_record() if maintenance_record.status == MaintenanceStatus.PENDING.value: - # Check all the ECS tasks + # Check if ECS tasks are running ecs_cluster_name = ParameterStoreManager.get_parameter_value( region=os.getenv('AWS_REGION', 'eu-west-1'), - parameter_path=f"/dataall/{os.getenv('envname', 'local')}/ecs/cluster/name" + parameter_path=f"/dataall/{os.getenv('envname', 'local')}/ecs/cluster/name", ) if Ecs.is_task_running(cluster_name=ecs_cluster_name): return maintenance_record @@ -106,15 +113,19 @@ def get_maintenance_window_status(engine): session.commit() return maintenance_record else: - logger.info("Maintenance window is not in PENDING state") + logger.info('Maintenance window is not in PENDING state') return maintenance_record - except Exception as e: - logger.error(f'Error while getting maintenance window status due to {e}') - raise e + except Exception as e: + logger.error(f'Error while getting maintenance window status due to {e}') + raise e @staticmethod - def get_maintenance_window_mode(engine): - logger.info("Fetching status of maintenance window") - with engine.scoped_session() as session: - maintenance_record = MaintenanceRepository(session).get_maintenance_record() - return maintenance_record.mode + def _get_maintenance_window_mode(engine): + logger.info('Fetching status of maintenance window') + try: + with engine.scoped_session() as session: + maintenance_record = MaintenanceRepository(session).get_maintenance_record() + return maintenance_record.mode + except Exception as e: + logger.error(f'Error while getting maintenance window mode due to {e}') + raise e diff --git a/backend/migrations/versions/b833ad41db68_maintenance_window_schema.py b/backend/migrations/versions/b833ad41db68_maintenance_window_schema.py index 20f8d0518..a15d8d57c 100644 --- a/backend/migrations/versions/b833ad41db68_maintenance_window_schema.py +++ b/backend/migrations/versions/b833ad41db68_maintenance_window_schema.py @@ -5,6 +5,7 @@ Create Date: 2024-04-16 19:30:05.226603 """ + import os from alembic import op @@ -45,7 +46,7 @@ def upgrade(): op.create_table( 'maintenance', sa.Column('status', sa.String(), nullable=False, primary_key=True), - sa.Column('mode', sa.String(), nullable=True, default='') + sa.Column('mode', sa.String(), nullable=True, default=''), ) maintenance_record: [Maintenance] = Maintenance(status='INACTIVE', mode='') @@ -54,7 +55,7 @@ def upgrade(): session.commit() except Exception as e: - print(f'Failed to create migration for maintenance table') + print('Failed to create migration for maintenance table') raise e diff --git a/backend/search_handler.py b/backend/search_handler.py index d5bf0d7d0..585eb377d 100644 --- a/backend/search_handler.py +++ b/backend/search_handler.py @@ -1,9 +1,16 @@ import json import os +from dataall.base.db import get_engine from dataall.base.searchproxy import connect, run_query +from dataall.base.config import config +from dataall.base.utils.api_handler_utils import send_unauthorized_response, get_custom_groups, get_cognito_groups +from dataall.core.permissions.services.tenant_policy_service import TenantPolicyValidationService +from dataall.modules.maintenance.api.enums import MaintenanceModes, MaintenanceStatus +from dataall.modules.maintenance.services.maintenance_service import MaintenanceService ENVNAME = os.getenv('envname', 'local') +ENGINE = get_engine(envname=ENVNAME) es = connect(envname=ENVNAME) @@ -21,26 +28,53 @@ def handler(event, context): }, } elif event['httpMethod'] == 'POST': - # If maintenance mode is enabled -> Check Status by using the graphQL Endpoint - # If groups doesn't contain data.all administrator group - # Check what is the access mode - # Return response with error "Maintenance Window is ON" - - body = event.get('body') - print(body) - success = True - try: - response = run_query(es, 'dataall-index', body) - except Exception: - success = False - response = {} - return { - 'statusCode': 200 if success else 400, - 'headers': { - 'content-type': 'application/json', - 'Access-Control-Allow-Origin': '*', - 'Access-Control-Allow-Headers': '*', - 'Access-Control-Allow-Methods': '*', - }, - 'body': json.dumps(response), - } + if 'authorizer' in event['requestContext']: + if 'claims' not in event['requestContext']['authorizer']: + claims = event['requestContext']['authorizer'] + else: + claims = event['requestContext']['authorizer']['claims'] + + # Needed for custom groups + user_id = claims['email'] + if 'user_id' in event['requestContext']['authorizer']: + user_id = event['requestContext']['authorizer']['user_id'] + + groups = [] + if os.environ.get('custom_auth', None): + groups.extend(get_custom_groups(user_id)) + else: + groups.extend(get_cognito_groups(claims)) + + # Check if maintenance window is enabled AND if the maintenance mode is NO-ACCESS + if config.get_property('modules.maintenance.active'): + if ( + (MaintenanceService._get_maintenance_window_mode(engine=ENGINE) == MaintenanceModes.NOACCESS.value) + and ( + MaintenanceService.get_maintenance_window_status(engine=ENGINE).status + is not MaintenanceStatus.INACTIVE.value + ) + and not TenantPolicyValidationService.is_tenant_admin(groups) + ): + send_unauthorized_response( + query={'operationName': 'OpensearchIndex'}, + message='Access Restricted: data.all is currently undergoing maintenance, and your actions are temporarily blocked.', + ) + + body = event.get('body') + print(body) + success = True + try: + response = run_query(es, 'dataall-index', body) + except Exception: + success = False + response = {} + return { + 'statusCode': 200 if success else 400, + 'headers': { + 'content-type': 'application/json', + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Headers': '*', + 'Access-Control-Allow-Methods': '*', + }, + 'body': json.dumps(response), + } diff --git a/config.json b/config.json index e85529f9e..769b06a2f 100644 --- a/config.json +++ b/config.json @@ -36,7 +36,8 @@ "active": true }, "maintenance": { - "active" : true + "active" : true, + "custom_maintenance_text" : "data.all in maintenance. Please reach out to us at #data-users slack channel" } }, "core": { diff --git a/deploy/stacks/lambda_api.py b/deploy/stacks/lambda_api.py index 3fce0d215..677e285d2 100644 --- a/deploy/stacks/lambda_api.py +++ b/deploy/stacks/lambda_api.py @@ -68,6 +68,9 @@ def __init__( self.esproxy_dlq = self.set_dlq(f'{resource_prefix}-{envname}-esproxy-dlq') esproxy_sg = self.create_lambda_sgs(envname, 'esproxy', resource_prefix, vpc) + esproxy_env = {'envname': envname, 'LOG_LEVEL': 'INFO'} + if custom_auth: + esproxy_env['custom_auth'] = custom_auth.get('provider', None) self.elasticsearch_proxy_handler = _lambda.DockerImageFunction( self, 'ElasticSearchProxyHandler', @@ -81,7 +84,7 @@ def __init__( security_groups=[esproxy_sg], memory_size=1664 if prod_sizing else 256, timeout=Duration.minutes(15), - environment={'envname': envname, 'LOG_LEVEL': 'INFO'}, + environment=esproxy_env, dead_letter_queue_enabled=True, dead_letter_queue=self.esproxy_dlq, on_failure=lambda_destination.SqsDestination(self.esproxy_dlq), @@ -386,6 +389,10 @@ def create_function_role(self, envname, resource_prefix, fn_name, pivot_role_nam f'arn:aws:aoss:{self.region}:{self.account}:collection/*', ], ), + iam.PolicyStatement( + actions=['events:EnableRule', 'events:DisableRule'], + resources=[f'arn:aws:events:{self.region}:{self.account}:rule/dataall*'], + ), ], ) role = iam.Role( diff --git a/deploy/stacks/param_store_stack.py b/deploy/stacks/param_store_stack.py index 02060f690..d4c7f9f6b 100644 --- a/deploy/stacks/param_store_stack.py +++ b/deploy/stacks/param_store_stack.py @@ -3,10 +3,7 @@ import string import boto3 -from aws_cdk import ( - aws_ssm, - custom_resources as cr -) +from aws_cdk import aws_ssm, custom_resources as cr from .pyNestedStack import pyNestedClass from .deploy_config import deploy_config @@ -134,20 +131,19 @@ def __init__( ) if prod_sizing: cr.AwsCustomResource( - self, - "SSMParamSettingHighThroughput", + self, + 'SSMParamSettingHighThroughput', on_update=cr.AwsSdkCall( - service="SSM", - action="UpdateServiceSettingCommand", - parameters={ - "SettingId": "/ssm/parameter-store/high-throughput-enabled", - "SettingValue": "true" - }, - physical_resource_id=cr.PhysicalResourceId.of(f"ssm-high-throughput-{self.account}-{self.region}") + service='SSM', + action='UpdateServiceSettingCommand', + parameters={'SettingId': '/ssm/parameter-store/high-throughput-enabled', 'SettingValue': 'true'}, + physical_resource_id=cr.PhysicalResourceId.of(f'ssm-high-throughput-{self.account}-{self.region}'), ), policy=cr.AwsCustomResourcePolicy.from_sdk_calls( - resources=[f"arn:aws:ssm:{self.region}:{self.account}:servicesetting/ssm/parameter-store/high-throughput-enabled"] - ) + resources=[ + f'arn:aws:ssm:{self.region}:{self.account}:servicesetting/ssm/parameter-store/high-throughput-enabled' + ] + ), ) diff --git a/frontend/src/App.js b/frontend/src/App.js index 1329c8b46..d8d9c2374 100644 --- a/frontend/src/App.js +++ b/frontend/src/App.js @@ -5,11 +5,10 @@ import { createMaterialTheme, useScrollReset, useSettings, - LoadingScreen, SplashScreen + LoadingScreen } from './design'; import routes from './routes'; import { useAuth } from './authentication'; -import {isMaintenanceMode} from "./services/graphql/MaintenanceWindow"; export const App = () => { const content = useRoutes(routes); @@ -27,8 +26,7 @@ export const App = () => { return ( - {/*{auth.isInitialized ? isMaintenanceMode() ? content : : }*/} - {auth.isInitialized ? content : } + {auth.isInitialized ? content : } ); }; diff --git a/frontend/src/authentication/components/AuthGuard.js b/frontend/src/authentication/components/AuthGuard.js index b1e74b0ff..a7712d496 100644 --- a/frontend/src/authentication/components/AuthGuard.js +++ b/frontend/src/authentication/components/AuthGuard.js @@ -1,5 +1,5 @@ import PropTypes from 'prop-types'; -import { useState } from 'react'; +import {useEffect, useState} from 'react'; import { Navigate, useLocation } from 'react-router-dom'; import { Login } from '../views/Login'; import { useAuth } from '../hooks'; @@ -7,16 +7,43 @@ import { RegexToValidateWindowPathName, WindowPathLengthThreshold } from '../../utils'; -import {isMaintenanceMode} from "../../services/graphql/MaintenanceWindow"; -import {useClient} from "../../services"; -import {NoAccessMaintenanceWindow} from "../../design"; +import {useClient, useGroups} from '../../services'; +import { NoAccessMaintenanceWindow } from '../../design'; +import {getMaintenanceStatus} from "../../services/graphql/MaintenanceWindow"; +import {ACTIVE_STATUS, PENDING_STATUS} from "../../modules/Administration/components"; +import config from "../../generated/config.json" +import {SET_ERROR, useDispatch} from "../../globalErrors"; export const AuthGuard = (props) => { const { children } = props; const auth = useAuth(); const location = useLocation(); const [requestedLocation, setRequestedLocation] = useState(null); + const [isNoAccessMaintenance, setNoAccessMaintenanceFlag] = useState(false) const client = useClient(); + const groups = useGroups(); + const dispatch = useDispatch(); + + const checkMaintenanceMode = async ()=>{ + const response = await client.query(getMaintenanceStatus()); + if (!response.errors && response.data.getMaintenanceWindowStatus != null){ + if ([PENDING_STATUS, ACTIVE_STATUS].includes(response.data.getMaintenanceWindowStatus.status) && response.data.getMaintenanceWindowStatus.mode === 'NO-ACCESS' + && !groups.includes('DAAdministrators')){ + setNoAccessMaintenanceFlag(true) + } + } + } + + useEffect(async () => { + // Check if the maintenance window is enabled and has NO-ACCESS Status + // If yes then display a blank screen with a message that data.all is in maintenance mode ( Check use of isNoAccessMaintenance state ) + if (config.modules.maintenance.active === true){ + if (client){ + checkMaintenanceMode().catch((e) => dispatch({ type: SET_ERROR, e })) + } + } + }, [client, groups]); + if (!auth.isAuthenticated) { if (location.pathname !== requestedLocation) { @@ -59,16 +86,10 @@ export const AuthGuard = (props) => { sessionStorage.removeItem('window-location'); } - // Check if the maintenance window is enabled and has NO-ACCESS Status - // If yes then display a blank screen with a message that data.all is in maintenance mode - if (client){ - // Replace this call with query call getting mode - if (isMaintenanceMode()){ - return - } + if (isNoAccessMaintenance){ + return ; } - return <>{children}; }; diff --git a/frontend/src/authentication/contexts/LocalAuthContext.js b/frontend/src/authentication/contexts/LocalAuthContext.js index 29fe3642c..4c6764fcc 100644 --- a/frontend/src/authentication/contexts/LocalAuthContext.js +++ b/frontend/src/authentication/contexts/LocalAuthContext.js @@ -3,9 +3,9 @@ import { createContext, useEffect, useReducer } from 'react'; import { SET_ERROR } from 'globalErrors'; const anonymousUser = { - id: 'someone@amazon.com', - email: 'someone@amazon.com', - name: 'someone@amazon.com' + id: 'anonymous@amazon.com', + email: 'anonymous@amazon.com', + name: 'anonymous@amazon.com' }; const initialState = { isAuthenticated: true, diff --git a/frontend/src/design/components/NoAccessMaintenanceWindow.js b/frontend/src/design/components/NoAccessMaintenanceWindow.js index e71145ba6..feeb99e33 100644 --- a/frontend/src/design/components/NoAccessMaintenanceWindow.js +++ b/frontend/src/design/components/NoAccessMaintenanceWindow.js @@ -1,6 +1,5 @@ -import {Box, Typography} from '@mui/material'; -import { Logo } from './Logo'; -import React from "react"; +import { Box, Typography } from '@mui/material'; +import React from 'react'; export const NoAccessMaintenanceWindow = () => ( ( zIndex: 2000 }} > - - data.all is in maintenance mode. Please contact data.all administrators for any assistance. - - + + data.all is in maintenance mode. Please contact data.all administrators + for any assistance. + ); diff --git a/frontend/src/design/components/index.js b/frontend/src/design/components/index.js index 064ad7229..cd01380bb 100644 --- a/frontend/src/design/components/index.js +++ b/frontend/src/design/components/index.js @@ -29,4 +29,4 @@ export * from './defaults'; export * from './layout'; export * from './popovers'; export * from './SanitizedHTML'; -export * from './NoAccessMaintenanceWindow' +export * from './NoAccessMaintenanceWindow'; diff --git a/frontend/src/design/components/layout/DefaultNavbar.js b/frontend/src/design/components/layout/DefaultNavbar.js index 761beb45a..74341942d 100644 --- a/frontend/src/design/components/layout/DefaultNavbar.js +++ b/frontend/src/design/components/layout/DefaultNavbar.js @@ -1,5 +1,5 @@ -import React from 'react'; -import {AppBar, Box, IconButton, Toolbar, Typography} from '@mui/material'; +import React, {useEffect, useState} from 'react'; +import { AppBar, Box, IconButton, Toolbar, Typography } from '@mui/material'; import { makeStyles } from '@mui/styles'; import { Menu } from '@mui/icons-material'; import PropTypes from 'prop-types'; @@ -7,7 +7,12 @@ import { AccountPopover, NotificationsPopover } from '../popovers'; import { Logo } from '../Logo'; import { SettingsDrawer } from '../SettingsDrawer'; import { ModuleNames, isModuleEnabled } from 'utils'; -import {isMaintenanceMode} from "../../../services/graphql/MaintenanceWindow"; +import config from '../../../generated/config.json' +import {ACTIVE_STATUS, MaintenanceViewer, PENDING_STATUS} from "../../../modules/Administration/components"; +import {useClient} from "../../../services"; +import {getMaintenanceStatus} from "../../../services/graphql/MaintenanceWindow"; +import {SET_ERROR, useDispatch} from "../../../globalErrors"; +import {SanitizedHTML} from "../SanitizedHTML"; const useStyles = makeStyles((theme) => ({ appBar: { @@ -18,14 +23,49 @@ const useStyles = makeStyles((theme) => ({ export const DefaultNavbar = ({ openDrawer, onOpenDrawerChange }) => { const classes = useStyles(); + const [isMaintenance, setMaintenanceFlag] = useState(false) + const dispatch = useDispatch(); + const client = useClient() + + useEffect(async () => { + if (client){ + const response = await client.query(getMaintenanceStatus()); + if ( + !response.errors && + response.data.getMaintenanceWindowStatus !== null + ) { + if (response.data.getMaintenanceWindowStatus.status === ACTIVE_STATUS || response.data.getMaintenanceWindowStatus.status === PENDING_STATUS ){ + setMaintenanceFlag(true) + } + }else{ + const error = response.errors + ? response.errors[0].message + : 'Could not fetch status of maintenance window'; + dispatch({ type: SET_ERROR, error }); + } + } + }, [client]); + return ( - {isMaintenanceMode() ? - - data.all is in maintenance mode. You can still navigate inside data.all but during this period, please do not make any modifications to any data.all assets ( datasets, environment, etc ). - - : <>} + {config.modules.maintenance.active && isMaintenance ? ( + + + {config.modules.maintenance.custom_maintenance_text !== undefined ? ( + + ) : ( + data.all is in maintenance mode. You can still navigate inside + data.all but during this period, please do not make any + modifications to any data.all assets ( datasets, environment, etc ).)} + + + + ) : ( + <> + )} {!openDrawer && ( diff --git a/frontend/src/modules/Administration/components/MaintenanceViewer.js b/frontend/src/modules/Administration/components/MaintenanceViewer.js index 18a384ed6..cc25cb8b6 100644 --- a/frontend/src/modules/Administration/components/MaintenanceViewer.js +++ b/frontend/src/modules/Administration/components/MaintenanceViewer.js @@ -1,289 +1,439 @@ import { - Box, - Button, - Card, - CardHeader, - CircularProgress, - Dialog, - Divider, - Grid, IconButton, - MenuItem, - TextField, - Typography -} from "@mui/material"; -import React, {useCallback, useEffect, useState} from "react"; -import {Article, CancelRounded, SystemUpdate} from "@mui/icons-material"; -import {LoadingButton} from "@mui/lab"; -import {Label} from "../../../design"; -import {isMaintenanceMode} from "../../../services/graphql/MaintenanceWindow"; -import {useClient} from "../../../services"; -import {SET_ERROR, useDispatch} from "../../../globalErrors"; -import {useSnackbar} from "notistack"; + Box, + Button, + Card, + CardHeader, + CircularProgress, + Dialog, + Divider, + Grid, + IconButton, + MenuItem, + TextField, + Typography +} from '@mui/material'; +import React, { useCallback, useEffect, useState } from 'react'; +import { Article, CancelRounded, SystemUpdate } from '@mui/icons-material'; +import { LoadingButton } from '@mui/lab'; +import { Label } from '../../../design'; +import { + getMaintenanceStatus, + stopMaintenanceWindow, + startMaintenanceWindow +} from '../../../services/graphql/MaintenanceWindow'; +import { useClient } from '../../../services'; +import { SET_ERROR, useDispatch } from '../../../globalErrors'; +import { useSnackbar } from 'notistack'; const maintenanceModes = [ - {value: "READ-ONLY", label: "Read-Only"}, - {value: "NO-ACCESS", label: "No-Access"} -] + { value: 'READ-ONLY', label: 'Read-Only' }, + { value: 'NO-ACCESS', label: 'No-Access' } +]; + +const START_MAINTENANCE = 'Start Maintenance'; +const END_MAINTENANCE = 'End Maintenance'; +export const PENDING_STATUS = 'PENDING' +export const ACTIVE_STATUS = 'ACTIVE' +export const INACTIVE_STATUS = 'INACTIVE' -export const MaintenanceConfirmationPopUp = ({popUp, setPopUp, mode, confirmedMode, setConfirmedMode, maintenanceButtonText, setMaintenanceButtonText, setDropDownStatus, refreshingTimer, startRefreshPolling}) => { +export const MaintenanceConfirmationPopUp = (props) => { + const { + popUp, + setPopUp, + confirmedMode, + setConfirmedMode, + maintenanceButtonText, + setMaintenanceButtonText, + setDropDownStatus, + refreshingTimer, + setMaintenanceWindowStatus + } = props; + const client = useClient(); + const dispatch = useDispatch(); + const { enqueueSnackbar } = useSnackbar(); - const handlePopUpModal = () => { - // Call the GrapQL API and then after the success is received change the UI - if (maintenanceButtonText === 'Start Maintenance') { - // Call the GraphQL to enable maintenance window - setMaintenanceButtonText('End Maintenance') - // Freeze the dropdown menu - setDropDownStatus(true) - // Start the Timer - startRefreshPolling() - }else if (maintenanceButtonText === 'End Maintenance'){ - // Call the GraphQL to disable maintenance window - setMaintenanceButtonText('Start Maintenance') - // Unfreeze the dropdown menu - setDropDownStatus(false) - // End the running timer as well - clearInterval(refreshingTimer) + const handlePopUpModal = async () => { + if (maintenanceButtonText === START_MAINTENANCE) { + if (!client) { + dispatch({ + type: SET_ERROR, + error: 'Client not initialized for starting maintenance window' + }); + } + const response = await client.mutate( + startMaintenanceWindow({ mode: confirmedMode }) + ); + if (!response.errors && response.data.startMaintenanceWindow != null) { + const respData = response.data.startMaintenanceWindow; + if (respData === true) { + setMaintenanceButtonText(END_MAINTENANCE); + setDropDownStatus(false); + enqueueSnackbar( + 'Maintenance Window Started. Please check the status', + { + anchorOrigin: { + horizontal: 'right', + vertical: 'top' + }, + variant: 'success' + } + ); + } else { + enqueueSnackbar('Could not start maintenance window', { + anchorOrigin: { + horizontal: 'right', + vertical: 'top' + }, + variant: 'success' + }); } - setConfirmedMode(mode) - setPopUp(false) + } else { + const error = response.errors + ? response.errors[0].message + : 'Something went wrong while starting maintenance window. Please check gql logs'; + dispatch({ type: SET_ERROR, error }); + } + } else if (maintenanceButtonText === END_MAINTENANCE) { + const response = await client.mutate(stopMaintenanceWindow()); + if ( + !response.errors && + response.data.stopMaintenanceWindow != null && + response.data.stopMaintenanceWindow === true + ) { + setMaintenanceButtonText(START_MAINTENANCE); + // Unfreeze the dropdown menu + setDropDownStatus(true); + // End the running timer as well + clearInterval(refreshingTimer); + setConfirmedMode(''); + setMaintenanceWindowStatus(INACTIVE_STATUS); + enqueueSnackbar('Maintenance Window Stopped', { + anchorOrigin: { + horizontal: 'right', + vertical: 'top' + }, + variant: 'success' + }); + } else { + const error = response.errors + ? response.errors[0].message + : 'Something went wrong while stopping maintenance window. Please check gql logs'; + dispatch({ type: SET_ERROR, error }); + } } + setPopUp(false); + }; - return ( - - - - - Are you sure you want to {maintenanceButtonText.toLowerCase()}? - - }/> - - - - - - - - - ) -} + return ( + + + + + Are you sure you want to {maintenanceButtonText.toLowerCase()}? + + } + /> + + + + + + + + + ); +}; export const MaintenanceViewer = () => { - const client = useClient(); - const [updating, setUpdating] = useState(false); - const [mode, setMode] = useState('') - const [popUp, setPopUp] = useState(false) - const [confirmedMode, setConfirmedMode] = useState('') - const [maintenanceButtonText, setMaintenanceButtonText] = useState('Start Maintenance') - const [maintenanceWindowStatus, setMaintenanceWindowStatus] = useState('INACTIVE') - const [dropDownStatus, setDropDownStatus] = useState(false) - const [refreshingTimer, setRefreshingTimer] = useState('') - const { enqueueSnackbar, closeSnackbar } = useSnackbar(); - const dispatch = useDispatch(); + const client = useClient(); + const [refreshing, setRefreshing] = useState(false); + const [updating, setUpdating] = useState(false); + const [mode, setMode] = useState(''); + const [popUp, setPopUp] = useState(false); + const [confirmedMode, setConfirmedMode] = useState(''); + const [maintenanceButtonText, setMaintenanceButtonText] = + useState(START_MAINTENANCE); + const [maintenanceWindowStatus, setMaintenanceWindowStatus] = + useState(INACTIVE_STATUS); + const [dropDownStatus, setDropDownStatus] = useState(false); + const [refreshingTimer, setRefreshingTimer] = useState(''); + const { enqueueSnackbar, closeSnackbar } = useSnackbar(); + const dispatch = useDispatch(); - const refreshMaintenanceView = async () =>{ - console.log("Refreshing the maintenance view now!!!") - // Call the que - setUpdating(true) - setTimeout(() =>{ - setUpdating(false) - }, 2000) + const refreshMaintenanceView = async () => { + setUpdating(true); + setRefreshing(true); + getMaintenanceWindowStatus() + .then((data) => { + setMaintenanceWindowStatus(data.status); + if (data.status === INACTIVE_STATUS) { + setMaintenanceButtonText(START_MAINTENANCE); + setConfirmedMode(''); + setDropDownStatus(true); + clearInterval(refreshingTimer); + } else { + setMaintenanceButtonText(END_MAINTENANCE); + setConfirmedMode( + maintenanceModes.find((obj) => obj.value === data.mode).label + ); + setDropDownStatus(false); + } + setUpdating(false); + setRefreshing(false); + }) + .catch((e) => dispatch({ type: SET_ERROR, e })); + }; - refreshStatus().catch((e) => dispatch({ type: SET_ERROR, error: e.message })) - return true + const getMaintenanceWindowStatus = async () => { + if (client) { + const response = await client.query(getMaintenanceStatus()); + if ( + !response.errors && + response.data.getMaintenanceWindowStatus !== null + ) { + return response.data.getMaintenanceWindowStatus; + } else { + const error = response.errors + ? response.errors[0].message + : 'Could not fetch status of maintenance window'; + dispatch({ type: SET_ERROR, error }); + } } + }; - const startMaintenanceWindow = () => { - // Check if proper maintenance mode is selected - // Use Formik forms for this in the future - console.log(`value of the mode is ${mode}`) - if (!['READ-ONLY', 'NO-ACCESS'].includes(mode) && maintenanceButtonText === 'Start Maintenance'){ - dispatch({ type: SET_ERROR, error: 'Please select correct maintenance mode' }) - return false; - } - setPopUp(true) - return true; + const startMaintenanceWindow = () => { + // Check if proper maintenance mode is selected + if ( + !maintenanceModes.map((obj) => obj.value).includes(mode) && + maintenanceButtonText === 'Start Maintenance' + ) { + dispatch({ + type: SET_ERROR, + error: 'Please select correct maintenance mode' + }); } + setConfirmedMode(mode); + setPopUp(true); + }; - const startRefreshPolling = useCallback( - async () => { - console.log("I am here in the refresh polling ") - if (client){ - const setTimer = setInterval(() => { - refreshStatus().catch((e) => dispatch({ type: SET_ERROR, error: e.message }))} - , [10000]) - setRefreshingTimer(setTimer) - } - } - , [client]) - - const refreshStatus = async () => { - closeSnackbar(); - await console.log("gsdlfjslf") - console.log("Refreshing the status of the maintenance window") - // Call the query to get the status of the maintenance window - // Update the status of the maintenance window - // Enqueue Snack bar to show that the maintenance window status is being polled + const refreshStatus = async () => { + closeSnackbar(); + const response = await client.query(getMaintenanceStatus()); + if (!response.errors && response.data.getMaintenanceWindowStatus !== null) { + const maintenanceStatusData = response.data.getMaintenanceWindowStatus; + setMaintenanceWindowStatus(maintenanceStatusData.status); + if ( + maintenanceStatusData.status === INACTIVE_STATUS || + maintenanceStatusData.status === ACTIVE_STATUS + ) { + clearInterval(refreshingTimer); + } else { enqueueSnackbar( - - - - - - - - Maintenance Window Status is being updated !! - - + + + + - , - { - key: new Date().getTime() + Math.random(), - anchorOrigin: { - horizontal: 'right', - vertical: 'top' - }, - variant: 'info', - persist: true, - action: (key) => ( - { - closeSnackbar(key); - }} + + - - - ) - } - ); + Maintenance Window Status is being updated !! + + + + , + { + key: new Date().getTime() + Math.random(), + anchorOrigin: { + horizontal: 'right', + vertical: 'top' + }, + variant: 'info', + persist: true, + action: (key) => ( + { + closeSnackbar(key); + }} + > + + + ) + } + ); + } + } else { + const error = response.errors + ? response.errors[0].message + : 'Maintenance Status not found. Something went wrong'; + dispatch({ type: SET_ERROR, error }); } + }; - useEffect(() => { - if (client) { - // For the first time - // Check if the maintenance mode is ON - if (isMaintenanceMode()){ - // If ON, then - // Fetch the value of the maintenance mode and paste it on the text field, disable the text field - // Make the button say "End Maintenance" mode - // Fetch the Status of the maintenance mode - // Also, edit the Maintenance mode value - const maintenanceMode = 'READ-ONLY' // GET THIS FROM GRAPHQL ENDPOINT - setMaintenanceButtonText('End Maintenance') - setMaintenanceWindowStatus('PENDING') // GET THIS FROM GRAPHQL ENDPOINT - setConfirmedMode(maintenanceMode) - setDropDownStatus(true) - - const setTimer = setInterval(() => { - refreshStatus().catch((e) => dispatch({ type: SET_ERROR, error: e.message }))} - , [10000]) - setRefreshingTimer(setTimer) - return () => clearInterval(setTimer) + const initializeMaintenanceView = useCallback(async () => { + const response = await client.query(getMaintenanceStatus()); + if (!response.errors && response.data.getMaintenanceWindowStatus !== null) { + const maintenanceStatusData = response.data.getMaintenanceWindowStatus; + if ( + maintenanceStatusData.status === PENDING_STATUS || + maintenanceStatusData.status === ACTIVE_STATUS + ) { + setMaintenanceButtonText('End Maintenance'); + setMaintenanceWindowStatus(maintenanceStatusData.status); + setConfirmedMode( + maintenanceModes.find( + (obj) => obj.value === maintenanceStatusData.mode + ).label + ); + setDropDownStatus(false); + } else if (maintenanceStatusData.status === INACTIVE_STATUS) { + setMaintenanceButtonText('Start Maintenance'); + setConfirmedMode(''); + setDropDownStatus(true); + } + } else { + const error = response.errors + ? response.errors[0].message + : 'Maintenance Status not found. Something went wrong'; + dispatch({ type: SET_ERROR, error }); + } + }, [client]); - }else{ - // If OFF, then - // Make the button say "Start Maintenance" - // Clear the status and maintenance mode values - setMaintenanceButtonText('Start Maintenance') - setConfirmedMode('') - } - } - }, [client]); + useEffect(() => { + if (client) { + initializeMaintenanceView().catch((e) => + dispatch({ type: SET_ERROR, e }) + ); + const setTimer = setInterval(() => { + refreshStatus().catch((e) => + dispatch({ type: SET_ERROR, error: e.message }) + ); + }, [10000]); + setRefreshingTimer(setTimer); + return () => clearInterval(setTimer); + } + }, [client]); - return ( + return ( + + {refreshing ? ( + + ) : ( - - - Create a Maintenance Window - - } /> - - - - - {setMode(event.target.value)}} - select - value={mode} - variant="outlined" - disabled={dropDownStatus} - > - {maintenanceModes.map((group) => ( - - {group.label} - - ))} - - - - - } - sx={{ m: 1 }} - variant="contained" - > - Refresh - - - - - - - Maintenance window status : {maintenanceWindowStatus === 'ACTIVE' ? () : maintenanceWindowStatus === 'PENDING' ? () : maintenanceWindowStatus === 'INACTIVE' ? () : <> - } - - - | - - - Current maintenance mode : {confirmedMode} - - - - - - Note - For safe deployments, please deploy when the status is ACTIVE - + + Create a Maintenance Window} /> + + + + + { + setMode(event.target.value); + }} + select + value={mode} + variant="outlined" + disabled={!dropDownStatus} + > + {maintenanceModes.map((group) => ( + + {group.label} + + ))} + - - + + + } + sx={{ m: 1 }} + variant="contained" + > + Refresh + + + + + + + Maintenance window status :{' '} + {maintenanceWindowStatus === ACTIVE_STATUS ? ( + + ) : maintenanceWindowStatus === PENDING_STATUS ? ( + + ) : maintenanceWindowStatus === INACTIVE_STATUS ? ( + + ) : ( + <> - + )} + + + | + + + Current maintenance mode : {confirmedMode} + + + + + + Note - For safe deployments, please deploy when the status is{' '} + + + + + - ) -} \ No newline at end of file + )} + + ); +}; diff --git a/frontend/src/modules/Administration/components/index.js b/frontend/src/modules/Administration/components/index.js index 27cd64355..99765e234 100644 --- a/frontend/src/modules/Administration/components/index.js +++ b/frontend/src/modules/Administration/components/index.js @@ -1,4 +1,4 @@ export * from './AdministrationTeams'; export * from './AdministratorDashboardViewer'; export * from './TeamPermissionsEditForm'; -export * from './MaintenanceViewer' +export * from './MaintenanceViewer'; diff --git a/frontend/src/modules/Administration/views/AdministrationView.js b/frontend/src/modules/Administration/views/AdministrationView.js index 15931713f..16128c926 100644 --- a/frontend/src/modules/Administration/views/AdministrationView.js +++ b/frontend/src/modules/Administration/views/AdministrationView.js @@ -13,13 +13,21 @@ import { useState } from 'react'; import { Helmet } from 'react-helmet-async'; import { Link as RouterLink } from 'react-router-dom'; import { ChevronRightIcon, useSettings } from 'design'; -import { AdministrationTeams, DashboardViewer, MaintenanceViewer } from '../components'; +import config from '../../../generated/config.json'; +import { + AdministrationTeams, + DashboardViewer, + MaintenanceViewer +} from '../components'; const tabs = [ { label: 'Teams', value: 'teams' }, - { label: 'Monitoring', value: 'dashboard' }, - { label: 'Maintenance', value: 'maintenance' } + { label: 'Monitoring', value: 'dashboard' } ]; +// Using 'config' as the isModulesEnabled needs Modules.MAINTENANCE which involves much bigger setup which is not needed for maintenance module +if (config.modules.maintenance.active) { + tabs.push({ label: 'Maintenance', value: 'maintenance' }); +} const AdministrationView = () => { const { settings } = useSettings(); diff --git a/frontend/src/services/graphql/MaintenanceWindow/getMaintenanceStatus.js b/frontend/src/services/graphql/MaintenanceWindow/getMaintenanceStatus.js new file mode 100644 index 000000000..3995cadc5 --- /dev/null +++ b/frontend/src/services/graphql/MaintenanceWindow/getMaintenanceStatus.js @@ -0,0 +1,12 @@ +import { gql } from 'apollo-boost'; + +export const getMaintenanceStatus = () => ({ + query: gql` + query getMaintenanceWindowStatus { + getMaintenanceWindowStatus { + status + mode + } + } + ` +}); diff --git a/frontend/src/services/graphql/MaintenanceWindow/index.js b/frontend/src/services/graphql/MaintenanceWindow/index.js index dd431b117..ca55e15e2 100644 --- a/frontend/src/services/graphql/MaintenanceWindow/index.js +++ b/frontend/src/services/graphql/MaintenanceWindow/index.js @@ -1 +1,3 @@ -export * from './isMaintenanceMode' \ No newline at end of file +export * from './getMaintenanceStatus'; +export * from './stopMaintenanceWindow'; +export * from './startMaintenanceWindow'; diff --git a/frontend/src/services/graphql/MaintenanceWindow/isMaintenanceMode.js b/frontend/src/services/graphql/MaintenanceWindow/isMaintenanceMode.js deleted file mode 100644 index 66ea88077..000000000 --- a/frontend/src/services/graphql/MaintenanceWindow/isMaintenanceMode.js +++ /dev/null @@ -1,12 +0,0 @@ -import { gql } from 'apollo-boost'; -export const isMaintenanceMode = () =>{ - return false; -} -// export const isMaintenanceMode = () => ({ -// query: gql` -// query isMaintenanceMode { -// isMaintenanceMode{ -// inMaintenance -// } -// }` -// }) \ No newline at end of file diff --git a/frontend/src/services/graphql/MaintenanceWindow/startMaintenanceWindow.js b/frontend/src/services/graphql/MaintenanceWindow/startMaintenanceWindow.js new file mode 100644 index 000000000..1005ba007 --- /dev/null +++ b/frontend/src/services/graphql/MaintenanceWindow/startMaintenanceWindow.js @@ -0,0 +1,10 @@ +import { gql } from 'apollo-boost'; + +export const startMaintenanceWindow = ({ mode }) => ({ + variables: { mode }, + mutation: gql` + mutation startMaintenanceWindow($mode: String) { + startMaintenanceWindow(mode: $mode) + } + ` +}); diff --git a/frontend/src/services/graphql/MaintenanceWindow/stopMaintenanceWindow.js b/frontend/src/services/graphql/MaintenanceWindow/stopMaintenanceWindow.js new file mode 100644 index 000000000..22cd88ea0 --- /dev/null +++ b/frontend/src/services/graphql/MaintenanceWindow/stopMaintenanceWindow.js @@ -0,0 +1,9 @@ +import { gql } from 'apollo-boost'; + +export const stopMaintenanceWindow = () => ({ + mutation: gql` + mutation stopMaintenanceWindow { + stopMaintenanceWindow + } + ` +}); From 1fe784b1dbf3397753d9c82d3f7eb3623ce41577 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Thu, 25 Apr 2024 14:53:39 -0500 Subject: [PATCH 14/36] Comments and make lint plus yarn lint fix --- backend/dataall/base/aws/event_bridge.py | 3 +- backend/dataall/base/aws/parameter_store.py | 5 +- .../modules/maintenance/api/resolvers.py | 1 - .../authentication/components/AuthGuard.js | 49 +++++++++------- .../components/NoAccessMaintenanceWindow.js | 18 ++++-- .../design/components/layout/DefaultNavbar.js | 58 +++++++++++-------- .../components/MaintenanceViewer.js | 20 +++---- 7 files changed, 88 insertions(+), 66 deletions(-) diff --git a/backend/dataall/base/aws/event_bridge.py b/backend/dataall/base/aws/event_bridge.py index d4a980e5a..0b489a029 100644 --- a/backend/dataall/base/aws/event_bridge.py +++ b/backend/dataall/base/aws/event_bridge.py @@ -1,5 +1,4 @@ import logging -import os import boto3 @@ -25,5 +24,5 @@ def disable_scheduled_ecs_tasks(self, list_of_tasks): for ecs_task in list_of_tasks: self.client.disable_rule(Name=ecs_task) except Exception as e: - logger.error(f'Error while re-enabling scheduled ecs tasks due to {e}') + logger.error(f'Error while disabling scheduled ecs tasks due to {e}') raise e diff --git a/backend/dataall/base/aws/parameter_store.py b/backend/dataall/base/aws/parameter_store.py index 7957e0ef0..9d40c9965 100644 --- a/backend/dataall/base/aws/parameter_store.py +++ b/backend/dataall/base/aws/parameter_store.py @@ -42,13 +42,12 @@ def get_parameters_by_path(AwsAccountId=None, region=None, parameter_path=None): if not parameter_path: raise Exception('Parameter name is None') try: - parameter_value = ParameterStoreManager.client(AwsAccountId, region).get_parameters_by_path( + parameter_values = ParameterStoreManager.client(AwsAccountId, region).get_parameters_by_path( Path=parameter_path )['Parameters'] - log.info(ParameterStoreManager.client(AwsAccountId, region).get_parameters_by_path(Path=parameter_path)) except ClientError as e: raise Exception(e) - return parameter_value + return parameter_values @staticmethod def update_parameter(AwsAccountId, region, parameter_name, parameter_value): diff --git a/backend/dataall/modules/maintenance/api/resolvers.py b/backend/dataall/modules/maintenance/api/resolvers.py index 132dcca97..1ad3127ec 100644 --- a/backend/dataall/modules/maintenance/api/resolvers.py +++ b/backend/dataall/modules/maintenance/api/resolvers.py @@ -24,4 +24,3 @@ def stop_maintenance_window(context: Context, source: Maintenance): def get_maintenance_window_status(context: Context, source: Maintenance): return MaintenanceService.get_maintenance_window_status(engine=context.engine) - diff --git a/frontend/src/authentication/components/AuthGuard.js b/frontend/src/authentication/components/AuthGuard.js index a7712d496..17c55b55d 100644 --- a/frontend/src/authentication/components/AuthGuard.js +++ b/frontend/src/authentication/components/AuthGuard.js @@ -1,5 +1,5 @@ import PropTypes from 'prop-types'; -import {useEffect, useState} from 'react'; +import { useEffect, useState } from 'react'; import { Navigate, useLocation } from 'react-router-dom'; import { Login } from '../views/Login'; import { useAuth } from '../hooks'; @@ -7,44 +7,51 @@ import { RegexToValidateWindowPathName, WindowPathLengthThreshold } from '../../utils'; -import {useClient, useGroups} from '../../services'; +import { useClient, useGroups } from '../../services'; import { NoAccessMaintenanceWindow } from '../../design'; -import {getMaintenanceStatus} from "../../services/graphql/MaintenanceWindow"; -import {ACTIVE_STATUS, PENDING_STATUS} from "../../modules/Administration/components"; -import config from "../../generated/config.json" -import {SET_ERROR, useDispatch} from "../../globalErrors"; +import { getMaintenanceStatus } from '../../services/graphql/MaintenanceWindow'; +import { + ACTIVE_STATUS, + PENDING_STATUS +} from '../../modules/Administration/components'; +import config from '../../generated/config.json'; +import { SET_ERROR, useDispatch } from '../../globalErrors'; export const AuthGuard = (props) => { const { children } = props; const auth = useAuth(); const location = useLocation(); const [requestedLocation, setRequestedLocation] = useState(null); - const [isNoAccessMaintenance, setNoAccessMaintenanceFlag] = useState(false) + const [isNoAccessMaintenance, setNoAccessMaintenanceFlag] = useState(false); const client = useClient(); const groups = useGroups(); const dispatch = useDispatch(); - const checkMaintenanceMode = async ()=>{ - const response = await client.query(getMaintenanceStatus()); - if (!response.errors && response.data.getMaintenanceWindowStatus != null){ - if ([PENDING_STATUS, ACTIVE_STATUS].includes(response.data.getMaintenanceWindowStatus.status) && response.data.getMaintenanceWindowStatus.mode === 'NO-ACCESS' - && !groups.includes('DAAdministrators')){ - setNoAccessMaintenanceFlag(true) - } - } - } + const checkMaintenanceMode = async () => { + const response = await client.query(getMaintenanceStatus()); + if (!response.errors && response.data.getMaintenanceWindowStatus != null) { + if ( + [PENDING_STATUS, ACTIVE_STATUS].includes( + response.data.getMaintenanceWindowStatus.status + ) && + response.data.getMaintenanceWindowStatus.mode === 'NO-ACCESS' && + !groups.includes('DAAdministrators') + ) { + setNoAccessMaintenanceFlag(true); + } + } + }; useEffect(async () => { // Check if the maintenance window is enabled and has NO-ACCESS Status // If yes then display a blank screen with a message that data.all is in maintenance mode ( Check use of isNoAccessMaintenance state ) - if (config.modules.maintenance.active === true){ - if (client){ - checkMaintenanceMode().catch((e) => dispatch({ type: SET_ERROR, e })) + if (config.modules.maintenance.active === true) { + if (client) { + checkMaintenanceMode().catch((e) => dispatch({ type: SET_ERROR, e })); } } }, [client, groups]); - if (!auth.isAuthenticated) { if (location.pathname !== requestedLocation) { setRequestedLocation(location.pathname); @@ -86,7 +93,7 @@ export const AuthGuard = (props) => { sessionStorage.removeItem('window-location'); } - if (isNoAccessMaintenance){ + if (isNoAccessMaintenance) { return ; } diff --git a/frontend/src/design/components/NoAccessMaintenanceWindow.js b/frontend/src/design/components/NoAccessMaintenanceWindow.js index feeb99e33..7036a8d11 100644 --- a/frontend/src/design/components/NoAccessMaintenanceWindow.js +++ b/frontend/src/design/components/NoAccessMaintenanceWindow.js @@ -1,5 +1,7 @@ import { Box, Typography } from '@mui/material'; import React from 'react'; +import config from '../../generated/config.json'; +import { SanitizedHTML } from './SanitizedHTML'; export const NoAccessMaintenanceWindow = () => ( ( zIndex: 2000 }} > - - data.all is in maintenance mode. Please contact data.all administrators - for any assistance. - + {config.modules.maintenance.custom_maintenance_text !== undefined ? ( + + + + ) : ( + + data.all is in maintenance mode. Please contact data.all administrators + for any assistance. + + )} ); diff --git a/frontend/src/design/components/layout/DefaultNavbar.js b/frontend/src/design/components/layout/DefaultNavbar.js index 74341942d..158c6edb4 100644 --- a/frontend/src/design/components/layout/DefaultNavbar.js +++ b/frontend/src/design/components/layout/DefaultNavbar.js @@ -1,4 +1,4 @@ -import React, {useEffect, useState} from 'react'; +import React, { useEffect, useState } from 'react'; import { AppBar, Box, IconButton, Toolbar, Typography } from '@mui/material'; import { makeStyles } from '@mui/styles'; import { Menu } from '@mui/icons-material'; @@ -7,12 +7,15 @@ import { AccountPopover, NotificationsPopover } from '../popovers'; import { Logo } from '../Logo'; import { SettingsDrawer } from '../SettingsDrawer'; import { ModuleNames, isModuleEnabled } from 'utils'; -import config from '../../../generated/config.json' -import {ACTIVE_STATUS, MaintenanceViewer, PENDING_STATUS} from "../../../modules/Administration/components"; -import {useClient} from "../../../services"; -import {getMaintenanceStatus} from "../../../services/graphql/MaintenanceWindow"; -import {SET_ERROR, useDispatch} from "../../../globalErrors"; -import {SanitizedHTML} from "../SanitizedHTML"; +import config from '../../../generated/config.json'; +import { + ACTIVE_STATUS, + PENDING_STATUS +} from '../../../modules/Administration/components'; +import { useClient } from '../../../services'; +import { getMaintenanceStatus } from '../../../services/graphql/MaintenanceWindow'; +import { SET_ERROR, useDispatch } from '../../../globalErrors'; +import { SanitizedHTML } from '../SanitizedHTML'; const useStyles = makeStyles((theme) => ({ appBar: { @@ -23,22 +26,25 @@ const useStyles = makeStyles((theme) => ({ export const DefaultNavbar = ({ openDrawer, onOpenDrawerChange }) => { const classes = useStyles(); - const [isMaintenance, setMaintenanceFlag] = useState(false) + const [isMaintenance, setMaintenanceFlag] = useState(false); const dispatch = useDispatch(); - const client = useClient() + const client = useClient(); useEffect(async () => { - if (client){ + if (client) { const response = await client.query(getMaintenanceStatus()); if ( !response.errors && response.data.getMaintenanceWindowStatus !== null ) { - if (response.data.getMaintenanceWindowStatus.status === ACTIVE_STATUS || response.data.getMaintenanceWindowStatus.status === PENDING_STATUS ){ - setMaintenanceFlag(true) + if ( + response.data.getMaintenanceWindowStatus.status === ACTIVE_STATUS || + response.data.getMaintenanceWindowStatus.status === PENDING_STATUS + ) { + setMaintenanceFlag(true); } - }else{ - const error = response.errors + } else { + const error = response.errors ? response.errors[0].message : 'Could not fetch status of maintenance window'; dispatch({ type: SET_ERROR, error }); @@ -46,22 +52,24 @@ export const DefaultNavbar = ({ openDrawer, onOpenDrawerChange }) => { } }, [client]); - return ( {config.modules.maintenance.active && isMaintenance ? ( - - {config.modules.maintenance.custom_maintenance_text !== undefined ? ( - - ) : ( + {config.modules.maintenance.custom_maintenance_text !== undefined ? ( + + + + ) : ( + data.all is in maintenance mode. You can still navigate inside - data.all but during this period, please do not make any - modifications to any data.all assets ( datasets, environment, etc ).)} - - + data.all but during this period, please do not make any + modifications to any data.all assets ( datasets, environment, etc + ). + + )} ) : ( <> diff --git a/frontend/src/modules/Administration/components/MaintenanceViewer.js b/frontend/src/modules/Administration/components/MaintenanceViewer.js index cc25cb8b6..459528940 100644 --- a/frontend/src/modules/Administration/components/MaintenanceViewer.js +++ b/frontend/src/modules/Administration/components/MaintenanceViewer.js @@ -32,9 +32,9 @@ const maintenanceModes = [ const START_MAINTENANCE = 'Start Maintenance'; const END_MAINTENANCE = 'End Maintenance'; -export const PENDING_STATUS = 'PENDING' -export const ACTIVE_STATUS = 'ACTIVE' -export const INACTIVE_STATUS = 'INACTIVE' +export const PENDING_STATUS = 'PENDING'; +export const ACTIVE_STATUS = 'ACTIVE'; +export const INACTIVE_STATUS = 'INACTIVE'; export const MaintenanceConfirmationPopUp = (props) => { const { @@ -67,6 +67,7 @@ export const MaintenanceConfirmationPopUp = (props) => { const respData = response.data.startMaintenanceWindow; if (respData === true) { setMaintenanceButtonText(END_MAINTENANCE); + setMaintenanceWindowStatus(PENDING_STATUS); setDropDownStatus(false); enqueueSnackbar( 'Maintenance Window Started. Please check the status', @@ -183,7 +184,7 @@ export const MaintenanceViewer = () => { const refreshMaintenanceView = async () => { setUpdating(true); setRefreshing(true); - getMaintenanceWindowStatus() + _getMaintenanceWindowStatus() .then((data) => { setMaintenanceWindowStatus(data.status); if (data.status === INACTIVE_STATUS) { @@ -204,7 +205,7 @@ export const MaintenanceViewer = () => { .catch((e) => dispatch({ type: SET_ERROR, e })); }; - const getMaintenanceWindowStatus = async () => { + const _getMaintenanceWindowStatus = async () => { if (client) { const response = await client.query(getMaintenanceStatus()); if ( @@ -225,7 +226,7 @@ export const MaintenanceViewer = () => { // Check if proper maintenance mode is selected if ( !maintenanceModes.map((obj) => obj.value).includes(mode) && - maintenanceButtonText === 'Start Maintenance' + maintenanceButtonText === START_MAINTENANCE ) { dispatch({ type: SET_ERROR, @@ -260,7 +261,7 @@ export const MaintenanceViewer = () => { sx={{ color: '#fff' }} variant="subtitle2" > - Maintenance Window Status is being updated !! + Maintenance Window Status is being updated @@ -301,7 +302,7 @@ export const MaintenanceViewer = () => { maintenanceStatusData.status === PENDING_STATUS || maintenanceStatusData.status === ACTIVE_STATUS ) { - setMaintenanceButtonText('End Maintenance'); + setMaintenanceButtonText(END_MAINTENANCE); setMaintenanceWindowStatus(maintenanceStatusData.status); setConfirmedMode( maintenanceModes.find( @@ -310,7 +311,7 @@ export const MaintenanceViewer = () => { ); setDropDownStatus(false); } else if (maintenanceStatusData.status === INACTIVE_STATUS) { - setMaintenanceButtonText('Start Maintenance'); + setMaintenanceButtonText(START_MAINTENANCE); setConfirmedMode(''); setDropDownStatus(true); } @@ -327,7 +328,6 @@ export const MaintenanceViewer = () => { initializeMaintenanceView().catch((e) => dispatch({ type: SET_ERROR, e }) ); - const setTimer = setInterval(() => { refreshStatus().catch((e) => dispatch({ type: SET_ERROR, error: e.message }) From 627d31ca60e78b517c515fe8eeedec515a754b49 Mon Sep 17 00:00:00 2001 From: trajopadhye Date: Thu, 25 Apr 2024 16:16:22 -0500 Subject: [PATCH 15/36] Making container id for ssm param to sched task id --- deploy/stacks/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/stacks/container.py b/deploy/stacks/container.py index 253f8ba53..83325e029 100644 --- a/deploy/stacks/container.py +++ b/deploy/stacks/container.py @@ -621,7 +621,7 @@ def set_scheduled_task( # Add the rule of the scheduled task to parameter store ssm.StringParameter( self, - f'ECSTaskRule-{scheduled_task.event_rule.rule_name}', + f'ECSTaskRule-{scheduled_task_id}', parameter_name=f'/dataall/{self._envname}/ecs/ecs_scheduled_tasks/rule/{scheduled_task_id}', string_value=scheduled_task.event_rule.rule_name, ) From 093fab2fc761d1d2358435645ec19fd5bae35a2b Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Tue, 30 Apr 2024 11:24:35 -0500 Subject: [PATCH 16/36] Changes from aws deployed data.all after testing --- backend/api_handler.py | 17 ++- deploy/stacks/backend_stack.py | 1 + .../authentication/components/AuthGuard.js | 21 ++- .../components/MaintenanceViewer.js | 5 +- frontend/src/services/hooks/useGroups.js | 9 +- .../db => modules/maintenance}/__init__.py | 0 .../maintenance/test_mainenance_gql.py | 124 ++++++++++++++++++ 7 files changed, 156 insertions(+), 21 deletions(-) rename tests/{core/permissions/db => modules/maintenance}/__init__.py (100%) create mode 100644 tests/modules/maintenance/test_mainenance_gql.py diff --git a/backend/api_handler.py b/backend/api_handler.py index aca36a91c..29faa9671 100644 --- a/backend/api_handler.py +++ b/backend/api_handler.py @@ -41,6 +41,10 @@ ENGINE = get_engine(envname=ENVNAME) Worker.queue = SqsQueue.send +TenantPolicyService.save_permissions_with_tenant(ENGINE) + +MAINTENANCE_ALLOWED_OPERATIONS = ["getGroupsForUser", "getMaintenanceWindowStatus"] + def resolver_adapter(resolver): def adapted(obj, info, **kwargs): @@ -162,10 +166,11 @@ def handler(event, context): ) and not TenantPolicyValidationService.is_tenant_admin(groups) ): - send_unauthorized_response( - query=query, - message='Access Restricted: data.all is currently undergoing maintenance, and your actions are temporarily blocked.', - ) + if query.get('operationName', '') not in MAINTENANCE_ALLOWED_OPERATIONS: + return send_unauthorized_response( + query=query, + message='Access Restricted: data.all is currently undergoing maintenance, and your actions are temporarily blocked.', + ) elif ( (MaintenanceService._get_maintenance_window_mode(engine=ENGINE) == MaintenanceModes.READONLY.value) and ( @@ -175,8 +180,8 @@ def handler(event, context): and not TenantPolicyValidationService.is_tenant_admin(groups) ): # If its mutation then block and return - if query.get('query', '').split(' ')[0] == 'mutation': - send_unauthorized_response( + if query.get('query', '').split()[0] == 'mutation': + return send_unauthorized_response( query=query, message='Access Restricted: data.all is currently undergoing maintenance, and your actions are temporarily blocked.', ) diff --git a/deploy/stacks/backend_stack.py b/deploy/stacks/backend_stack.py index 61c5a1ee8..90bccfaf0 100644 --- a/deploy/stacks/backend_stack.py +++ b/deploy/stacks/backend_stack.py @@ -303,6 +303,7 @@ def __init__( lambdas=[ self.lambda_api_stack.aws_handler, self.lambda_api_stack.api_handler, + self.lambda_api_stack.elasticsearch_proxy_handler, ], ecs_security_groups=self.ecs_stack.ecs_security_groups, prod_sizing=prod_sizing, diff --git a/frontend/src/authentication/components/AuthGuard.js b/frontend/src/authentication/components/AuthGuard.js index 17c55b55d..e7ef31b47 100644 --- a/frontend/src/authentication/components/AuthGuard.js +++ b/frontend/src/authentication/components/AuthGuard.js @@ -8,7 +8,7 @@ import { WindowPathLengthThreshold } from '../../utils'; import { useClient, useGroups } from '../../services'; -import { NoAccessMaintenanceWindow } from '../../design'; +import { LoadingScreen, NoAccessMaintenanceWindow } from '../../design'; import { getMaintenanceStatus } from '../../services/graphql/MaintenanceWindow'; import { ACTIVE_STATUS, @@ -22,7 +22,7 @@ export const AuthGuard = (props) => { const auth = useAuth(); const location = useLocation(); const [requestedLocation, setRequestedLocation] = useState(null); - const [isNoAccessMaintenance, setNoAccessMaintenanceFlag] = useState(false); + const [isNoAccessMaintenance, setNoAccessMaintenanceFlag] = useState(null); const client = useClient(); const groups = useGroups(); const dispatch = useDispatch(); @@ -38,6 +38,8 @@ export const AuthGuard = (props) => { !groups.includes('DAAdministrators') ) { setNoAccessMaintenanceFlag(true); + } else { + setNoAccessMaintenanceFlag(false); } } }; @@ -69,6 +71,17 @@ export const AuthGuard = (props) => { return ; } + if ( + isNoAccessMaintenance == null && + config.modules.maintenance.active === true + ) { + return ; + } + + if (isNoAccessMaintenance === true) { + return ; + } + if (requestedLocation && location.pathname !== requestedLocation) { setRequestedLocation(null); return ; @@ -93,10 +106,6 @@ export const AuthGuard = (props) => { sessionStorage.removeItem('window-location'); } - if (isNoAccessMaintenance) { - return ; - } - return <>{children}; }; diff --git a/frontend/src/modules/Administration/components/MaintenanceViewer.js b/frontend/src/modules/Administration/components/MaintenanceViewer.js index 459528940..c49f7f37a 100644 --- a/frontend/src/modules/Administration/components/MaintenanceViewer.js +++ b/frontend/src/modules/Administration/components/MaintenanceViewer.js @@ -45,7 +45,6 @@ export const MaintenanceConfirmationPopUp = (props) => { maintenanceButtonText, setMaintenanceButtonText, setDropDownStatus, - refreshingTimer, setMaintenanceWindowStatus } = props; const client = useClient(); @@ -104,8 +103,6 @@ export const MaintenanceConfirmationPopUp = (props) => { setMaintenanceButtonText(START_MAINTENANCE); // Unfreeze the dropdown menu setDropDownStatus(true); - // End the running timer as well - clearInterval(refreshingTimer); setConfirmedMode(''); setMaintenanceWindowStatus(INACTIVE_STATUS); enqueueSnackbar('Maintenance Window Stopped', { @@ -295,6 +292,7 @@ export const MaintenanceViewer = () => { }; const initializeMaintenanceView = useCallback(async () => { + setRefreshing(true); const response = await client.query(getMaintenanceStatus()); if (!response.errors && response.data.getMaintenanceWindowStatus !== null) { const maintenanceStatusData = response.data.getMaintenanceWindowStatus; @@ -321,6 +319,7 @@ export const MaintenanceViewer = () => { : 'Maintenance Status not found. Something went wrong'; dispatch({ type: SET_ERROR, error }); } + setRefreshing(false); }, [client]); useEffect(() => { diff --git a/frontend/src/services/hooks/useGroups.js b/frontend/src/services/hooks/useGroups.js index 87d7919ed..09afa1e5a 100644 --- a/frontend/src/services/hooks/useGroups.js +++ b/frontend/src/services/hooks/useGroups.js @@ -16,12 +16,9 @@ export const useGroups = () => { ) { setGroups(['Engineers', 'Scientists', 'DAAdministrators']); } else if (process.env.REACT_APP_CUSTOM_AUTH) { - if (!auth.user) { - dispatch({ - type: SET_ERROR, - error: 'Cannot Set User Groups as the User is not defined' - }); - } + // Returning when auth.user is not present + // Not dispatching error as useGroups is triggered in auth guard when the user is not authenticated + if (!auth.user) return; // return if the client is null, and then trigger this when the client is present if (client == null) return; const response = await client.query(getGroupsForUser(auth.user.short_id)); diff --git a/tests/core/permissions/db/__init__.py b/tests/modules/maintenance/__init__.py similarity index 100% rename from tests/core/permissions/db/__init__.py rename to tests/modules/maintenance/__init__.py diff --git a/tests/modules/maintenance/test_mainenance_gql.py b/tests/modules/maintenance/test_mainenance_gql.py new file mode 100644 index 000000000..f60461840 --- /dev/null +++ b/tests/modules/maintenance/test_mainenance_gql.py @@ -0,0 +1,124 @@ +from unittest.mock import MagicMock +from dataall.base.config import config + +import pytest + +from dataall.modules.maintenance.db.maintenance_models import Maintenance + + +@pytest.fixture(scope='module') +def mock_ecs_client(module_mocker): + module_mocker.patch( + 'dataall.modules.maintenance.services.maintenance_service.ParameterStoreManager.get_parameters_by_path', + return_value=[{'item': 'task1', 'Value': 'task1'}, {'item': 'task2', 'Value': 'task2'}], + ) + mock_events = MagicMock() + module_mocker.patch( + 'dataall.modules.maintenance.services.maintenance_service.EventBridge', return_value=mock_events + ) + mock_events().disable_scheduled_ecs_tasks.return_value = True + yield mock_events + + +@pytest.fixture(scope='function') +def init_maintenance_record(db): + with db.scoped_session() as session: + maintenance_record = Maintenance(status='INACTIVE', mode='') + session.add(maintenance_record) + session.commit() + yield + with db.scoped_session() as session: + maintenance_record = session.query(Maintenance).one() + session.delete(maintenance_record) + session.commit() + + +@pytest.mark.skipif(not config.get_property('modules.maintenance.active'), reason='Module disabled by config') +def test_start_maintenance_window(db, client, mock_ecs_client, init_maintenance_record): + response = client.query( + """ + mutation startMaintenanceWindow($mode: String!){ + startMaintenanceWindow(mode: $mode) + } + """, + mode='READ-ONLY', + username='alice', + groups=['DAAdministrators', 'Engineers'], + ) + + assert response + assert response.data.startMaintenanceWindow is True + + with db.scoped_session() as session: + maintenance_record = session.query(Maintenance).one() + assert maintenance_record.status == 'PENDING' + assert maintenance_record.mode == 'READ-ONLY' + + +@pytest.mark.skipif(not config.get_property('modules.maintenance.active'), reason='Module disabled by config') +def test_start_maintenance_window_with_team_not_a_data_admin(client, mock_ecs_client, init_maintenance_record): + response = client.query( + """ + mutation startMaintenanceWindow($mode: String!){ + startMaintenanceWindow(mode: $mode) + } + """, + mode='READ-ONLY', + username='alice', + groups=['Engineers'], + ) + + assert response + assert 'Only data.all admin group members can start maintenance window' in response.errors[0]['message'] + + +@pytest.mark.skipif(not config.get_property('modules.maintenance.active'), reason='Module disabled by config') +def test_stop_maintenance_window(db, client, mock_ecs_client, init_maintenance_record): + # Initialize the maintenance window with ACTIVE status and READ-ONLY mode + with db.scoped_session() as session: + maintenance_record = session.query(Maintenance).one() + maintenance_record.mode = 'READ-ONLY' + maintenance_record.status = 'ACTIVE' + session.add(maintenance_record) + session.commit() + + response = client.query( + """ + mutation stopMaintenanceWindow{ + stopMaintenanceWindow + } + """, + username='alice', + groups=['DAAdministrators', 'Engineers'], + ) + + assert response + assert response.data.stopMaintenanceWindow is True + + +@pytest.mark.skipif(not config.get_property('modules.maintenance.active'), reason='Module disabled by config') +def test_get_maintenance_window_status(db, client, mock_ecs_client, init_maintenance_record): + # Initialize the maintenance window with ACTIVE status and READ-ONLY mode + with db.scoped_session() as session: + maintenance_record = session.query(Maintenance).one() + maintenance_record.mode = 'READ-ONLY' + maintenance_record.status = 'ACTIVE' + session.add(maintenance_record) + session.commit() + + response = client.query( + """ + query getMaintenanceWindowStatus{ + getMaintenanceWindowStatus{ + status, + mode + } + } + """, + username='alice', + groups=['DAAdministrators', 'Engineers'], + ) + + assert response + assert response.data.getMaintenanceWindowStatus.status == 'ACTIVE' + assert response.data.getMaintenanceWindowStatus.mode == 'READ-ONLY' From fed1609ca25a7e99326400fdc6af2a98d3bef2c7 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Tue, 30 Apr 2024 12:20:31 -0500 Subject: [PATCH 17/36] Linting fixes --- backend/api_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/api_handler.py b/backend/api_handler.py index 29faa9671..b8da337cd 100644 --- a/backend/api_handler.py +++ b/backend/api_handler.py @@ -43,7 +43,7 @@ TenantPolicyService.save_permissions_with_tenant(ENGINE) -MAINTENANCE_ALLOWED_OPERATIONS = ["getGroupsForUser", "getMaintenanceWindowStatus"] +MAINTENANCE_ALLOWED_OPERATIONS = ['getGroupsForUser', 'getMaintenanceWindowStatus'] def resolver_adapter(resolver): From 11deb7b45d3101bf8ada078e0667f90c39bbb847 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Tue, 30 Apr 2024 12:35:45 -0500 Subject: [PATCH 18/36] Removing custom maintenance text --- config.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config.json b/config.json index dc1bd4863..521d3790e 100644 --- a/config.json +++ b/config.json @@ -36,8 +36,7 @@ "active": true }, "maintenance": { - "active" : true, - "custom_maintenance_text" : "data.all in maintenance. Please reach out to us at #data-users slack channel" + "active" : true } }, "core": { From f9964716e646fb7e6cab48ca600f1495a157c040 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Tue, 30 Apr 2024 12:39:38 -0500 Subject: [PATCH 19/36] Alembic script upgrade fix --- .../versions/b833ad41db68_maintenance_window_schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/migrations/versions/b833ad41db68_maintenance_window_schema.py b/backend/migrations/versions/b833ad41db68_maintenance_window_schema.py index a15d8d57c..c1c3e352d 100644 --- a/backend/migrations/versions/b833ad41db68_maintenance_window_schema.py +++ b/backend/migrations/versions/b833ad41db68_maintenance_window_schema.py @@ -16,7 +16,7 @@ # revision identifiers, used by Alembic. revision = 'b833ad41db68' -down_revision = '194608b1ff7f' +down_revision = 'c6d01930179d' branch_labels = None depends_on = None From 16a7dc1563a7ac622500d243a1be17020bea152b Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Tue, 30 Apr 2024 12:43:18 -0500 Subject: [PATCH 20/36] Fixing integration tests --- .../maintenance/services/maintenance_service.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/backend/dataall/modules/maintenance/services/maintenance_service.py b/backend/dataall/modules/maintenance/services/maintenance_service.py index b788f752f..9a131edb1 100644 --- a/backend/dataall/modules/maintenance/services/maintenance_service.py +++ b/backend/dataall/modules/maintenance/services/maintenance_service.py @@ -3,24 +3,11 @@ Defines functions and business logic to be performed for maintenance window """ -import dataclasses import logging import os -from dataclasses import dataclass, field -from typing import List, Dict from dataall.base.aws.event_bridge import EventBridge from dataall.base.aws.parameter_store import ParameterStoreManager -from dataall.base.context import get_context as context -from dataall.core.environment.db.environment_models import Environment -from dataall.core.environment.env_permission_checker import has_group_permission -from dataall.core.environment.services.environment_service import EnvironmentService -from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService -from dataall.core.permissions.services.tenant_policy_service import TenantPolicyService -from dataall.core.stacks.api import stack_helper -from dataall.core.stacks.db.keyvaluetag_repositories import KeyValueTag -from dataall.core.stacks.db.stack_repositories import Stack -from dataall.base.db import exceptions from dataall.modules.maintenance.api.enums import MaintenanceStatus from dataall.modules.maintenance.db.maintenance_repository import MaintenanceRepository from dataall.core.stacks.aws.ecs import Ecs From 672ee82d4aa4e1ee68f9175e754e3c1202199498 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Tue, 30 Apr 2024 13:44:52 -0500 Subject: [PATCH 21/36] Few final changes --- backend/api_handler.py | 3 --- .../services/maintenance_service.py | 21 +++++++++++-------- .../contexts/LocalAuthContext.js | 6 +++--- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/backend/api_handler.py b/backend/api_handler.py index b8da337cd..a2d07b953 100644 --- a/backend/api_handler.py +++ b/backend/api_handler.py @@ -11,7 +11,6 @@ ) from dataall.base.api import bootstrap as bootstrap_schema, get_executable_schema -from dataall.base.services.service_provider_factory import ServiceProviderFactory from dataall.base.utils.api_handler_utils import get_custom_groups, get_cognito_groups, send_unauthorized_response from dataall.core.tasks.service_handlers import Worker from dataall.base.aws.sqs import SqsQueue @@ -41,7 +40,6 @@ ENGINE = get_engine(envname=ENVNAME) Worker.queue = SqsQueue.send -TenantPolicyService.save_permissions_with_tenant(ENGINE) MAINTENANCE_ALLOWED_OPERATIONS = ['getGroupsForUser', 'getMaintenanceWindowStatus'] @@ -156,7 +154,6 @@ def handler(event, context): # Check if in some maintenance mode # Check if in maintenance status is not INACTIVE # Check if the user belongs to a 'DAAdministrators' group - # Todo : Add check to see if maintenance module is enabled or not from the config if config.get_property('modules.maintenance.active'): if ( (MaintenanceService._get_maintenance_window_mode(engine=ENGINE) == MaintenanceModes.NOACCESS.value) diff --git a/backend/dataall/modules/maintenance/services/maintenance_service.py b/backend/dataall/modules/maintenance/services/maintenance_service.py index 9a131edb1..f3ebacac4 100644 --- a/backend/dataall/modules/maintenance/services/maintenance_service.py +++ b/backend/dataall/modules/maintenance/services/maintenance_service.py @@ -16,10 +16,11 @@ class MaintenanceService: + + # Update the RDS table with the mode and status to PENDING + # Disable all scheduled ECS tasks which are created by data.all @staticmethod def start_maintenance_window(engine, mode: str = None): - # Update the RDS table with the mode and status to PENDING - # Disable all scheduled ECS tasks which are created by data.all logger.info('Putting data.all into maintenance') try: with engine.scoped_session() as session: @@ -36,7 +37,7 @@ def start_maintenance_window(engine, mode: str = None): maintenance_status=MaintenanceStatus.PENDING.value, maintenance_mode=mode ) # Disable scheduled ECS tasks - # Get all the SSMs related to the scheduled tasks + # Get all the SSM Params related to the scheduled tasks ecs_scheduled_rules = ParameterStoreManager.get_parameters_by_path( region=os.getenv('AWS_REGION', 'eu-west-1'), parameter_path=f"/dataall/{os.getenv('envname', 'local')}/ecs/ecs_scheduled_tasks/rule", @@ -50,11 +51,12 @@ def start_maintenance_window(engine, mode: str = None): logger.error(f'Error occurred while starting maintenance window due to {e}') return False + # Update the RDS table by changing mode to - '' + # Update the RDS table by changing the status to INACTIVE + # Enable all the ECS Scheduled task @staticmethod def stop_maintenance_window(engine): - # Update the RDS table by changing mode to - '' - # Update the RDS table by changing the status to INACTIVE - # Enabled all the ECS Scheduled task + logger.info('Stopping maintenance mode') try: with engine.scoped_session() as session: @@ -79,11 +81,11 @@ def stop_maintenance_window(engine): logger.error(f'Error occurred while stopping maintenance window due to {e}') return False + # Checks if all ECS tasks in the data.all infra account have completed + # Updates the maintenance status and returns maintenance record @staticmethod def get_maintenance_window_status(engine): logger.info('Checking maintenance window status') - # Checks if all ECS tasks in the data.all infra account have completed - # Updates the maintenance status and returns maintenance record try: with engine.scoped_session() as session: maintenance_record = MaintenanceRepository(session).get_maintenance_record() @@ -106,9 +108,10 @@ def get_maintenance_window_status(engine): logger.error(f'Error while getting maintenance window status due to {e}') raise e + # Fetches the mode of maintenance window @staticmethod def _get_maintenance_window_mode(engine): - logger.info('Fetching status of maintenance window') + logger.info('Fetching mode of maintenance window') try: with engine.scoped_session() as session: maintenance_record = MaintenanceRepository(session).get_maintenance_record() diff --git a/frontend/src/authentication/contexts/LocalAuthContext.js b/frontend/src/authentication/contexts/LocalAuthContext.js index 4c6764fcc..29fe3642c 100644 --- a/frontend/src/authentication/contexts/LocalAuthContext.js +++ b/frontend/src/authentication/contexts/LocalAuthContext.js @@ -3,9 +3,9 @@ import { createContext, useEffect, useReducer } from 'react'; import { SET_ERROR } from 'globalErrors'; const anonymousUser = { - id: 'anonymous@amazon.com', - email: 'anonymous@amazon.com', - name: 'anonymous@amazon.com' + id: 'someone@amazon.com', + email: 'someone@amazon.com', + name: 'someone@amazon.com' }; const initialState = { isAuthenticated: true, From cdf7eee4bea0fb1164cc4dc5aa7c6357652d75a9 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Tue, 30 Apr 2024 13:48:08 -0500 Subject: [PATCH 22/36] Linting --- .../dataall/modules/maintenance/services/maintenance_service.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/dataall/modules/maintenance/services/maintenance_service.py b/backend/dataall/modules/maintenance/services/maintenance_service.py index f3ebacac4..70e5a136a 100644 --- a/backend/dataall/modules/maintenance/services/maintenance_service.py +++ b/backend/dataall/modules/maintenance/services/maintenance_service.py @@ -16,7 +16,6 @@ class MaintenanceService: - # Update the RDS table with the mode and status to PENDING # Disable all scheduled ECS tasks which are created by data.all @staticmethod @@ -56,7 +55,6 @@ def start_maintenance_window(engine, mode: str = None): # Enable all the ECS Scheduled task @staticmethod def stop_maintenance_window(engine): - logger.info('Stopping maintenance mode') try: with engine.scoped_session() as session: From 5b83e7415f53bf4b3d427137d7fa432f73043f08 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Tue, 7 May 2024 12:59:03 -0500 Subject: [PATCH 23/36] Adding changes after code review --- .../dataall/base/utils/api_handler_utils.py | 3 +- .../modules/maintenance/api/resolvers.py | 10 ++---- .../dataall/modules/maintenance/api/types.py | 2 +- .../maintenance/db/maintenance_repository.py | 4 --- .../services/maintenance_service.py | 16 +++++++-- .../authentication/components/AuthGuard.js | 12 +++---- .../design/components/layout/DefaultNavbar.js | 33 ++++++++++--------- .../Administration/components/index.js | 3 +- .../views/AdministrationView.js | 8 ++--- frontend/src/modules/Maintenance/index.js | 5 +++ .../services}/getMaintenanceStatus.js | 0 .../Maintenance/services}/index.js | 0 .../services}/startMaintenanceWindow.js | 0 .../services}/stopMaintenanceWindow.js | 0 .../views}/MaintenanceViewer.js | 2 +- frontend/src/modules/index.js | 1 + frontend/src/utils/helpers/moduleUtils.js | 1 + .../maintenance/test_mainenance_gql.py | 26 ++++++++++++--- 18 files changed, 77 insertions(+), 49 deletions(-) create mode 100644 frontend/src/modules/Maintenance/index.js rename frontend/src/{services/graphql/MaintenanceWindow => modules/Maintenance/services}/getMaintenanceStatus.js (100%) rename frontend/src/{services/graphql/MaintenanceWindow => modules/Maintenance/services}/index.js (100%) rename frontend/src/{services/graphql/MaintenanceWindow => modules/Maintenance/services}/startMaintenanceWindow.js (100%) rename frontend/src/{services/graphql/MaintenanceWindow => modules/Maintenance/services}/stopMaintenanceWindow.js (100%) rename frontend/src/modules/{Administration/components => Maintenance/views}/MaintenanceViewer.js (99%) diff --git a/backend/dataall/base/utils/api_handler_utils.py b/backend/dataall/base/utils/api_handler_utils.py index 88e82f206..9b0e0bda5 100644 --- a/backend/dataall/base/utils/api_handler_utils.py +++ b/backend/dataall/base/utils/api_handler_utils.py @@ -11,8 +11,9 @@ def get_cognito_groups(claims): ) groups = list() saml_groups = claims.get('custom:saml.groups', '') + translation_table = str.maketrans({'[': None, ']': None, ', ': ','}) if len(saml_groups): - groups: list = saml_groups.replace('[', '').replace(']', '').replace(', ', ',').split(',') + groups = saml_groups.translate(translation_table).split(',') cognito_groups = claims.get('cognito:groups', '') if len(cognito_groups): groups.extend(cognito_groups.split(',')) diff --git a/backend/dataall/modules/maintenance/api/resolvers.py b/backend/dataall/modules/maintenance/api/resolvers.py index 1ad3127ec..606dec6c5 100644 --- a/backend/dataall/modules/maintenance/api/resolvers.py +++ b/backend/dataall/modules/maintenance/api/resolvers.py @@ -9,17 +9,11 @@ def start_maintenance_window(context: Context, source: Maintenance, mode: str): """Starts the maintenance window""" if mode not in [item.value for item in list(MaintenanceModes)]: raise Exception('Mode is not conforming to the MaintenanceModes enum') - # Check from the context if the groups contains the DAAAdminstrators group - if not TenantPolicyValidationService.is_tenant_admin(context.groups): - raise Exception('Only data.all admin group members can start maintenance window') - return MaintenanceService.start_maintenance_window(engine=context.engine, mode=mode) + return MaintenanceService.start_maintenance_window(engine=context.engine, mode=mode, groups=context.groups) def stop_maintenance_window(context: Context, source: Maintenance): - # Check from the context if the groups contains the DAAAdminstrators group - if not TenantPolicyValidationService.is_tenant_admin(context.groups): - raise Exception('Only data.all admin group members can stop maintenance window') - return MaintenanceService.stop_maintenance_window(engine=context.engine) + return MaintenanceService.stop_maintenance_window(engine=context.engine, groups=None) def get_maintenance_window_status(context: Context, source: Maintenance): diff --git a/backend/dataall/modules/maintenance/api/types.py b/backend/dataall/modules/maintenance/api/types.py index f464f3b50..52ffaaf1c 100644 --- a/backend/dataall/modules/maintenance/api/types.py +++ b/backend/dataall/modules/maintenance/api/types.py @@ -4,5 +4,5 @@ Maintenance = gql.ObjectType( - name='Maintenance', fields=[gql.Field(name='status', type=gql.String), gql.Field(name='mode', type=gql.String)] + name='Maintenance', fields=[gql.NonNullableType(name='status', type=gql.String), gql.Field(name='mode', type=gql.String)] ) diff --git a/backend/dataall/modules/maintenance/db/maintenance_repository.py b/backend/dataall/modules/maintenance/db/maintenance_repository.py index f45614205..1a47e4a36 100644 --- a/backend/dataall/modules/maintenance/db/maintenance_repository.py +++ b/backend/dataall/modules/maintenance/db/maintenance_repository.py @@ -21,7 +21,3 @@ def save_maintenance_status_and_mode(self, maintenance_status: str, maintenance_ def get_maintenance_record(self): return self._session.query(Maintenance).one() - - def get_maintenance_mode(self): - maintenance_record = self._session.query(Maintenance) - return maintenance_record.mode diff --git a/backend/dataall/modules/maintenance/services/maintenance_service.py b/backend/dataall/modules/maintenance/services/maintenance_service.py index 70e5a136a..0037bb9ff 100644 --- a/backend/dataall/modules/maintenance/services/maintenance_service.py +++ b/backend/dataall/modules/maintenance/services/maintenance_service.py @@ -8,6 +8,7 @@ from dataall.base.aws.event_bridge import EventBridge from dataall.base.aws.parameter_store import ParameterStoreManager +from dataall.core.permissions.services.tenant_policy_service import TenantPolicyValidationService from dataall.modules.maintenance.api.enums import MaintenanceStatus from dataall.modules.maintenance.db.maintenance_repository import MaintenanceRepository from dataall.core.stacks.aws.ecs import Ecs @@ -19,7 +20,13 @@ class MaintenanceService: # Update the RDS table with the mode and status to PENDING # Disable all scheduled ECS tasks which are created by data.all @staticmethod - def start_maintenance_window(engine, mode: str = None): + def start_maintenance_window(engine, mode: str = None, groups=None): + # Check from the context if the groups contains the DAAAdminstrators group + if groups is None: + groups = [] + if not TenantPolicyValidationService.is_tenant_admin(groups): + raise Exception('Only data.all admin group members can start maintenance window') + logger.info('Putting data.all into maintenance') try: with engine.scoped_session() as session: @@ -54,7 +61,12 @@ def start_maintenance_window(engine, mode: str = None): # Update the RDS table by changing the status to INACTIVE # Enable all the ECS Scheduled task @staticmethod - def stop_maintenance_window(engine): + def stop_maintenance_window(engine, groups=None): + # Check from the context if the groups contains the DAAAdminstrators group + if groups is None: + groups = [] + if not TenantPolicyValidationService.is_tenant_admin(groups): + raise Exception('Only data.all admin group members can stop maintenance window') logger.info('Stopping maintenance mode') try: with engine.scoped_session() as session: diff --git a/frontend/src/authentication/components/AuthGuard.js b/frontend/src/authentication/components/AuthGuard.js index e7ef31b47..63d558308 100644 --- a/frontend/src/authentication/components/AuthGuard.js +++ b/frontend/src/authentication/components/AuthGuard.js @@ -4,16 +4,14 @@ import { Navigate, useLocation } from 'react-router-dom'; import { Login } from '../views/Login'; import { useAuth } from '../hooks'; import { + isModuleEnabled, ModuleNames, RegexToValidateWindowPathName, WindowPathLengthThreshold } from '../../utils'; import { useClient, useGroups } from '../../services'; import { LoadingScreen, NoAccessMaintenanceWindow } from '../../design'; -import { getMaintenanceStatus } from '../../services/graphql/MaintenanceWindow'; -import { - ACTIVE_STATUS, - PENDING_STATUS -} from '../../modules/Administration/components'; +import { getMaintenanceStatus } from '../../modules/Maintenance/services'; +import {PENDING_STATUS, ACTIVE_STATUS} from "../../modules/Maintenance/views/MaintenanceViewer"; import config from '../../generated/config.json'; import { SET_ERROR, useDispatch } from '../../globalErrors'; @@ -47,7 +45,7 @@ export const AuthGuard = (props) => { useEffect(async () => { // Check if the maintenance window is enabled and has NO-ACCESS Status // If yes then display a blank screen with a message that data.all is in maintenance mode ( Check use of isNoAccessMaintenance state ) - if (config.modules.maintenance.active === true) { + if (isModuleEnabled(ModuleNames.MAINTENANCE) === true) { if (client) { checkMaintenanceMode().catch((e) => dispatch({ type: SET_ERROR, e })); } @@ -73,7 +71,7 @@ export const AuthGuard = (props) => { if ( isNoAccessMaintenance == null && - config.modules.maintenance.active === true + isModuleEnabled(ModuleNames.MAINTENANCE) === true ) { return ; } diff --git a/frontend/src/design/components/layout/DefaultNavbar.js b/frontend/src/design/components/layout/DefaultNavbar.js index 158c6edb4..d2202dfd6 100644 --- a/frontend/src/design/components/layout/DefaultNavbar.js +++ b/frontend/src/design/components/layout/DefaultNavbar.js @@ -8,12 +8,9 @@ import { Logo } from '../Logo'; import { SettingsDrawer } from '../SettingsDrawer'; import { ModuleNames, isModuleEnabled } from 'utils'; import config from '../../../generated/config.json'; -import { - ACTIVE_STATUS, - PENDING_STATUS -} from '../../../modules/Administration/components'; +import {PENDING_STATUS, ACTIVE_STATUS} from "../../../modules/Maintenance/views/MaintenanceViewer"; import { useClient } from '../../../services'; -import { getMaintenanceStatus } from '../../../services/graphql/MaintenanceWindow'; +import { getMaintenanceStatus } from '../../../modules/Maintenance/services'; import { SET_ERROR, useDispatch } from '../../../globalErrors'; import { SanitizedHTML } from '../SanitizedHTML'; @@ -30,31 +27,37 @@ export const DefaultNavbar = ({ openDrawer, onOpenDrawerChange }) => { const dispatch = useDispatch(); const client = useClient(); - useEffect(async () => { - if (client) { + const _getMaintenanceStatus = async() =>{ const response = await client.query(getMaintenanceStatus()); if ( - !response.errors && - response.data.getMaintenanceWindowStatus !== null + !response.errors && + response.data.getMaintenanceWindowStatus !== null ) { if ( - response.data.getMaintenanceWindowStatus.status === ACTIVE_STATUS || - response.data.getMaintenanceWindowStatus.status === PENDING_STATUS + response.data.getMaintenanceWindowStatus.status === ACTIVE_STATUS || + response.data.getMaintenanceWindowStatus.status === PENDING_STATUS ) { setMaintenanceFlag(true); } } else { const error = response.errors - ? response.errors[0].message - : 'Could not fetch status of maintenance window'; - dispatch({ type: SET_ERROR, error }); + ? response.errors[0].message + : 'Could not fetch status of maintenance window'; + dispatch({type: SET_ERROR, error}); } } + + + useEffect(async () => { + console.log("Loading the useffect of deafult nav bar ") + if (client && isModuleEnabled(ModuleNames.MAINTENANCE)) { + _getMaintenanceStatus().catch((err) => dispatch({ type: SET_ERROR, err })); + } }, [client]); return ( - {config.modules.maintenance.active && isMaintenance ? ( + {isModuleEnabled(ModuleNames.MAINTENANCE) && isMaintenance ? ( {config.modules.maintenance.custom_maintenance_text !== undefined ? ( diff --git a/frontend/src/modules/Administration/components/index.js b/frontend/src/modules/Administration/components/index.js index 99765e234..abdf3aa25 100644 --- a/frontend/src/modules/Administration/components/index.js +++ b/frontend/src/modules/Administration/components/index.js @@ -1,4 +1,3 @@ export * from './AdministrationTeams'; export * from './AdministratorDashboardViewer'; -export * from './TeamPermissionsEditForm'; -export * from './MaintenanceViewer'; +export * from './TeamPermissionsEditForm'; \ No newline at end of file diff --git a/frontend/src/modules/Administration/views/AdministrationView.js b/frontend/src/modules/Administration/views/AdministrationView.js index 16128c926..615702a6d 100644 --- a/frontend/src/modules/Administration/views/AdministrationView.js +++ b/frontend/src/modules/Administration/views/AdministrationView.js @@ -13,19 +13,19 @@ import { useState } from 'react'; import { Helmet } from 'react-helmet-async'; import { Link as RouterLink } from 'react-router-dom'; import { ChevronRightIcon, useSettings } from 'design'; -import config from '../../../generated/config.json'; import { AdministrationTeams, DashboardViewer, - MaintenanceViewer } from '../components'; +import {MaintenanceViewer} from "../../Maintenance/views/MaintenanceViewer"; +import {isModuleEnabled, ModuleNames} from "../../../utils"; const tabs = [ { label: 'Teams', value: 'teams' }, { label: 'Monitoring', value: 'dashboard' } ]; -// Using 'config' as the isModulesEnabled needs Modules.MAINTENANCE which involves much bigger setup which is not needed for maintenance module -if (config.modules.maintenance.active) { + +if (isModuleEnabled(ModuleNames.MAINTENANCE)) { tabs.push({ label: 'Maintenance', value: 'maintenance' }); } diff --git a/frontend/src/modules/Maintenance/index.js b/frontend/src/modules/Maintenance/index.js new file mode 100644 index 000000000..00e6f6477 --- /dev/null +++ b/frontend/src/modules/Maintenance/index.js @@ -0,0 +1,5 @@ +export const MaintenanceModule = { + moduleDefinition: true, + name: 'maintenance', + isEnvironmentModule: false +}; \ No newline at end of file diff --git a/frontend/src/services/graphql/MaintenanceWindow/getMaintenanceStatus.js b/frontend/src/modules/Maintenance/services/getMaintenanceStatus.js similarity index 100% rename from frontend/src/services/graphql/MaintenanceWindow/getMaintenanceStatus.js rename to frontend/src/modules/Maintenance/services/getMaintenanceStatus.js diff --git a/frontend/src/services/graphql/MaintenanceWindow/index.js b/frontend/src/modules/Maintenance/services/index.js similarity index 100% rename from frontend/src/services/graphql/MaintenanceWindow/index.js rename to frontend/src/modules/Maintenance/services/index.js diff --git a/frontend/src/services/graphql/MaintenanceWindow/startMaintenanceWindow.js b/frontend/src/modules/Maintenance/services/startMaintenanceWindow.js similarity index 100% rename from frontend/src/services/graphql/MaintenanceWindow/startMaintenanceWindow.js rename to frontend/src/modules/Maintenance/services/startMaintenanceWindow.js diff --git a/frontend/src/services/graphql/MaintenanceWindow/stopMaintenanceWindow.js b/frontend/src/modules/Maintenance/services/stopMaintenanceWindow.js similarity index 100% rename from frontend/src/services/graphql/MaintenanceWindow/stopMaintenanceWindow.js rename to frontend/src/modules/Maintenance/services/stopMaintenanceWindow.js diff --git a/frontend/src/modules/Administration/components/MaintenanceViewer.js b/frontend/src/modules/Maintenance/views/MaintenanceViewer.js similarity index 99% rename from frontend/src/modules/Administration/components/MaintenanceViewer.js rename to frontend/src/modules/Maintenance/views/MaintenanceViewer.js index c49f7f37a..76f43d175 100644 --- a/frontend/src/modules/Administration/components/MaintenanceViewer.js +++ b/frontend/src/modules/Maintenance/views/MaintenanceViewer.js @@ -20,7 +20,7 @@ import { getMaintenanceStatus, stopMaintenanceWindow, startMaintenanceWindow -} from '../../../services/graphql/MaintenanceWindow'; +} from '../services'; import { useClient } from '../../../services'; import { SET_ERROR, useDispatch } from '../../../globalErrors'; import { useSnackbar } from 'notistack'; diff --git a/frontend/src/modules/index.js b/frontend/src/modules/index.js index 873e5177d..6d71b50c6 100644 --- a/frontend/src/modules/index.js +++ b/frontend/src/modules/index.js @@ -8,3 +8,4 @@ export * from './Notifications'; export * from './Pipelines'; export * from './Shares'; export * from './Worksheets'; +export * from './Maintenance' diff --git a/frontend/src/utils/helpers/moduleUtils.js b/frontend/src/utils/helpers/moduleUtils.js index 648d675c0..e6238e8c8 100644 --- a/frontend/src/utils/helpers/moduleUtils.js +++ b/frontend/src/utils/helpers/moduleUtils.js @@ -63,6 +63,7 @@ function _modulesNameMap() { const upperCaseModule = module.name.toUpperCase(); map[upperCaseModule] = module.name; } + console.log(map) return map; } diff --git a/tests/modules/maintenance/test_mainenance_gql.py b/tests/modules/maintenance/test_mainenance_gql.py index f60461840..9fbf3a996 100644 --- a/tests/modules/maintenance/test_mainenance_gql.py +++ b/tests/modules/maintenance/test_mainenance_gql.py @@ -33,7 +33,6 @@ def init_maintenance_record(db): session.commit() -@pytest.mark.skipif(not config.get_property('modules.maintenance.active'), reason='Module disabled by config') def test_start_maintenance_window(db, client, mock_ecs_client, init_maintenance_record): response = client.query( """ @@ -55,7 +54,6 @@ def test_start_maintenance_window(db, client, mock_ecs_client, init_maintenance_ assert maintenance_record.mode == 'READ-ONLY' -@pytest.mark.skipif(not config.get_property('modules.maintenance.active'), reason='Module disabled by config') def test_start_maintenance_window_with_team_not_a_data_admin(client, mock_ecs_client, init_maintenance_record): response = client.query( """ @@ -72,7 +70,6 @@ def test_start_maintenance_window_with_team_not_a_data_admin(client, mock_ecs_cl assert 'Only data.all admin group members can start maintenance window' in response.errors[0]['message'] -@pytest.mark.skipif(not config.get_property('modules.maintenance.active'), reason='Module disabled by config') def test_stop_maintenance_window(db, client, mock_ecs_client, init_maintenance_record): # Initialize the maintenance window with ACTIVE status and READ-ONLY mode with db.scoped_session() as session: @@ -95,8 +92,29 @@ def test_stop_maintenance_window(db, client, mock_ecs_client, init_maintenance_r assert response assert response.data.stopMaintenanceWindow is True +def test_stop_maintenance_window_no_dataall_admin(db, client, mock_ecs_client, init_maintenance_record): + # Initialize the maintenance window with ACTIVE status and READ-ONLY mode + with db.scoped_session() as session: + maintenance_record = session.query(Maintenance).one() + maintenance_record.mode = 'READ-ONLY' + maintenance_record.status = 'ACTIVE' + session.add(maintenance_record) + session.commit() + + response = client.query( + """ + mutation stopMaintenanceWindow{ + stopMaintenanceWindow + } + """, + username='alice', + groups=['Engineers'], + ) + + assert response + assert 'Only data.all admin group members can stop maintenance window' in response.errors[0]['message'] + -@pytest.mark.skipif(not config.get_property('modules.maintenance.active'), reason='Module disabled by config') def test_get_maintenance_window_status(db, client, mock_ecs_client, init_maintenance_record): # Initialize the maintenance window with ACTIVE status and READ-ONLY mode with db.scoped_session() as session: From 2ae74ce352eda79c15e48b9418f10871f39254e1 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Wed, 8 May 2024 15:29:02 -0500 Subject: [PATCH 24/36] Code changes after review comments --- backend/api_handler.py | 169 ++++++++++-------- .../modules/maintenance/api/resolvers.py | 2 +- .../dataall/modules/maintenance/api/types.py | 3 +- .../services/maintenance_service.py | 39 +++- .../authentication/components/AuthGuard.js | 9 +- .../design/components/layout/DefaultNavbar.js | 38 ++-- .../Administration/components/index.js | 2 +- .../views/AdministrationView.js | 9 +- frontend/src/modules/Maintenance/index.js | 2 +- frontend/src/modules/index.js | 2 +- frontend/src/utils/helpers/moduleUtils.js | 1 - .../maintenance/test_mainenance_gql.py | 1 + 12 files changed, 158 insertions(+), 119 deletions(-) diff --git a/backend/api_handler.py b/backend/api_handler.py index a2d07b953..d16e9e6ea 100644 --- a/backend/api_handler.py +++ b/backend/api_handler.py @@ -116,31 +116,11 @@ def handler(event, context): if 'user_id' in event['requestContext']['authorizer']: user_id = event['requestContext']['authorizer']['user_id'] log.debug('username is %s', username) - try: - groups = [] - if os.environ.get('custom_auth', None): - groups.extend(get_custom_groups(user_id)) - else: - groups.extend(get_cognito_groups(claims)) - log.debug('groups are %s', ','.join(groups)) - with ENGINE.scoped_session() as session: - for group in groups: - policy = TenantPolicyService.find_tenant_policy(session, group, TenantPolicyService.TENANT_NAME) - if not policy: - print(f'No policy found for Team {group}. Attaching TENANT_ALL permissions') - TenantPolicyService.attach_group_tenant_policy( - session=session, - group=group, - permissions=TENANT_ALL, - tenant_name=TenantPolicyService.TENANT_NAME, - ) - except Exception as e: - print(f'Error managing groups due to: {e}') - groups = [] + groups: list = extract_groups_and_attach_tenant_policy(user_id=user_id, claims=claims) + # Set Context set_context(RequestContext(ENGINE, username, groups, user_id)) - app_context = { 'engine': ENGINE, 'username': username, @@ -150,55 +130,50 @@ def handler(event, context): query = json.loads(event.get('body')) - # Logic to block when in maintenance - # Check if in some maintenance mode - # Check if in maintenance status is not INACTIVE - # Check if the user belongs to a 'DAAdministrators' group - if config.get_property('modules.maintenance.active'): - if ( - (MaintenanceService._get_maintenance_window_mode(engine=ENGINE) == MaintenanceModes.NOACCESS.value) - and ( - MaintenanceService.get_maintenance_window_status(engine=ENGINE).status - is not MaintenanceStatus.INACTIVE.value - ) - and not TenantPolicyValidationService.is_tenant_admin(groups) - ): - if query.get('operationName', '') not in MAINTENANCE_ALLOWED_OPERATIONS: - return send_unauthorized_response( - query=query, - message='Access Restricted: data.all is currently undergoing maintenance, and your actions are temporarily blocked.', - ) - elif ( - (MaintenanceService._get_maintenance_window_mode(engine=ENGINE) == MaintenanceModes.READONLY.value) - and ( - MaintenanceService.get_maintenance_window_status(engine=ENGINE).status - is not MaintenanceStatus.INACTIVE.value - ) - and not TenantPolicyValidationService.is_tenant_admin(groups) - ): - # If its mutation then block and return - if query.get('query', '').split()[0] == 'mutation': - return send_unauthorized_response( - query=query, - message='Access Restricted: data.all is currently undergoing maintenance, and your actions are temporarily blocked.', - ) + isBlockedResponse = validate_and_block_if_maintenance_window(query=query, groups=groups) + if isBlockedResponse is not None: + return isBlockedResponse + isReauthResponse = check_reauth(query=query, auth_time=claims['auth_time'], username=username) + if isReauthResponse is not None: + return isReauthResponse - # Determine if there are any Operations that Require ReAuth From SSM Parameter - try: - reauth_apis = ParameterStoreManager.get_parameter_value( - region=os.getenv('AWS_REGION', 'eu-west-1'), parameter_path=f'/dataall/{ENVNAME}/reauth/apis' - ).split(',') - except Exception: - log.info('No ReAuth APIs Found in SSM') - reauth_apis = None else: raise Exception(f'Could not initialize user context from event {event}') + success, response = graphql_sync(schema=executable_schema, data=query, context_value=app_context) + + dispose_context() + response = json.dumps(response) + + log.info('Lambda Response %s', response) + + return { + 'statusCode': 200 if success else 400, + 'headers': { + 'content-type': 'application/json', + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Headers': '*', + 'Access-Control-Allow-Methods': '*', + }, + 'body': response, + } + + +def check_reauth(query, auth_time, username): + # Determine if there are any Operations that Require ReAuth From SSM Parameter + try: + reauth_apis = ParameterStoreManager.get_parameter_value( + region=os.getenv('AWS_REGION', 'eu-west-1'), parameter_path=f'/dataall/{ENVNAME}/reauth/apis' + ).split(',') + except Exception: + log.info('No ReAuth APIs Found in SSM') + reauth_apis = None + # If The Operation is a ReAuth Operation - Ensure A Non-Expired Session or Return Error if reauth_apis and query.get('operationName', None) in reauth_apis: now = datetime.datetime.now(datetime.timezone.utc) try: - auth_time_datetime = datetime.datetime.fromtimestamp(int(claims['auth_time']), tz=datetime.timezone.utc) + auth_time_datetime = datetime.datetime.fromtimestamp(int(auth_time), tz=datetime.timezone.utc) if auth_time_datetime + datetime.timedelta(minutes=REAUTH_TTL) < now: raise Exception('ReAuth') except Exception as e: @@ -225,20 +200,60 @@ def handler(event, context): 'body': json.dumps(response), } - success, response = graphql_sync(schema=executable_schema, data=query, context_value=app_context) - dispose_context() - response = json.dumps(response) +def validate_and_block_if_maintenance_window(query, groups): + # Logic to block when in maintenance + # Check if in some maintenance mode + # Check if in maintenance status is not INACTIVE + # Check if the user belongs to a 'DAAdministrators' group + if config.get_property('modules.maintenance.active'): + maintenance_mode = MaintenanceService._get_maintenance_window_mode(engine=ENGINE) + maintenance_status = MaintenanceService.get_maintenance_window_status(engine=ENGINE).status + isAdmin = TenantPolicyValidationService.is_tenant_admin(groups) + + if ( + (maintenance_mode == MaintenanceModes.NOACCESS.value) + and (maintenance_status is not MaintenanceStatus.INACTIVE.value) + and not isAdmin + ): + if query.get('operationName', '') not in MAINTENANCE_ALLOWED_OPERATIONS: + return send_unauthorized_response( + query=query, + message='Access Restricted: data.all is currently undergoing maintenance, and your actions are temporarily blocked.', + ) + elif ( + (maintenance_mode == MaintenanceModes.READONLY.value) + and (maintenance_status is not MaintenanceStatus.INACTIVE.value) + and not isAdmin + ): + # If its mutation then block and return + if query.get('query', '').split()[0] == 'mutation': + return send_unauthorized_response( + query=query, + message='Access Restricted: data.all is currently undergoing maintenance, and your actions are temporarily blocked.', + ) - log.info('Lambda Response %s', response) - return { - 'statusCode': 200 if success else 400, - 'headers': { - 'content-type': 'application/json', - 'Access-Control-Allow-Origin': '*', - 'Access-Control-Allow-Headers': '*', - 'Access-Control-Allow-Methods': '*', - }, - 'body': response, - } +def extract_groups_and_attach_tenant_policy(user_id, claims): + groups = [] + try: + if os.environ.get('custom_auth', None): + groups.extend(get_custom_groups(user_id)) + else: + groups.extend(get_cognito_groups(claims)) + log.debug('groups are %s', ','.join(groups)) + with ENGINE.scoped_session() as session: + for group in groups: + policy = TenantPolicyService.find_tenant_policy(session, group, TenantPolicyService.TENANT_NAME) + if not policy: + print(f'No policy found for Team {group}. Attaching TENANT_ALL permissions') + TenantPolicyService.attach_group_tenant_policy( + session=session, + group=group, + permissions=TENANT_ALL, + tenant_name=TenantPolicyService.TENANT_NAME, + ) + return groups + except Exception as e: + print(f'Error managing groups due to: {e}') + return groups diff --git a/backend/dataall/modules/maintenance/api/resolvers.py b/backend/dataall/modules/maintenance/api/resolvers.py index 606dec6c5..8955b40b4 100644 --- a/backend/dataall/modules/maintenance/api/resolvers.py +++ b/backend/dataall/modules/maintenance/api/resolvers.py @@ -13,7 +13,7 @@ def start_maintenance_window(context: Context, source: Maintenance, mode: str): def stop_maintenance_window(context: Context, source: Maintenance): - return MaintenanceService.stop_maintenance_window(engine=context.engine, groups=None) + return MaintenanceService.stop_maintenance_window(engine=context.engine, groups=context.groups) def get_maintenance_window_status(context: Context, source: Maintenance): diff --git a/backend/dataall/modules/maintenance/api/types.py b/backend/dataall/modules/maintenance/api/types.py index 52ffaaf1c..422f664fe 100644 --- a/backend/dataall/modules/maintenance/api/types.py +++ b/backend/dataall/modules/maintenance/api/types.py @@ -4,5 +4,6 @@ Maintenance = gql.ObjectType( - name='Maintenance', fields=[gql.NonNullableType(name='status', type=gql.String), gql.Field(name='mode', type=gql.String)] + name='Maintenance', + fields=[gql.Field(name='status', type=gql.NonNullableType(gql.String)), gql.Field(name='mode', type=gql.String)], ) diff --git a/backend/dataall/modules/maintenance/services/maintenance_service.py b/backend/dataall/modules/maintenance/services/maintenance_service.py index 0037bb9ff..69550f28e 100644 --- a/backend/dataall/modules/maintenance/services/maintenance_service.py +++ b/backend/dataall/modules/maintenance/services/maintenance_service.py @@ -17,10 +17,18 @@ class MaintenanceService: - # Update the RDS table with the mode and status to PENDING - # Disable all scheduled ECS tasks which are created by data.all @staticmethod def start_maintenance_window(engine, mode: str = None, groups=None): + """ + Start maintenance window by performing following actions + 1. Perform validation to check if the user belongs to the DAAdministrators group + 2. Put the maintenance window status to PENDING and update the maintenance mode + 3. Get all the ECS Scheduled tasks and disable the schedule for them + @param engine: db engine + @param mode: mode to set for maintenance window + @param groups: user groups from context.groups + @return: returns True if successful or False + """ # Check from the context if the groups contains the DAAAdminstrators group if groups is None: groups = [] @@ -57,11 +65,18 @@ def start_maintenance_window(engine, mode: str = None, groups=None): logger.error(f'Error occurred while starting maintenance window due to {e}') return False - # Update the RDS table by changing mode to - '' - # Update the RDS table by changing the status to INACTIVE - # Enable all the ECS Scheduled task @staticmethod def stop_maintenance_window(engine, groups=None): + """ + Stop maintenance window by performing following actions + 1. Perform validation to check if the user belongs to the DAAdministrators group + 2. Update the RDS table by changing the status to INACTIVE and mode to '-' + 3. Enable all data.all related ECS scheduled tasks + @param engine: db engine + @param groups: user groups from context.groups + @return: return True if successful or False + """ + # Check from the context if the groups contains the DAAAdminstrators group if groups is None: groups = [] @@ -91,10 +106,14 @@ def stop_maintenance_window(engine, groups=None): logger.error(f'Error occurred while stopping maintenance window due to {e}') return False - # Checks if all ECS tasks in the data.all infra account have completed - # Updates the maintenance status and returns maintenance record @staticmethod def get_maintenance_window_status(engine): + """ + Get the status of maintenance window + Maintenance record is returned after checking if all ECS tasks in the data.all created cluster have completed. + @param engine: db object + @return: Maintenance object containing status and mode + """ logger.info('Checking maintenance window status') try: with engine.scoped_session() as session: @@ -106,13 +125,17 @@ def get_maintenance_window_status(engine): parameter_path=f"/dataall/{os.getenv('envname', 'local')}/ecs/cluster/name", ) if Ecs.is_task_running(cluster_name=ecs_cluster_name): + logger.info(f'Current maintenance window status - {maintenance_record.status}') return maintenance_record else: + logger.info( + 'All pending ECS tasks have completed running. Setting Maintenance Status to ACTIVE' + ) maintenance_record.status = MaintenanceStatus.ACTIVE.value session.commit() return maintenance_record else: - logger.info('Maintenance window is not in PENDING state') + logger.info(f'Current maintenance window status - {maintenance_record.status}') return maintenance_record except Exception as e: logger.error(f'Error while getting maintenance window status due to {e}') diff --git a/frontend/src/authentication/components/AuthGuard.js b/frontend/src/authentication/components/AuthGuard.js index 63d558308..98f995061 100644 --- a/frontend/src/authentication/components/AuthGuard.js +++ b/frontend/src/authentication/components/AuthGuard.js @@ -4,15 +4,18 @@ import { Navigate, useLocation } from 'react-router-dom'; import { Login } from '../views/Login'; import { useAuth } from '../hooks'; import { - isModuleEnabled, ModuleNames, + isModuleEnabled, + ModuleNames, RegexToValidateWindowPathName, WindowPathLengthThreshold } from '../../utils'; import { useClient, useGroups } from '../../services'; import { LoadingScreen, NoAccessMaintenanceWindow } from '../../design'; import { getMaintenanceStatus } from '../../modules/Maintenance/services'; -import {PENDING_STATUS, ACTIVE_STATUS} from "../../modules/Maintenance/views/MaintenanceViewer"; -import config from '../../generated/config.json'; +import { + PENDING_STATUS, + ACTIVE_STATUS +} from '../../modules/Maintenance/views/MaintenanceViewer'; import { SET_ERROR, useDispatch } from '../../globalErrors'; export const AuthGuard = (props) => { diff --git a/frontend/src/design/components/layout/DefaultNavbar.js b/frontend/src/design/components/layout/DefaultNavbar.js index d2202dfd6..1ba7da090 100644 --- a/frontend/src/design/components/layout/DefaultNavbar.js +++ b/frontend/src/design/components/layout/DefaultNavbar.js @@ -8,7 +8,10 @@ import { Logo } from '../Logo'; import { SettingsDrawer } from '../SettingsDrawer'; import { ModuleNames, isModuleEnabled } from 'utils'; import config from '../../../generated/config.json'; -import {PENDING_STATUS, ACTIVE_STATUS} from "../../../modules/Maintenance/views/MaintenanceViewer"; +import { + PENDING_STATUS, + ACTIVE_STATUS +} from '../../../modules/Maintenance/views/MaintenanceViewer'; import { useClient } from '../../../services'; import { getMaintenanceStatus } from '../../../modules/Maintenance/services'; import { SET_ERROR, useDispatch } from '../../../globalErrors'; @@ -27,31 +30,28 @@ export const DefaultNavbar = ({ openDrawer, onOpenDrawerChange }) => { const dispatch = useDispatch(); const client = useClient(); - const _getMaintenanceStatus = async() =>{ - const response = await client.query(getMaintenanceStatus()); + const _getMaintenanceStatus = async () => { + const response = await client.query(getMaintenanceStatus()); + if (!response.errors && response.data.getMaintenanceWindowStatus !== null) { if ( - !response.errors && - response.data.getMaintenanceWindowStatus !== null + response.data.getMaintenanceWindowStatus.status === ACTIVE_STATUS || + response.data.getMaintenanceWindowStatus.status === PENDING_STATUS ) { - if ( - response.data.getMaintenanceWindowStatus.status === ACTIVE_STATUS || - response.data.getMaintenanceWindowStatus.status === PENDING_STATUS - ) { - setMaintenanceFlag(true); - } - } else { - const error = response.errors - ? response.errors[0].message - : 'Could not fetch status of maintenance window'; - dispatch({type: SET_ERROR, error}); + setMaintenanceFlag(true); } + } else { + const error = response.errors + ? response.errors[0].message + : 'Could not fetch status of maintenance window'; + dispatch({ type: SET_ERROR, error }); } - + }; useEffect(async () => { - console.log("Loading the useffect of deafult nav bar ") if (client && isModuleEnabled(ModuleNames.MAINTENANCE)) { - _getMaintenanceStatus().catch((err) => dispatch({ type: SET_ERROR, err })); + _getMaintenanceStatus().catch((err) => + dispatch({ type: SET_ERROR, err }) + ); } }, [client]); diff --git a/frontend/src/modules/Administration/components/index.js b/frontend/src/modules/Administration/components/index.js index abdf3aa25..06ef79939 100644 --- a/frontend/src/modules/Administration/components/index.js +++ b/frontend/src/modules/Administration/components/index.js @@ -1,3 +1,3 @@ export * from './AdministrationTeams'; export * from './AdministratorDashboardViewer'; -export * from './TeamPermissionsEditForm'; \ No newline at end of file +export * from './TeamPermissionsEditForm'; diff --git a/frontend/src/modules/Administration/views/AdministrationView.js b/frontend/src/modules/Administration/views/AdministrationView.js index 615702a6d..a70183cb7 100644 --- a/frontend/src/modules/Administration/views/AdministrationView.js +++ b/frontend/src/modules/Administration/views/AdministrationView.js @@ -13,12 +13,9 @@ import { useState } from 'react'; import { Helmet } from 'react-helmet-async'; import { Link as RouterLink } from 'react-router-dom'; import { ChevronRightIcon, useSettings } from 'design'; -import { - AdministrationTeams, - DashboardViewer, -} from '../components'; -import {MaintenanceViewer} from "../../Maintenance/views/MaintenanceViewer"; -import {isModuleEnabled, ModuleNames} from "../../../utils"; +import { AdministrationTeams, DashboardViewer } from '../components'; +import { MaintenanceViewer } from '../../Maintenance/views/MaintenanceViewer'; +import { isModuleEnabled, ModuleNames } from '../../../utils'; const tabs = [ { label: 'Teams', value: 'teams' }, diff --git a/frontend/src/modules/Maintenance/index.js b/frontend/src/modules/Maintenance/index.js index 00e6f6477..58db6fd95 100644 --- a/frontend/src/modules/Maintenance/index.js +++ b/frontend/src/modules/Maintenance/index.js @@ -2,4 +2,4 @@ export const MaintenanceModule = { moduleDefinition: true, name: 'maintenance', isEnvironmentModule: false -}; \ No newline at end of file +}; diff --git a/frontend/src/modules/index.js b/frontend/src/modules/index.js index 6d71b50c6..67fca62a4 100644 --- a/frontend/src/modules/index.js +++ b/frontend/src/modules/index.js @@ -8,4 +8,4 @@ export * from './Notifications'; export * from './Pipelines'; export * from './Shares'; export * from './Worksheets'; -export * from './Maintenance' +export * from './Maintenance'; diff --git a/frontend/src/utils/helpers/moduleUtils.js b/frontend/src/utils/helpers/moduleUtils.js index e6238e8c8..648d675c0 100644 --- a/frontend/src/utils/helpers/moduleUtils.js +++ b/frontend/src/utils/helpers/moduleUtils.js @@ -63,7 +63,6 @@ function _modulesNameMap() { const upperCaseModule = module.name.toUpperCase(); map[upperCaseModule] = module.name; } - console.log(map) return map; } diff --git a/tests/modules/maintenance/test_mainenance_gql.py b/tests/modules/maintenance/test_mainenance_gql.py index 9fbf3a996..7f475b2c2 100644 --- a/tests/modules/maintenance/test_mainenance_gql.py +++ b/tests/modules/maintenance/test_mainenance_gql.py @@ -92,6 +92,7 @@ def test_stop_maintenance_window(db, client, mock_ecs_client, init_maintenance_r assert response assert response.data.stopMaintenanceWindow is True + def test_stop_maintenance_window_no_dataall_admin(db, client, mock_ecs_client, init_maintenance_record): # Initialize the maintenance window with ACTIVE status and READ-ONLY mode with db.scoped_session() as session: From f3eab6f011df63a50f5ce7058740845dc33f2d94 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Wed, 8 May 2024 16:22:15 -0500 Subject: [PATCH 25/36] Resolving review comments - reusing code --- .../services/maintenance_service.py | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/backend/dataall/modules/maintenance/services/maintenance_service.py b/backend/dataall/modules/maintenance/services/maintenance_service.py index 69550f28e..4d02b7d4d 100644 --- a/backend/dataall/modules/maintenance/services/maintenance_service.py +++ b/backend/dataall/modules/maintenance/services/maintenance_service.py @@ -52,12 +52,7 @@ def start_maintenance_window(engine, mode: str = None, groups=None): ) # Disable scheduled ECS tasks # Get all the SSM Params related to the scheduled tasks - ecs_scheduled_rules = ParameterStoreManager.get_parameters_by_path( - region=os.getenv('AWS_REGION', 'eu-west-1'), - parameter_path=f"/dataall/{os.getenv('envname', 'local')}/ecs/ecs_scheduled_tasks/rule", - ) - logger.debug(ecs_scheduled_rules) - ecs_scheduled_rules_list = [item['Value'] for item in ecs_scheduled_rules] + ecs_scheduled_rules_list = MaintenanceService._get_ecs_rules() event_bridge_session = EventBridge(region=os.getenv('AWS_REGION', 'eu-west-1')) event_bridge_session.disable_scheduled_ecs_tasks(ecs_scheduled_rules_list) return True @@ -93,12 +88,7 @@ def stop_maintenance_window(engine, groups=None): maintenance_status='INACTIVE', maintenance_mode='' ) # Enable scheduled ECS tasks - ecs_scheduled_rules = ParameterStoreManager.get_parameters_by_path( - region=os.getenv('AWS_REGION', 'eu-west-1'), - parameter_path=f"/dataall/{os.getenv('envname', 'local')}/ecs/ecs_scheduled_tasks/rule", - ) - logger.debug(ecs_scheduled_rules) - ecs_scheduled_rules_list = [item['Value'] for item in ecs_scheduled_rules] + ecs_scheduled_rules_list = MaintenanceService._get_ecs_rules() event_bridge_session = EventBridge(region=os.getenv('AWS_REGION', 'eu-west-1')) event_bridge_session.enable_scheduled_ecs_tasks(ecs_scheduled_rules_list) return True @@ -152,3 +142,12 @@ def _get_maintenance_window_mode(engine): except Exception as e: logger.error(f'Error while getting maintenance window mode due to {e}') raise e + + @staticmethod + def _get_ecs_rules(): + ecs_scheduled_rules = ParameterStoreManager.get_parameters_by_path( + region=os.getenv('AWS_REGION', 'eu-west-1'), + parameter_path=f"/dataall/{os.getenv('envname', 'local')}/ecs/ecs_scheduled_tasks/rule", + ) + logger.debug(ecs_scheduled_rules) + return [item['Value'] for item in ecs_scheduled_rules] From 02abe68c9ba386d9d032d58a3e783c6e144fa550 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Wed, 8 May 2024 16:29:42 -0500 Subject: [PATCH 26/36] Resolving issue with alembic --- .../versions/b833ad41db68_maintenance_window_schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/migrations/versions/b833ad41db68_maintenance_window_schema.py b/backend/migrations/versions/b833ad41db68_maintenance_window_schema.py index c1c3e352d..231834e57 100644 --- a/backend/migrations/versions/b833ad41db68_maintenance_window_schema.py +++ b/backend/migrations/versions/b833ad41db68_maintenance_window_schema.py @@ -16,7 +16,7 @@ # revision identifiers, used by Alembic. revision = 'b833ad41db68' -down_revision = 'c6d01930179d' +down_revision = '458572580709' branch_labels = None depends_on = None From ea896f8761a7d8cb16abedca5a9c5ca1d420934e Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Mon, 13 May 2024 14:08:46 -0500 Subject: [PATCH 27/36] Addressing comments after 2nd review --- backend/api_handler.py | 130 ++---------------- .../dataall/base/utils/api_handler_utils.py | 113 ++++++++++++++- .../modules/maintenance/api/resolvers.py | 6 +- .../services/maintenance_service.py | 33 ++--- backend/search_handler.py | 33 +---- .../authentication/components/AuthGuard.js | 10 +- .../design/components/layout/DefaultNavbar.js | 6 +- .../views/AdministrationView.js | 4 +- .../MaintenanceViewer.js | 6 +- 9 files changed, 158 insertions(+), 183 deletions(-) rename frontend/src/modules/Maintenance/{views => components}/MaintenanceViewer.js (98%) diff --git a/backend/api_handler.py b/backend/api_handler.py index d16e9e6ea..e3847f21c 100644 --- a/backend/api_handler.py +++ b/backend/api_handler.py @@ -1,7 +1,6 @@ import json import logging import os -import datetime from argparse import Namespace from time import perf_counter @@ -11,18 +10,14 @@ ) from dataall.base.api import bootstrap as bootstrap_schema, get_executable_schema -from dataall.base.utils.api_handler_utils import get_custom_groups, get_cognito_groups, send_unauthorized_response +from dataall.base.utils.api_handler_utils import extract_groups, attach_tenant_policy_for_groups, check_reauth, \ + validate_and_block_if_maintenance_window from dataall.core.tasks.service_handlers import Worker from dataall.base.aws.sqs import SqsQueue -from dataall.base.aws.parameter_store import ParameterStoreManager from dataall.base.context import set_context, dispose_context, RequestContext -from dataall.core.permissions.services.tenant_policy_service import TenantPolicyService, TenantPolicyValidationService from dataall.base.db import get_engine -from dataall.core.permissions.services.tenant_permissions import TENANT_ALL from dataall.base.loader import load_modules, ImportMode -from dataall.modules.maintenance.api.enums import MaintenanceModes, MaintenanceStatus -from dataall.modules.maintenance.services.maintenance_service import MaintenanceService -from dataall.base.config import config + logger = logging.getLogger() logger.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) @@ -35,15 +30,10 @@ load_modules(modes={ImportMode.API}) SCHEMA = bootstrap_schema() TYPE_DEFS = gql(SCHEMA.gql(with_directives=False)) -REAUTH_TTL = int(os.environ.get('REAUTH_TTL', '5')) ENVNAME = os.getenv('envname', 'local') ENGINE = get_engine(envname=ENVNAME) Worker.queue = SqsQueue.send - -MAINTENANCE_ALLOWED_OPERATIONS = ['getGroupsForUser', 'getMaintenanceWindowStatus'] - - def resolver_adapter(resolver): def adapted(obj, info, **kwargs): return resolver( @@ -117,7 +107,8 @@ def handler(event, context): user_id = event['requestContext']['authorizer']['user_id'] log.debug('username is %s', username) - groups: list = extract_groups_and_attach_tenant_policy(user_id=user_id, claims=claims) + groups: list = extract_groups(user_id=user_id, claims=claims) + attach_tenant_policy_for_groups(groups=groups) # Set Context set_context(RequestContext(ENGINE, username, groups, user_id)) @@ -130,12 +121,12 @@ def handler(event, context): query = json.loads(event.get('body')) - isBlockedResponse = validate_and_block_if_maintenance_window(query=query, groups=groups) - if isBlockedResponse is not None: - return isBlockedResponse - isReauthResponse = check_reauth(query=query, auth_time=claims['auth_time'], username=username) - if isReauthResponse is not None: - return isReauthResponse + maintenance_window_validation_response = validate_and_block_if_maintenance_window(query=query, groups=groups) + if maintenance_window_validation_response is not None: + return maintenance_window_validation_response + reauth_validation_response = check_reauth(query=query, auth_time=claims['auth_time'], username=username) + if reauth_validation_response is not None: + return reauth_validation_response else: raise Exception(f'Could not initialize user context from event {event}') @@ -158,102 +149,3 @@ def handler(event, context): 'body': response, } - -def check_reauth(query, auth_time, username): - # Determine if there are any Operations that Require ReAuth From SSM Parameter - try: - reauth_apis = ParameterStoreManager.get_parameter_value( - region=os.getenv('AWS_REGION', 'eu-west-1'), parameter_path=f'/dataall/{ENVNAME}/reauth/apis' - ).split(',') - except Exception: - log.info('No ReAuth APIs Found in SSM') - reauth_apis = None - - # If The Operation is a ReAuth Operation - Ensure A Non-Expired Session or Return Error - if reauth_apis and query.get('operationName', None) in reauth_apis: - now = datetime.datetime.now(datetime.timezone.utc) - try: - auth_time_datetime = datetime.datetime.fromtimestamp(int(auth_time), tz=datetime.timezone.utc) - if auth_time_datetime + datetime.timedelta(minutes=REAUTH_TTL) < now: - raise Exception('ReAuth') - except Exception as e: - log.info(f'ReAuth Required for User {username} on Operation {query.get("operationName", "")}, Error: {e}') - response = { - 'data': {query.get('operationName', 'operation'): None}, - 'errors': [ - { - 'message': f"ReAuth Required To Perform This Action {query.get('operationName', '')}", - 'locations': None, - 'path': [query.get('operationName', '')], - 'extensions': {'code': 'REAUTH'}, - } - ], - } - return { - 'statusCode': 401, - 'headers': { - 'content-type': 'application/json', - 'Access-Control-Allow-Origin': '*', - 'Access-Control-Allow-Headers': '*', - 'Access-Control-Allow-Methods': '*', - }, - 'body': json.dumps(response), - } - - -def validate_and_block_if_maintenance_window(query, groups): - # Logic to block when in maintenance - # Check if in some maintenance mode - # Check if in maintenance status is not INACTIVE - # Check if the user belongs to a 'DAAdministrators' group - if config.get_property('modules.maintenance.active'): - maintenance_mode = MaintenanceService._get_maintenance_window_mode(engine=ENGINE) - maintenance_status = MaintenanceService.get_maintenance_window_status(engine=ENGINE).status - isAdmin = TenantPolicyValidationService.is_tenant_admin(groups) - - if ( - (maintenance_mode == MaintenanceModes.NOACCESS.value) - and (maintenance_status is not MaintenanceStatus.INACTIVE.value) - and not isAdmin - ): - if query.get('operationName', '') not in MAINTENANCE_ALLOWED_OPERATIONS: - return send_unauthorized_response( - query=query, - message='Access Restricted: data.all is currently undergoing maintenance, and your actions are temporarily blocked.', - ) - elif ( - (maintenance_mode == MaintenanceModes.READONLY.value) - and (maintenance_status is not MaintenanceStatus.INACTIVE.value) - and not isAdmin - ): - # If its mutation then block and return - if query.get('query', '').split()[0] == 'mutation': - return send_unauthorized_response( - query=query, - message='Access Restricted: data.all is currently undergoing maintenance, and your actions are temporarily blocked.', - ) - - -def extract_groups_and_attach_tenant_policy(user_id, claims): - groups = [] - try: - if os.environ.get('custom_auth', None): - groups.extend(get_custom_groups(user_id)) - else: - groups.extend(get_cognito_groups(claims)) - log.debug('groups are %s', ','.join(groups)) - with ENGINE.scoped_session() as session: - for group in groups: - policy = TenantPolicyService.find_tenant_policy(session, group, TenantPolicyService.TENANT_NAME) - if not policy: - print(f'No policy found for Team {group}. Attaching TENANT_ALL permissions') - TenantPolicyService.attach_group_tenant_policy( - session=session, - group=group, - permissions=TENANT_ALL, - tenant_name=TenantPolicyService.TENANT_NAME, - ) - return groups - except Exception as e: - print(f'Error managing groups due to: {e}') - return groups diff --git a/backend/dataall/base/utils/api_handler_utils.py b/backend/dataall/base/utils/api_handler_utils.py index 9b0e0bda5..9d6c3ef11 100644 --- a/backend/dataall/base/utils/api_handler_utils.py +++ b/backend/dataall/base/utils/api_handler_utils.py @@ -1,6 +1,26 @@ +import datetime import json +import os +import logging +from dataall.base.aws.parameter_store import ParameterStoreManager +from dataall.base.db import get_engine from dataall.base.services.service_provider_factory import ServiceProviderFactory +from dataall.core.permissions.services.tenant_permissions import TENANT_ALL +from dataall.core.permissions.services.tenant_policy_service import TenantPolicyService +from dataall.modules.maintenance.api.enums import MaintenanceModes, MaintenanceStatus +from dataall.modules.maintenance.services.maintenance_service import MaintenanceService +from dataall.base.config import config +from dataall.core.permissions.services.tenant_policy_service import TenantPolicyValidationService + +logger = logging.getLogger() +logger.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) +log = logging.getLogger(__name__) + +ENVNAME = os.getenv('envname', 'local') +REAUTH_TTL = int(os.environ.get('REAUTH_TTL', '5')) +MAINTENANCE_ALLOWED_OPERATIONS = ['getGroupsForUser', 'getMaintenanceWindowStatus'] +ENGINE = get_engine(envname=ENVNAME) def get_cognito_groups(claims): @@ -25,17 +45,19 @@ def get_custom_groups(user_id): return service_provider.get_groups_for_user(user_id) -def send_unauthorized_response(query, message=''): +def send_unauthorized_response(operation='', message='', extension=None): response = { - 'data': {query.get('operationName', 'operation'): None}, + 'data': { operation : None}, 'errors': [ { 'message': message, 'locations': None, - 'path': [query.get('operationName', '')], + 'path': [operation], } ], } + if extension is not None: + response['errors'][0]['extensions'] = extension return { 'statusCode': 401, 'headers': { @@ -46,3 +68,88 @@ def send_unauthorized_response(query, message=''): }, 'body': json.dumps(response), } + +def extract_groups(user_id, claims): + groups = [] + try: + if os.environ.get('custom_auth', None): + groups.extend(get_custom_groups(user_id)) + else: + groups.extend(get_cognito_groups(claims)) + log.debug('groups are %s', ','.join(groups)) + return groups + except Exception as e: + log.exception(f'Error managing groups due to: {e}') + return groups + +def attach_tenant_policy_for_groups(groups=None): + if groups is None: + groups = [] + with ENGINE.scoped_session() as session: + for group in groups: + policy = TenantPolicyService.find_tenant_policy(session, group, TenantPolicyService.TENANT_NAME) + if not policy: + print(f'No policy found for Team {group}. Attaching TENANT_ALL permissions') + TenantPolicyService.attach_group_tenant_policy( + session=session, + group=group, + permissions=TENANT_ALL, + tenant_name=TenantPolicyService.TENANT_NAME, + ) + +def check_reauth(query, auth_time, username): + # Determine if there are any Operations that Require ReAuth From SSM Parameter + try: + reauth_apis = ParameterStoreManager.get_parameter_value( + region=os.getenv('AWS_REGION', 'eu-west-1'), parameter_path=f'/dataall/{ENVNAME}/reauth/apis' + ).split(',') + except Exception: + log.info('No ReAuth APIs Found in SSM') + reauth_apis = None + + # If The Operation is a ReAuth Operation - Ensure A Non-Expired Session or Return Error + if reauth_apis and query.get('operationName', None) in reauth_apis: + now = datetime.datetime.now(datetime.timezone.utc) + try: + auth_time_datetime = datetime.datetime.fromtimestamp(int(auth_time), tz=datetime.timezone.utc) + if auth_time_datetime + datetime.timedelta(minutes=REAUTH_TTL) < now: + raise Exception('ReAuth') + except Exception as e: + log.info(f'ReAuth Required for User {username} on Operation {query.get("operationName", "")}, Error: {e}') + return send_unauthorized_response(operation=query.get('operationName', 'operation'), + message=f"ReAuth Required To Perform This Action {query.get('operationName', '')}", + extension={'code': 'REAUTH'}) + +def validate_and_block_if_maintenance_window(query, groups, blocked_for_mode=None): + # Logic to block when in maintenance + # Check if in some maintenance mode + # Check if in maintenance status is not INACTIVE + # Check if the user belongs to a 'DAAdministrators' group + if config.get_property('modules.maintenance.active'): + maintenance_mode = MaintenanceService._get_maintenance_window_mode(engine=ENGINE) + maintenance_status = MaintenanceService.get_maintenance_window_status().status + isAdmin = TenantPolicyValidationService.is_tenant_admin(groups) + + if ( + (maintenance_mode == MaintenanceModes.NOACCESS.value) + and (maintenance_status is not MaintenanceStatus.INACTIVE.value) + and not isAdmin + and (blocked_for_mode is None or blocked_for_mode == MaintenanceModes.NOACCESS.value) + ): + if query.get('operationName', '') not in MAINTENANCE_ALLOWED_OPERATIONS: + return send_unauthorized_response( + operation=query.get('operationName', 'operation'), + message='Access Restricted: data.all is currently undergoing maintenance, and your actions are temporarily blocked.', + ) + elif ( + (maintenance_mode == MaintenanceModes.READONLY.value) + and (maintenance_status is not MaintenanceStatus.INACTIVE.value) + and not isAdmin + and (blocked_for_mode is None or blocked_for_mode == MaintenanceModes.READONLY.value) + ): + # If its mutation then block and return + if query.get('query', '').split()[0] == 'mutation': + return send_unauthorized_response( + operation=query.get('operationName', 'operation'), + message='Access Restricted: data.all is currently undergoing maintenance, and your actions are temporarily blocked.', + ) \ No newline at end of file diff --git a/backend/dataall/modules/maintenance/api/resolvers.py b/backend/dataall/modules/maintenance/api/resolvers.py index 8955b40b4..182d15f47 100644 --- a/backend/dataall/modules/maintenance/api/resolvers.py +++ b/backend/dataall/modules/maintenance/api/resolvers.py @@ -9,12 +9,12 @@ def start_maintenance_window(context: Context, source: Maintenance, mode: str): """Starts the maintenance window""" if mode not in [item.value for item in list(MaintenanceModes)]: raise Exception('Mode is not conforming to the MaintenanceModes enum') - return MaintenanceService.start_maintenance_window(engine=context.engine, mode=mode, groups=context.groups) + return MaintenanceService.start_maintenance_window(mode=mode) def stop_maintenance_window(context: Context, source: Maintenance): - return MaintenanceService.stop_maintenance_window(engine=context.engine, groups=context.groups) + return MaintenanceService.stop_maintenance_window() def get_maintenance_window_status(context: Context, source: Maintenance): - return MaintenanceService.get_maintenance_window_status(engine=context.engine) + return MaintenanceService.get_maintenance_window_status() diff --git a/backend/dataall/modules/maintenance/services/maintenance_service.py b/backend/dataall/modules/maintenance/services/maintenance_service.py index 4d02b7d4d..bb63e3bc7 100644 --- a/backend/dataall/modules/maintenance/services/maintenance_service.py +++ b/backend/dataall/modules/maintenance/services/maintenance_service.py @@ -8,6 +8,7 @@ from dataall.base.aws.event_bridge import EventBridge from dataall.base.aws.parameter_store import ParameterStoreManager +from dataall.base.context import get_context from dataall.core.permissions.services.tenant_policy_service import TenantPolicyValidationService from dataall.modules.maintenance.api.enums import MaintenanceStatus from dataall.modules.maintenance.db.maintenance_repository import MaintenanceRepository @@ -18,26 +19,23 @@ class MaintenanceService: @staticmethod - def start_maintenance_window(engine, mode: str = None, groups=None): + def start_maintenance_window(mode: str = None): """ Start maintenance window by performing following actions 1. Perform validation to check if the user belongs to the DAAdministrators group 2. Put the maintenance window status to PENDING and update the maintenance mode 3. Get all the ECS Scheduled tasks and disable the schedule for them - @param engine: db engine @param mode: mode to set for maintenance window - @param groups: user groups from context.groups @return: returns True if successful or False """ # Check from the context if the groups contains the DAAAdminstrators group - if groups is None: - groups = [] + groups = get_context().groups if get_context().groups is not None else [] if not TenantPolicyValidationService.is_tenant_admin(groups): raise Exception('Only data.all admin group members can start maintenance window') logger.info('Putting data.all into maintenance') try: - with engine.scoped_session() as session: + with get_context().db_engine.scoped_session() as session: maintenance_record = MaintenanceRepository(session).get_maintenance_record() if ( maintenance_record.status == MaintenanceStatus.PENDING.value @@ -53,33 +51,31 @@ def start_maintenance_window(engine, mode: str = None, groups=None): # Disable scheduled ECS tasks # Get all the SSM Params related to the scheduled tasks ecs_scheduled_rules_list = MaintenanceService._get_ecs_rules() - event_bridge_session = EventBridge(region=os.getenv('AWS_REGION', 'eu-west-1')) - event_bridge_session.disable_scheduled_ecs_tasks(ecs_scheduled_rules_list) + event_bridge_client = EventBridge(region=os.getenv('AWS_REGION', 'eu-west-1')) + event_bridge_client.disable_scheduled_ecs_tasks(ecs_scheduled_rules_list) return True except Exception as e: logger.error(f'Error occurred while starting maintenance window due to {e}') return False @staticmethod - def stop_maintenance_window(engine, groups=None): + def stop_maintenance_window(): """ Stop maintenance window by performing following actions 1. Perform validation to check if the user belongs to the DAAdministrators group 2. Update the RDS table by changing the status to INACTIVE and mode to '-' 3. Enable all data.all related ECS scheduled tasks - @param engine: db engine - @param groups: user groups from context.groups @return: return True if successful or False """ # Check from the context if the groups contains the DAAAdminstrators group - if groups is None: - groups = [] + groups = get_context().groups if get_context().groups is not None else [] + if not TenantPolicyValidationService.is_tenant_admin(groups): raise Exception('Only data.all admin group members can stop maintenance window') logger.info('Stopping maintenance mode') try: - with engine.scoped_session() as session: + with get_context().db_engine.scoped_session() as session: maintenance_record = MaintenanceRepository(session).get_maintenance_record() if maintenance_record.status == MaintenanceStatus.INACTIVE.value: logger.error('Maintenance window already in INACTIVE state. Cannot stop maintenance window') @@ -89,24 +85,23 @@ def stop_maintenance_window(engine, groups=None): ) # Enable scheduled ECS tasks ecs_scheduled_rules_list = MaintenanceService._get_ecs_rules() - event_bridge_session = EventBridge(region=os.getenv('AWS_REGION', 'eu-west-1')) - event_bridge_session.enable_scheduled_ecs_tasks(ecs_scheduled_rules_list) + event_bridge_client = EventBridge(region=os.getenv('AWS_REGION', 'eu-west-1')) + event_bridge_client.enable_scheduled_ecs_tasks(ecs_scheduled_rules_list) return True except Exception as e: logger.error(f'Error occurred while stopping maintenance window due to {e}') return False @staticmethod - def get_maintenance_window_status(engine): + def get_maintenance_window_status(): """ Get the status of maintenance window Maintenance record is returned after checking if all ECS tasks in the data.all created cluster have completed. - @param engine: db object @return: Maintenance object containing status and mode """ logger.info('Checking maintenance window status') try: - with engine.scoped_session() as session: + with get_context().db_engine.scoped_session() as session: maintenance_record = MaintenanceRepository(session).get_maintenance_record() if maintenance_record.status == MaintenanceStatus.PENDING.value: # Check if ECS tasks are running diff --git a/backend/search_handler.py b/backend/search_handler.py index 585eb377d..bb2cab291 100644 --- a/backend/search_handler.py +++ b/backend/search_handler.py @@ -1,16 +1,11 @@ import json import os - -from dataall.base.db import get_engine from dataall.base.searchproxy import connect, run_query -from dataall.base.config import config -from dataall.base.utils.api_handler_utils import send_unauthorized_response, get_custom_groups, get_cognito_groups -from dataall.core.permissions.services.tenant_policy_service import TenantPolicyValidationService -from dataall.modules.maintenance.api.enums import MaintenanceModes, MaintenanceStatus -from dataall.modules.maintenance.services.maintenance_service import MaintenanceService +from dataall.base.utils.api_handler_utils import validate_and_block_if_maintenance_window, extract_groups +from dataall.modules.maintenance.api.enums import MaintenanceModes + ENVNAME = os.getenv('envname', 'local') -ENGINE = get_engine(envname=ENVNAME) es = connect(envname=ENVNAME) @@ -39,26 +34,12 @@ def handler(event, context): if 'user_id' in event['requestContext']['authorizer']: user_id = event['requestContext']['authorizer']['user_id'] - groups = [] - if os.environ.get('custom_auth', None): - groups.extend(get_custom_groups(user_id)) - else: - groups.extend(get_cognito_groups(claims)) + groups: list = extract_groups(user_id, claims) # Check if maintenance window is enabled AND if the maintenance mode is NO-ACCESS - if config.get_property('modules.maintenance.active'): - if ( - (MaintenanceService._get_maintenance_window_mode(engine=ENGINE) == MaintenanceModes.NOACCESS.value) - and ( - MaintenanceService.get_maintenance_window_status(engine=ENGINE).status - is not MaintenanceStatus.INACTIVE.value - ) - and not TenantPolicyValidationService.is_tenant_admin(groups) - ): - send_unauthorized_response( - query={'operationName': 'OpensearchIndex'}, - message='Access Restricted: data.all is currently undergoing maintenance, and your actions are temporarily blocked.', - ) + maintenance_window_validation_response = validate_and_block_if_maintenance_window(query={'operationName': 'OpensearchIndex'}, groups=groups, blocked_for_mode=MaintenanceModes.NOACCESS) + if maintenance_window_validation_response is not None: + return maintenance_window_validation_response body = event.get('body') print(body) diff --git a/frontend/src/authentication/components/AuthGuard.js b/frontend/src/authentication/components/AuthGuard.js index 98f995061..063c80eb1 100644 --- a/frontend/src/authentication/components/AuthGuard.js +++ b/frontend/src/authentication/components/AuthGuard.js @@ -8,15 +8,15 @@ import { ModuleNames, RegexToValidateWindowPathName, WindowPathLengthThreshold -} from '../../utils'; -import { useClient, useGroups } from '../../services'; -import { LoadingScreen, NoAccessMaintenanceWindow } from '../../design'; +} from 'utils'; +import { useClient, useGroups } from 'services'; +import { LoadingScreen, NoAccessMaintenanceWindow } from 'design'; import { getMaintenanceStatus } from '../../modules/Maintenance/services'; import { PENDING_STATUS, ACTIVE_STATUS -} from '../../modules/Maintenance/views/MaintenanceViewer'; -import { SET_ERROR, useDispatch } from '../../globalErrors'; +} from '../../modules/Maintenance/components/MaintenanceViewer'; +import { SET_ERROR, useDispatch } from 'globalErrors'; export const AuthGuard = (props) => { const { children } = props; diff --git a/frontend/src/design/components/layout/DefaultNavbar.js b/frontend/src/design/components/layout/DefaultNavbar.js index 1ba7da090..a861011fb 100644 --- a/frontend/src/design/components/layout/DefaultNavbar.js +++ b/frontend/src/design/components/layout/DefaultNavbar.js @@ -11,10 +11,10 @@ import config from '../../../generated/config.json'; import { PENDING_STATUS, ACTIVE_STATUS -} from '../../../modules/Maintenance/views/MaintenanceViewer'; -import { useClient } from '../../../services'; +} from '../../../modules/Maintenance/components/MaintenanceViewer'; +import { useClient } from 'services'; import { getMaintenanceStatus } from '../../../modules/Maintenance/services'; -import { SET_ERROR, useDispatch } from '../../../globalErrors'; +import { SET_ERROR, useDispatch } from 'globalErrors'; import { SanitizedHTML } from '../SanitizedHTML'; const useStyles = makeStyles((theme) => ({ diff --git a/frontend/src/modules/Administration/views/AdministrationView.js b/frontend/src/modules/Administration/views/AdministrationView.js index a70183cb7..060396548 100644 --- a/frontend/src/modules/Administration/views/AdministrationView.js +++ b/frontend/src/modules/Administration/views/AdministrationView.js @@ -14,8 +14,8 @@ import { Helmet } from 'react-helmet-async'; import { Link as RouterLink } from 'react-router-dom'; import { ChevronRightIcon, useSettings } from 'design'; import { AdministrationTeams, DashboardViewer } from '../components'; -import { MaintenanceViewer } from '../../Maintenance/views/MaintenanceViewer'; -import { isModuleEnabled, ModuleNames } from '../../../utils'; +import { MaintenanceViewer } from '../../Maintenance/components/MaintenanceViewer'; +import { isModuleEnabled, ModuleNames } from 'utils'; const tabs = [ { label: 'Teams', value: 'teams' }, diff --git a/frontend/src/modules/Maintenance/views/MaintenanceViewer.js b/frontend/src/modules/Maintenance/components/MaintenanceViewer.js similarity index 98% rename from frontend/src/modules/Maintenance/views/MaintenanceViewer.js rename to frontend/src/modules/Maintenance/components/MaintenanceViewer.js index 76f43d175..2feb3713e 100644 --- a/frontend/src/modules/Maintenance/views/MaintenanceViewer.js +++ b/frontend/src/modules/Maintenance/components/MaintenanceViewer.js @@ -15,14 +15,14 @@ import { import React, { useCallback, useEffect, useState } from 'react'; import { Article, CancelRounded, SystemUpdate } from '@mui/icons-material'; import { LoadingButton } from '@mui/lab'; -import { Label } from '../../../design'; +import { Label } from 'design'; import { getMaintenanceStatus, stopMaintenanceWindow, startMaintenanceWindow } from '../services'; -import { useClient } from '../../../services'; -import { SET_ERROR, useDispatch } from '../../../globalErrors'; +import { useClient } from 'services'; +import { SET_ERROR, useDispatch } from 'globalErrors'; import { useSnackbar } from 'notistack'; const maintenanceModes = [ From f7fb57711dca53e750630737df97e80ffad690bc Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Mon, 13 May 2024 17:03:24 -0500 Subject: [PATCH 28/36] Making enums conform to graphQLMapper --- backend/dataall/modules/maintenance/api/enums.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/backend/dataall/modules/maintenance/api/enums.py b/backend/dataall/modules/maintenance/api/enums.py index 8771188e7..293d731a1 100644 --- a/backend/dataall/modules/maintenance/api/enums.py +++ b/backend/dataall/modules/maintenance/api/enums.py @@ -1,16 +1,15 @@ """Contains the enums used in maintenance module""" +from dataall.base.api import GraphQLEnumMapper -from enum import Enum - -class MaintenanceModes(Enum): +class MaintenanceModes(GraphQLEnumMapper): """Describes the Maintenance Modes""" READONLY = 'READ-ONLY' NOACCESS = 'NO-ACCESS' -class MaintenanceStatus(Enum): +class MaintenanceStatus(GraphQLEnumMapper): """Describe the various statuses for maintenance""" PENDING = 'PENDING' From e7eba0c635551b961c88e0779667c472c9a7fae6 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Tue, 14 May 2024 09:40:31 -0500 Subject: [PATCH 29/36] trying few things --- backend/local_graphql_server.py | 5 +++++ backend/search_handler.py | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/backend/local_graphql_server.py b/backend/local_graphql_server.py index b92e86ca9..47251508b 100644 --- a/backend/local_graphql_server.py +++ b/backend/local_graphql_server.py @@ -5,6 +5,7 @@ from ariadne.constants import PLAYGROUND_HTML from flask import Flask, request, jsonify from flask_cors import CORS +from graphql import parse from dataall.base.api import get_executable_schema from dataall.core.tasks.service_handlers import Worker @@ -128,6 +129,10 @@ def graphql_server(): data = request.get_json() print('*** Request ***', request.data) + query = parse(data) + print("***** Printing Query ****** \n\n") + print(query) + context = request_context(request.headers, mock=True) logger.debug(context) diff --git a/backend/search_handler.py b/backend/search_handler.py index bb2cab291..967926542 100644 --- a/backend/search_handler.py +++ b/backend/search_handler.py @@ -1,5 +1,8 @@ import json import os + +from dataall.base.context import RequestContext, set_context +from dataall.base.db import get_engine from dataall.base.searchproxy import connect, run_query from dataall.base.utils.api_handler_utils import validate_and_block_if_maintenance_window, extract_groups from dataall.modules.maintenance.api.enums import MaintenanceModes @@ -7,6 +10,7 @@ ENVNAME = os.getenv('envname', 'local') es = connect(envname=ENVNAME) +ENGINE = get_engine(envname=ENVNAME) def handler(event, context): @@ -29,6 +33,8 @@ def handler(event, context): else: claims = event['requestContext']['authorizer']['claims'] + username = claims['email'] + # Needed for custom groups user_id = claims['email'] if 'user_id' in event['requestContext']['authorizer']: @@ -36,6 +42,8 @@ def handler(event, context): groups: list = extract_groups(user_id, claims) + set_context(RequestContext(ENGINE, username, groups, user_id)) + # Check if maintenance window is enabled AND if the maintenance mode is NO-ACCESS maintenance_window_validation_response = validate_and_block_if_maintenance_window(query={'operationName': 'OpensearchIndex'}, groups=groups, blocked_for_mode=MaintenanceModes.NOACCESS) if maintenance_window_validation_response is not None: From 85803fbba63642945539629f6ab3ab7541abe232 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Tue, 14 May 2024 17:46:59 -0500 Subject: [PATCH 30/36] resolving last of code review steps --- .../dataall/base/utils/api_handler_utils.py | 44 +++++++++++++------ .../services/maintenance_service.py | 1 + backend/search_handler.py | 6 ++- frontend/src/services/hooks/useClient.js | 4 ++ 4 files changed, 40 insertions(+), 15 deletions(-) diff --git a/backend/dataall/base/utils/api_handler_utils.py b/backend/dataall/base/utils/api_handler_utils.py index 9d6c3ef11..deda452ea 100644 --- a/backend/dataall/base/utils/api_handler_utils.py +++ b/backend/dataall/base/utils/api_handler_utils.py @@ -3,6 +3,7 @@ import os import logging +from graphql import parse, utilities, OperationType, GraphQLSyntaxError from dataall.base.aws.parameter_store import ParameterStoreManager from dataall.base.db import get_engine from dataall.base.services.service_provider_factory import ServiceProviderFactory @@ -19,7 +20,8 @@ ENVNAME = os.getenv('envname', 'local') REAUTH_TTL = int(os.environ.get('REAUTH_TTL', '5')) -MAINTENANCE_ALLOWED_OPERATIONS = ['getGroupsForUser', 'getMaintenanceWindowStatus'] +# ALLOWED OPERATIONS WHEN A USER IS NOT DATAALL ADMIN AND NO-ACCESS MODE IS SELECTED +MAINTENANCE_ALLOWED_OPERATIONS_WHEN_NO_ACCESS = [item.casefold() for item in ['getGroupsForUser', 'getMaintenanceWindowStatus']] ENGINE = get_engine(envname=ENVNAME) @@ -120,11 +122,19 @@ def check_reauth(query, auth_time, username): message=f"ReAuth Required To Perform This Action {query.get('operationName', '')}", extension={'code': 'REAUTH'}) -def validate_and_block_if_maintenance_window(query, groups, blocked_for_mode=None): - # Logic to block when in maintenance - # Check if in some maintenance mode - # Check if in maintenance status is not INACTIVE - # Check if the user belongs to a 'DAAdministrators' group +def validate_and_block_if_maintenance_window(query, groups, blocked_for_mode_enum=None): + """ + When the maintenance module is set to active, checks + - If the maintenance mode is enabled + - Based on the maintenance mode, actions which can be taken by user can be modified + - READ-ONLY -> Block All Mutation calls and allow query graphql calls + - NO-ACCESS -> Block All graphql query call irrespective of type + - Check if the user belongs to the DAAdministrators group + @param query: graphql query dict containing operation, query, variables + @param groups: user groups + @param blocked_for_mode_enum: sets the mode for blocking only specific modes. When set to None, both graphql types ( Query and Mutation ) will be blocked. When a specific mode is set, blocking will only occure for that mode + @return: error response if maintenance window is blocking gql calls else None + """ if config.get_property('modules.maintenance.active'): maintenance_mode = MaintenanceService._get_maintenance_window_mode(engine=ENGINE) maintenance_status = MaintenanceService.get_maintenance_window_status().status @@ -134,9 +144,9 @@ def validate_and_block_if_maintenance_window(query, groups, blocked_for_mode=Non (maintenance_mode == MaintenanceModes.NOACCESS.value) and (maintenance_status is not MaintenanceStatus.INACTIVE.value) and not isAdmin - and (blocked_for_mode is None or blocked_for_mode == MaintenanceModes.NOACCESS.value) + and (blocked_for_mode_enum is None or blocked_for_mode_enum == MaintenanceModes.NOACCESS) ): - if query.get('operationName', '') not in MAINTENANCE_ALLOWED_OPERATIONS: + if query.get('operationName', '').casefold() not in MAINTENANCE_ALLOWED_OPERATIONS_WHEN_NO_ACCESS: return send_unauthorized_response( operation=query.get('operationName', 'operation'), message='Access Restricted: data.all is currently undergoing maintenance, and your actions are temporarily blocked.', @@ -145,11 +155,17 @@ def validate_and_block_if_maintenance_window(query, groups, blocked_for_mode=Non (maintenance_mode == MaintenanceModes.READONLY.value) and (maintenance_status is not MaintenanceStatus.INACTIVE.value) and not isAdmin - and (blocked_for_mode is None or blocked_for_mode == MaintenanceModes.READONLY.value) + and (blocked_for_mode_enum is None or blocked_for_mode_enum == MaintenanceModes.READONLY) ): # If its mutation then block and return - if query.get('query', '').split()[0] == 'mutation': - return send_unauthorized_response( - operation=query.get('operationName', 'operation'), - message='Access Restricted: data.all is currently undergoing maintenance, and your actions are temporarily blocked.', - ) \ No newline at end of file + try: + parsed_query_document = parse(query.get('query', '')) + graphQL_operation_type = utilities.get_operation_ast(parsed_query_document) + if graphQL_operation_type.operation == OperationType.MUTATION: + return send_unauthorized_response( + operation=query.get('operationName', 'operation'), + message='Access Restricted: data.all is currently undergoing maintenance, and your actions are temporarily blocked.', + ) + except GraphQLSyntaxError as e: + log.error(f'Error occured while parsing query when validating for {maintenance_mode} maintenance mode due to - {e}') + raise e \ No newline at end of file diff --git a/backend/dataall/modules/maintenance/services/maintenance_service.py b/backend/dataall/modules/maintenance/services/maintenance_service.py index bb63e3bc7..b77e97cc1 100644 --- a/backend/dataall/modules/maintenance/services/maintenance_service.py +++ b/backend/dataall/modules/maintenance/services/maintenance_service.py @@ -133,6 +133,7 @@ def _get_maintenance_window_mode(engine): try: with engine.scoped_session() as session: maintenance_record = MaintenanceRepository(session).get_maintenance_record() + logger.debug(f'Current maintenance window mode - {maintenance_record.mode}') return maintenance_record.mode except Exception as e: logger.error(f'Error while getting maintenance window mode due to {e}') diff --git a/backend/search_handler.py b/backend/search_handler.py index 967926542..f994e08c7 100644 --- a/backend/search_handler.py +++ b/backend/search_handler.py @@ -45,7 +45,11 @@ def handler(event, context): set_context(RequestContext(ENGINE, username, groups, user_id)) # Check if maintenance window is enabled AND if the maintenance mode is NO-ACCESS - maintenance_window_validation_response = validate_and_block_if_maintenance_window(query={'operationName': 'OpensearchIndex'}, groups=groups, blocked_for_mode=MaintenanceModes.NOACCESS) + maintenance_window_validation_response = validate_and_block_if_maintenance_window( + query={'operationName': 'OpensearchIndex'}, + groups=groups, + blocked_for_mode_enum=MaintenanceModes.NOACCESS + ) if maintenance_window_validation_response is not None: return maintenance_window_validation_response diff --git a/frontend/src/services/hooks/useClient.js b/frontend/src/services/hooks/useClient.js index 061b98f0e..80967bb86 100644 --- a/frontend/src/services/hooks/useClient.js +++ b/frontend/src/services/hooks/useClient.js @@ -75,6 +75,10 @@ export const useClient = () => { if (extensions?.code === 'REAUTH') { setReAuth(operation); } + // Dispatch to show message when a 4xx network error is returned + if (networkError) { + dispatch({ type: SET_ERROR, error: `${message}` }); + } } ); } From 713605f889b3594556e0c7dfebe6ce552297e5a8 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Tue, 14 May 2024 17:48:05 -0500 Subject: [PATCH 31/36] fixing linting --- backend/api_handler.py | 10 ++++--- .../dataall/base/utils/api_handler_utils.py | 26 +++++++++++++------ .../dataall/modules/maintenance/api/enums.py | 1 + backend/local_graphql_server.py | 2 +- backend/search_handler.py | 2 +- 5 files changed, 28 insertions(+), 13 deletions(-) diff --git a/backend/api_handler.py b/backend/api_handler.py index e3847f21c..144511441 100644 --- a/backend/api_handler.py +++ b/backend/api_handler.py @@ -10,8 +10,12 @@ ) from dataall.base.api import bootstrap as bootstrap_schema, get_executable_schema -from dataall.base.utils.api_handler_utils import extract_groups, attach_tenant_policy_for_groups, check_reauth, \ - validate_and_block_if_maintenance_window +from dataall.base.utils.api_handler_utils import ( + extract_groups, + attach_tenant_policy_for_groups, + check_reauth, + validate_and_block_if_maintenance_window, +) from dataall.core.tasks.service_handlers import Worker from dataall.base.aws.sqs import SqsQueue from dataall.base.context import set_context, dispose_context, RequestContext @@ -34,6 +38,7 @@ ENGINE = get_engine(envname=ENVNAME) Worker.queue = SqsQueue.send + def resolver_adapter(resolver): def adapted(obj, info, **kwargs): return resolver( @@ -148,4 +153,3 @@ def handler(event, context): }, 'body': response, } - diff --git a/backend/dataall/base/utils/api_handler_utils.py b/backend/dataall/base/utils/api_handler_utils.py index deda452ea..bf011f669 100644 --- a/backend/dataall/base/utils/api_handler_utils.py +++ b/backend/dataall/base/utils/api_handler_utils.py @@ -12,7 +12,7 @@ from dataall.modules.maintenance.api.enums import MaintenanceModes, MaintenanceStatus from dataall.modules.maintenance.services.maintenance_service import MaintenanceService from dataall.base.config import config -from dataall.core.permissions.services.tenant_policy_service import TenantPolicyValidationService +from dataall.core.permissions.services.tenant_policy_service import TenantPolicyValidationService logger = logging.getLogger() logger.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) @@ -21,7 +21,9 @@ ENVNAME = os.getenv('envname', 'local') REAUTH_TTL = int(os.environ.get('REAUTH_TTL', '5')) # ALLOWED OPERATIONS WHEN A USER IS NOT DATAALL ADMIN AND NO-ACCESS MODE IS SELECTED -MAINTENANCE_ALLOWED_OPERATIONS_WHEN_NO_ACCESS = [item.casefold() for item in ['getGroupsForUser', 'getMaintenanceWindowStatus']] +MAINTENANCE_ALLOWED_OPERATIONS_WHEN_NO_ACCESS = [ + item.casefold() for item in ['getGroupsForUser', 'getMaintenanceWindowStatus'] +] ENGINE = get_engine(envname=ENVNAME) @@ -49,7 +51,7 @@ def get_custom_groups(user_id): def send_unauthorized_response(operation='', message='', extension=None): response = { - 'data': { operation : None}, + 'data': {operation: None}, 'errors': [ { 'message': message, @@ -71,6 +73,7 @@ def send_unauthorized_response(operation='', message='', extension=None): 'body': json.dumps(response), } + def extract_groups(user_id, claims): groups = [] try: @@ -84,6 +87,7 @@ def extract_groups(user_id, claims): log.exception(f'Error managing groups due to: {e}') return groups + def attach_tenant_policy_for_groups(groups=None): if groups is None: groups = [] @@ -99,6 +103,7 @@ def attach_tenant_policy_for_groups(groups=None): tenant_name=TenantPolicyService.TENANT_NAME, ) + def check_reauth(query, auth_time, username): # Determine if there are any Operations that Require ReAuth From SSM Parameter try: @@ -118,9 +123,12 @@ def check_reauth(query, auth_time, username): raise Exception('ReAuth') except Exception as e: log.info(f'ReAuth Required for User {username} on Operation {query.get("operationName", "")}, Error: {e}') - return send_unauthorized_response(operation=query.get('operationName', 'operation'), - message=f"ReAuth Required To Perform This Action {query.get('operationName', '')}", - extension={'code': 'REAUTH'}) + return send_unauthorized_response( + operation=query.get('operationName', 'operation'), + message=f"ReAuth Required To Perform This Action {query.get('operationName', '')}", + extension={'code': 'REAUTH'}, + ) + def validate_and_block_if_maintenance_window(query, groups, blocked_for_mode_enum=None): """ @@ -167,5 +175,7 @@ def validate_and_block_if_maintenance_window(query, groups, blocked_for_mode_enu message='Access Restricted: data.all is currently undergoing maintenance, and your actions are temporarily blocked.', ) except GraphQLSyntaxError as e: - log.error(f'Error occured while parsing query when validating for {maintenance_mode} maintenance mode due to - {e}') - raise e \ No newline at end of file + log.error( + f'Error occured while parsing query when validating for {maintenance_mode} maintenance mode due to - {e}' + ) + raise e diff --git a/backend/dataall/modules/maintenance/api/enums.py b/backend/dataall/modules/maintenance/api/enums.py index 293d731a1..36c328913 100644 --- a/backend/dataall/modules/maintenance/api/enums.py +++ b/backend/dataall/modules/maintenance/api/enums.py @@ -1,4 +1,5 @@ """Contains the enums used in maintenance module""" + from dataall.base.api import GraphQLEnumMapper diff --git a/backend/local_graphql_server.py b/backend/local_graphql_server.py index 47251508b..44ed6bf1f 100644 --- a/backend/local_graphql_server.py +++ b/backend/local_graphql_server.py @@ -130,7 +130,7 @@ def graphql_server(): print('*** Request ***', request.data) query = parse(data) - print("***** Printing Query ****** \n\n") + print('***** Printing Query ****** \n\n') print(query) context = request_context(request.headers, mock=True) diff --git a/backend/search_handler.py b/backend/search_handler.py index f994e08c7..7985be272 100644 --- a/backend/search_handler.py +++ b/backend/search_handler.py @@ -48,7 +48,7 @@ def handler(event, context): maintenance_window_validation_response = validate_and_block_if_maintenance_window( query={'operationName': 'OpensearchIndex'}, groups=groups, - blocked_for_mode_enum=MaintenanceModes.NOACCESS + blocked_for_mode_enum=MaintenanceModes.NOACCESS, ) if maintenance_window_validation_response is not None: return maintenance_window_validation_response From 6b383cef7dfe334c253f709ee32a47a4d4659b9f Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Tue, 14 May 2024 18:04:05 -0500 Subject: [PATCH 32/36] Minor Changes --- backend/dataall/base/utils/api_handler_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/dataall/base/utils/api_handler_utils.py b/backend/dataall/base/utils/api_handler_utils.py index bf011f669..71f4351cf 100644 --- a/backend/dataall/base/utils/api_handler_utils.py +++ b/backend/dataall/base/utils/api_handler_utils.py @@ -95,7 +95,7 @@ def attach_tenant_policy_for_groups(groups=None): for group in groups: policy = TenantPolicyService.find_tenant_policy(session, group, TenantPolicyService.TENANT_NAME) if not policy: - print(f'No policy found for Team {group}. Attaching TENANT_ALL permissions') + log.exception(f'No policy found for Team {group}. Attaching TENANT_ALL permissions') TenantPolicyService.attach_group_tenant_policy( session=session, group=group, From 8180382ad7e713be05f0a83b9859f5f924427f07 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Tue, 14 May 2024 18:04:58 -0500 Subject: [PATCH 33/36] Minor Changes - 1 --- backend/dataall/base/utils/api_handler_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/dataall/base/utils/api_handler_utils.py b/backend/dataall/base/utils/api_handler_utils.py index 71f4351cf..1413a151c 100644 --- a/backend/dataall/base/utils/api_handler_utils.py +++ b/backend/dataall/base/utils/api_handler_utils.py @@ -95,7 +95,7 @@ def attach_tenant_policy_for_groups(groups=None): for group in groups: policy = TenantPolicyService.find_tenant_policy(session, group, TenantPolicyService.TENANT_NAME) if not policy: - log.exception(f'No policy found for Team {group}. Attaching TENANT_ALL permissions') + log.info(f'No policy found for Team {group}. Attaching TENANT_ALL permissions') TenantPolicyService.attach_group_tenant_policy( session=session, group=group, From c5d6be93defe160b726e7b9e48304a2f616ec6c0 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Thu, 23 May 2024 12:43:14 -0500 Subject: [PATCH 34/36] Moving code and addressing code review comments --- backend/api_handler.py | 1 - backend/dataall/modules/maintenance/api/resolvers.py | 1 - backend/dataall/modules/maintenance/aws/__init__.py | 0 .../maintenance}/aws/event_bridge.py | 0 .../maintenance/services/maintenance_service.py | 12 ++++++------ 5 files changed, 6 insertions(+), 8 deletions(-) create mode 100644 backend/dataall/modules/maintenance/aws/__init__.py rename backend/dataall/{base => modules/maintenance}/aws/event_bridge.py (100%) diff --git a/backend/api_handler.py b/backend/api_handler.py index 144511441..74559b1ac 100644 --- a/backend/api_handler.py +++ b/backend/api_handler.py @@ -115,7 +115,6 @@ def handler(event, context): groups: list = extract_groups(user_id=user_id, claims=claims) attach_tenant_policy_for_groups(groups=groups) - # Set Context set_context(RequestContext(ENGINE, username, groups, user_id)) app_context = { 'engine': ENGINE, diff --git a/backend/dataall/modules/maintenance/api/resolvers.py b/backend/dataall/modules/maintenance/api/resolvers.py index 182d15f47..8d6a09829 100644 --- a/backend/dataall/modules/maintenance/api/resolvers.py +++ b/backend/dataall/modules/maintenance/api/resolvers.py @@ -1,5 +1,4 @@ from dataall.base.api.context import Context -from dataall.core.permissions.services.tenant_policy_service import TenantPolicyValidationService from dataall.modules.maintenance.api.enums import MaintenanceModes from dataall.modules.maintenance.api.types import Maintenance from dataall.modules.maintenance.services.maintenance_service import MaintenanceService diff --git a/backend/dataall/modules/maintenance/aws/__init__.py b/backend/dataall/modules/maintenance/aws/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/dataall/base/aws/event_bridge.py b/backend/dataall/modules/maintenance/aws/event_bridge.py similarity index 100% rename from backend/dataall/base/aws/event_bridge.py rename to backend/dataall/modules/maintenance/aws/event_bridge.py diff --git a/backend/dataall/modules/maintenance/services/maintenance_service.py b/backend/dataall/modules/maintenance/services/maintenance_service.py index b77e97cc1..4fb270859 100644 --- a/backend/dataall/modules/maintenance/services/maintenance_service.py +++ b/backend/dataall/modules/maintenance/services/maintenance_service.py @@ -6,7 +6,7 @@ import logging import os -from dataall.base.aws.event_bridge import EventBridge +from dataall.modules.maintenance.aws.event_bridge import EventBridge from dataall.base.aws.parameter_store import ParameterStoreManager from dataall.base.context import get_context from dataall.core.permissions.services.tenant_policy_service import TenantPolicyValidationService @@ -51,8 +51,8 @@ def start_maintenance_window(mode: str = None): # Disable scheduled ECS tasks # Get all the SSM Params related to the scheduled tasks ecs_scheduled_rules_list = MaintenanceService._get_ecs_rules() - event_bridge_client = EventBridge(region=os.getenv('AWS_REGION', 'eu-west-1')) - event_bridge_client.disable_scheduled_ecs_tasks(ecs_scheduled_rules_list) + event_bridge = EventBridge(region=os.getenv('AWS_REGION', 'eu-west-1')) + event_bridge.disable_scheduled_ecs_tasks(ecs_scheduled_rules_list) return True except Exception as e: logger.error(f'Error occurred while starting maintenance window due to {e}') @@ -81,12 +81,12 @@ def stop_maintenance_window(): logger.error('Maintenance window already in INACTIVE state. Cannot stop maintenance window') return False MaintenanceRepository(session).save_maintenance_status_and_mode( - maintenance_status='INACTIVE', maintenance_mode='' + maintenance_status=MaintenanceStatus.INACTIVE.value, maintenance_mode='' ) # Enable scheduled ECS tasks ecs_scheduled_rules_list = MaintenanceService._get_ecs_rules() - event_bridge_client = EventBridge(region=os.getenv('AWS_REGION', 'eu-west-1')) - event_bridge_client.enable_scheduled_ecs_tasks(ecs_scheduled_rules_list) + event_bridge = EventBridge(region=os.getenv('AWS_REGION', 'eu-west-1')) + event_bridge.enable_scheduled_ecs_tasks(ecs_scheduled_rules_list) return True except Exception as e: logger.error(f'Error occurred while stopping maintenance window due to {e}') From e5728a5ed22bc5702d84867f68cde63a9a9a7283 Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Thu, 23 May 2024 12:50:29 -0500 Subject: [PATCH 35/36] Resolving migration tasks --- .../versions/d059eead99c2_rename_dataset_table_as_s3_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/migrations/versions/d059eead99c2_rename_dataset_table_as_s3_dataset.py b/backend/migrations/versions/d059eead99c2_rename_dataset_table_as_s3_dataset.py index 0c3dcdcba..4b6630f55 100644 --- a/backend/migrations/versions/d059eead99c2_rename_dataset_table_as_s3_dataset.py +++ b/backend/migrations/versions/d059eead99c2_rename_dataset_table_as_s3_dataset.py @@ -12,7 +12,7 @@ # revision identifiers, used by Alembic. revision = 'd059eead99c2' -down_revision = '458572580709' +down_revision = 'b833ad41db68' branch_labels = None depends_on = None From 9088ea54903db4346e3805cd0cc57f4f4703220f Mon Sep 17 00:00:00 2001 From: Tejas Rajopadhye Date: Thu, 23 May 2024 13:06:16 -0500 Subject: [PATCH 36/36] Removing unneccsary comments --- backend/dataall/modules/maintenance/api/resolvers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/dataall/modules/maintenance/api/resolvers.py b/backend/dataall/modules/maintenance/api/resolvers.py index 8d6a09829..88597f324 100644 --- a/backend/dataall/modules/maintenance/api/resolvers.py +++ b/backend/dataall/modules/maintenance/api/resolvers.py @@ -5,7 +5,6 @@ def start_maintenance_window(context: Context, source: Maintenance, mode: str): - """Starts the maintenance window""" if mode not in [item.value for item in list(MaintenanceModes)]: raise Exception('Mode is not conforming to the MaintenanceModes enum') return MaintenanceService.start_maintenance_window(mode=mode)