[Gh-1032] Feature flags for topics and confidentiality and custom confidentiality list

backend/dataall/modules/datasets/api/dataset/input_types.py

@@ -19,7 +19,7 @@
         gql.Argument(
             name='businessOwnerDelegationEmails', type=gql.ArrayType(gql.String)
         ),
-        gql.Argument('confidentiality', gql.Ref('ConfidentialityClassification')),
+        gql.Argument('confidentiality', gql.String),
         gql.Argument(name='stewards', type=gql.String),
         gql.Argument(name='autoApprovalEnabled', type=gql.Boolean)
     ],
@@ -36,7 +36,7 @@
         gql.Argument('businessOwnerDelegationEmails', gql.ArrayType(gql.String)),
         gql.Argument('businessOwnerEmail', gql.String),
         gql.Argument('language', gql.Ref('Language')),
-        gql.Argument('confidentiality', gql.Ref('ConfidentialityClassification')),
+        gql.Argument('confidentiality', gql.String),
         gql.Argument(name='stewards', type=gql.String),
         gql.Argument('KmsAlias', gql.NonNullableType(gql.String)),
         gql.Argument(name='autoApprovalEnabled', type=gql.Boolean)
@@ -103,7 +103,7 @@
         gql.Argument(
             name='businessOwnerDelegationEmails', type=gql.ArrayType(gql.String)
         ),
-        gql.Argument('confidentiality', gql.Ref('ConfidentialityClassification')),
+        gql.Argument('confidentiality', gql.String),
         gql.Argument(name='stewards', type=gql.String),
         gql.Argument(name='autoApprovalEnabled', type=gql.Boolean)
 

backend/dataall/modules/datasets/api/dataset/resolvers.py

@@ -9,22 +9,25 @@
 from dataall.base.db.exceptions import RequiredParameter, InvalidInput
 from dataall.modules.dataset_sharing.db.share_object_models import ShareObject
 from dataall.modules.datasets_base.db.dataset_models import Dataset
-from dataall.modules.datasets_base.services.datasets_base_enums import DatasetRole
+from dataall.modules.datasets_base.services.datasets_base_enums import DatasetRole, ConfidentialityClassification
 from dataall.modules.datasets.services.dataset_service import DatasetService
+from dataall.base.config import config
+
+custom_confidentiality_mapping = config.get_property('modules.datasets.features.custom_confidentiality_mapping', {})
 
 log = logging.getLogger(__name__)
 
 
 def create_dataset(context: Context, source, input=None):
-    RequestValidator.validate_creation_request(input)
+    RequestValidator.validate_creation_request(input, context)
 
     admin_group = input['SamlAdminGroupName']
     uri = input['environmentUri']
     return DatasetService.create_dataset(uri=uri, admin_group=admin_group, data=input)
 
 
 def import_dataset(context: Context, source, input=None):
-    RequestValidator.validate_import_request(input)
+    RequestValidator.validate_import_request(input, context)
 
     admin_group = input['SamlAdminGroupName']
     uri = input['environmentUri']
@@ -192,7 +195,7 @@ def list_datasets_owned_by_env_group(
 
 class RequestValidator:
     @staticmethod
-    def validate_creation_request(data):
+    def validate_creation_request(data, context):
         if not data:
             raise RequiredParameter(data)
         if not data.get('environmentUri'):
@@ -201,13 +204,19 @@ def validate_creation_request(data):
             raise RequiredParameter('group')
         if not data.get('label'):
             raise RequiredParameter('label')
+        if context.engine.dbconfig.schema == 'pytest':
+            confidentiality = data.get('confidentiality') or ConfidentialityClassification.Unclassified.value
+        else:
+            confidentiality = data.get('confidentiality') if not custom_confidentiality_mapping else custom_confidentiality_mapping[data.get('confidentiality')]
+        if config.get_property('modules.datasets.features.confidentiality_dropdown', False) and confidentiality not in [item.value for item in list(ConfidentialityClassification)]:
+            raise InvalidInput('Confidentiality Name', confidentiality, 'does not conform to the confidentiality classification. Hint: Check your confidentiality value OR check your mapping if you are using custom confidentiality values')
         if len(data['label']) > 52:
             raise InvalidInput(
                 'Dataset name', data['label'], 'less than 52 characters'
             )
 
     @staticmethod
-    def validate_import_request(data):
-        RequestValidator.validate_creation_request(data)
+    def validate_import_request(data, context):
+        RequestValidator.validate_creation_request(data, context)
         if not data.get('bucketName'):
             raise RequiredParameter('bucketName')

backend/dataall/modules/datasets/api/dataset/types.py

@@ -129,7 +129,7 @@
         ),
         gql.Field(name='topics', type=gql.ArrayType(gql.Ref('Topic'))),
         gql.Field(
-            name='confidentiality', type=gql.Ref('ConfidentialityClassification')
+            name='confidentiality', type=gql.String
         ),
         gql.Field(name='language', type=gql.Ref('Language')),
         gql.Field(

backend/dataall/modules/datasets/services/dataset_column_service.py

@@ -33,7 +33,7 @@ def paginate_active_columns_for_table(uri: str, filter=None):
             table: DatasetTable = DatasetTableRepository.get_dataset_table_by_uri(session, uri)
             dataset = DatasetRepository.get_dataset_by_uri(session, table.datasetUri)
             if (
-                    dataset.confidentiality != ConfidentialityClassification.Unclassified.value
+                    ConfidentialityClassification.get_confidentiality_level(dataset.confidentiality, context) != ConfidentialityClassification.Unclassified.value
             ):
                 ResourcePolicy.check_user_resource_permission(
                     session=session,

backend/dataall/modules/datasets/services/dataset_profiling_service.py

@@ -110,7 +110,7 @@ def _check_preview_permissions_if_needed(session, table_uri):
             session, table_uri
         )
         dataset = DatasetRepository.get_dataset_by_uri(session, table.datasetUri)
-        if dataset.confidentiality != ConfidentialityClassification.Unclassified.value:
+        if ConfidentialityClassification.get_confidentiality_level(dataset.confidentiality, context) != ConfidentialityClassification.Unclassified.value:
             ResourcePolicy.check_user_resource_permission(
                 session=session,
                 username=context.username,

backend/dataall/modules/datasets/services/dataset_table_service.py

@@ -85,7 +85,7 @@ def preview(table_uri: str):
             )
             dataset = DatasetRepository.get_dataset_by_uri(session, table.datasetUri)
             if (
-                    dataset.confidentiality != ConfidentialityClassification.Unclassified.value
+                ConfidentialityClassification.get_confidentiality_level(dataset.confidentiality, context) != ConfidentialityClassification.Unclassified.value
             ):
                 ResourcePolicy.check_user_resource_permission(
                     session=session,

backend/dataall/modules/datasets_base/services/datasets_base_enums.py

@@ -1,4 +1,6 @@
 from dataall.base.api.constants import GraphQLEnumMapper
+from dataall.base.config import config
+custom_confidentiality_mapping = config.get_property('modules.datasets.features.custom_confidentiality_mapping', {})
 
 
 class DatasetRole(GraphQLEnumMapper):
@@ -22,6 +24,13 @@ class ConfidentialityClassification(GraphQLEnumMapper):
     Official = 'Official'
     Secret = 'Secret'
 
+    @staticmethod
+    def get_confidentiality_level(confidentiality, context):
+        if context.db_engine.dbconfig.schema == 'pytest':
+            return confidentiality
+        return confidentiality if not custom_confidentiality_mapping else custom_confidentiality_mapping.get(
+            confidentiality, None)
+
 
 class Language(GraphQLEnumMapper):
     English = 'English'

config.json

@@ -24,7 +24,9 @@
                     }
                 },
                 "preview_data": true,
-                "glue_crawler": true
+                "glue_crawler": true,
+                "confidentiality_dropdown" : true,
+                "topics_dropdown" : true
             }
         },
         "worksheets": {

frontend/src/modules/Catalog/views/Catalog.js

@@ -32,6 +32,7 @@
   useSettings
 } from 'design';
 import { GlossarySearchWrapper, GlossarySearchResultItem } from '../components';
+import config from '../../../generated/config.json';
 
 const useStyles = makeStyles((theme) => ({
   mainSearch: {
@@ -171,7 +172,14 @@
   const classes = useStyles();
   const anchorRef = useRef(null);
   const [openMenu, setOpenMenu] = useState(false);
-  const [filterItems] = useState([
+  const dataFieldList = ['label', 'name', 'description', 'region', 'tags'];
+
+  if (config.modules.datasets.features.topics_dropdown === true)
+    dataFieldList.push('topics');
+  if (config.modules.datasets.features.confidentiality_dropdown === true)
+    dataFieldList.push('classification');
+
+  const filterItemsInit = [
     {
       title: 'Type',
       dataField: 'resourceKind',
@@ -184,25 +192,30 @@
       componentId: 'TagSensor',
       filterLabel: 'Tags'
     },
-    {
-      title: 'Topics',
-      dataField: 'topics',
-      componentId: 'TopicSensor',
-      filterLabel: 'Topics'
-    },
     {
       title: 'Region',
       dataField: 'region',
       componentId: 'RegionSensor',
       filterLabel: 'Region'
-    },
-    {
+    }
+  ];
+
+  if (config.modules.datasets.features.topics_dropdown === true)
+    filterItemsInit.push({
+      title: 'Topics',
+      dataField: 'topics',
+      componentId: 'TopicSensor',
+      filterLabel: 'Topics'
+    });
+  if (config.modules.datasets.features.confidentiality_dropdown === true)
+    filterItemsInit.push({
       title: 'Classification',
       dataField: 'classification',
       componentId: 'ClassificationSensor',
       filterLabel: 'Classification'
-    }
-  ]);
+    });
+
+  const [filterItems] = useState(filterItemsInit);
   const [listClass, setListClass] = useState(
     settings.theme === THEMES.LIGHT
       ? classes.lightListSearch
@@ -337,14 +350,7 @@
                     fuzziness="AUTO"
                     componentId="SearchSensor"
                     filterLabel="text"
-                    dataField={[
-                      'label',
-                      'name',
-                      'description',
-                      'region',
-                      'topics',
-                      'tags'
-                    ]}
+                    dataField={dataFieldList}
                     placeholder="Search"
                   />
                 </Box>
@@ -440,4 +446,4 @@
   );
 };
 
 export default Catalog;

frontend/src/modules/Datasets/components/DatasetGovernance.js

@@ -9,6 +9,7 @@ import {
 } from '@mui/material';
 import PropTypes from 'prop-types';
 import { Label } from 'design';
+import { isFeatureEnabled } from 'utils';
 
 export const DatasetGovernance = (props) => {
   const { dataset } = props;
@@ -38,41 +39,36 @@ export const DatasetGovernance = (props) => {
           {dataset.stewards}
         </Typography>
       </CardContent>
-      <CardContent>
-        <Typography color="textSecondary" variant="subtitle2">
-          Auto-Approval
-        </Typography>
-        <Box sx={{ mt: 1 }}>
-          <Label color="primary">
-            {dataset.autoApprovalEnabled ? 'Enabled' : 'Disabled'}
-          </Label>
-        </Box>
-      </CardContent>
-      <CardContent>
-        <Typography color="textSecondary" variant="subtitle2">
-          Classification
-        </Typography>
-        <Box sx={{ mt: 1 }}>
-          <Label color="primary">{dataset.confidentiality}</Label>
-        </Box>
-      </CardContent>
-      <CardContent>
-        <Typography color="textSecondary" variant="subtitle2">
-          Topics
-        </Typography>
-        <Box sx={{ mt: 1 }}>
-          {dataset.topics &&
-            dataset.topics.length > 0 &&
-            dataset.topics.map((t) => (
-              <Chip
-                sx={{ mr: 0.5, mb: 0.5 }}
-                key={t}
-                label={t}
-                variant="outlined"
-              />
-            ))}
-        </Box>
-      </CardContent>
+      {isFeatureEnabled('datasets', 'confidentiality_dropdown') && (
+        <CardContent>
+          <Typography color="textSecondary" variant="subtitle2">
+            Classification
+          </Typography>
+          <Box sx={{ mt: 1 }}>
+            <Label color="primary">{dataset.confidentiality}</Label>
+          </Box>
+        </CardContent>
+      )}
+      {isFeatureEnabled('datasets', 'topics_dropdown') && (
+        <CardContent>
+          <Typography color="textSecondary" variant="subtitle2">
+            Topics
+          </Typography>
+          <Box sx={{ mt: 1 }}>
+            {dataset.topics &&
+              dataset.topics.length > 0 &&
+              dataset.topics.map((t) => (
+                <Chip
+                  sx={{ mr: 0.5, mb: 0.5 }}
+                  key={t}
+                  label={t}
+                  variant="outlined"
+                />
+              ))}
+          </Box>
+        </CardContent>
+      )}
+
       <CardContent>
         <Typography color="textSecondary" variant="subtitle2">
           Tags