feat(ui/ingest): ingestion form for Okta and AzureAD

datahub-web-react/src/app/ingest/source/builder/RecipeForm/azure.ts

@@ -0,0 +1,174 @@
+import { RecipeField, FieldType, setListValuesOnRecipe } from './common';
+
+const validateURL = (fieldName) => {
+    return {
+        validator(_, value) {
+            const URLPattern = new RegExp(/^(?:http(s)?:\/\/)?[\w.-]+(?:\.[\w.-]+)+[\w\-._~:/?#[\]@!$&'()*+,;=.]+$/);
+            const isURLValid = URLPattern.test(value);
+            if (!value || isURLValid) {
+                return Promise.resolve();
+            }
+            return Promise.reject(new Error(`A valid ${fieldName} is required.`));
+        },
+    };
+};
+
+export const AZURE_CLIENT_ID: RecipeField = {
+    name: 'client_id',
+    label: 'Client ID',
+    tooltip: 'Application ID. Found in your app registration on Azure AD Portal',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.client_id',
+    placeholder: '00000000-0000-0000-0000-000000000000',
+    required: true,
+    rules: null,
+};
+
+export const AZURE_TENANT_ID: RecipeField = {
+    name: 'tenant_id',
+    label: 'Tenant ID',
+    tooltip: 'Directory ID. Found in your app registration on Azure AD Portal',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.tenant_id',
+    placeholder: '00000000-0000-0000-0000-000000000000',
+    required: true,
+    rules: null,
+};
+
+export const AZURE_CLIENT_SECRET: RecipeField = {
+    name: 'client_secret',
+    label: 'Client Secret',
+    tooltip: 'The Azure client secret.',
+    type: FieldType.SECRET,
+    fieldPath: 'source.config.client_secret',
+    placeholder: '00000000-0000-0000-0000-000000000000',
+    required: true,
+    rules: null,
+};
+
+export const AZURE_REDIRECT_URL: RecipeField = {
+    name: 'redirect',
+    label: 'Redirect URL',
+    tooltip: 'Redirect URL. Found in your app registration on Azure AD Portal.',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.redirect',
+    placeholder: 'https://login.microsoftonline.com/common/oauth2/nativeclient',
+    required: true,
+    rules: [() => validateURL('Redirect URI')],
+};
+
+export const AZURE_AUTHORITY_URL: RecipeField = {
+    name: 'authority',
+    label: 'Authority URL',
+    tooltip: 'Is a URL that indicates a directory that MSAL can request tokens from..',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.authority',
+    placeholder: 'https://login.microsoftonline.com/00000000-0000-0000-0000-000000000000',
+    required: true,
+    rules: [() => validateURL('Azure authority URL')],
+};
+
+export const AZURE_TOKEN_URL: RecipeField = {
+    name: 'token_url',
+    label: 'Token URL',
+    tooltip:
+        'The token URL that acquires a token from Azure AD for authorizing requests. This source will only work with v1.0 endpoint.',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.token_url',
+    placeholder: 'https://login.microsoftonline.com/00000000-0000-0000-0000-000000000000/oauth2/token',
+    required: true,
+    rules: [() => validateURL('Azure token URL')],
+};
+
+export const AZURE_GRAPH_URL: RecipeField = {
+    name: 'graph_url',
+    label: 'Graph URL',
+    tooltip: 'Microsoft Graph API endpoint',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.graph_url',
+    placeholder: 'https://graph.microsoft.com/v1.0',
+    required: true,
+    rules: [() => validateURL('Graph url URL')],
+};
+
+export const AZURE_INGEST_USERS: RecipeField = {
+    name: 'ingest_users',
+    label: 'Ingest Users',
+    tooltip: 'Flag to determine whether to ingest users from Azure AD or not.',
+    type: FieldType.BOOLEAN,
+    fieldPath: 'source.config.ingest_users',
+    rules: null,
+};
+
+export const AZURE_INGEST_GROUPS: RecipeField = {
+    name: 'ingest_groups',
+    label: 'Ingest Groups',
+    tooltip: 'Flag to determine whether to ingest groups from Azure AD or not.',
+    type: FieldType.BOOLEAN,
+    fieldPath: 'source.config.ingest_groups',
+    rules: null,
+};
+
+const schemaAllowFieldPathGroup = 'source.config.groups_pattern.allow';
+export const GROUP_ALLOW: RecipeField = {
+    name: 'groups.allow',
+    label: 'Allow Patterns',
+    tooltip:
+        'Only include specific schemas by providing the name of a schema, or a regular expression (regex) to include specific schemas. If not provided, all schemas inside allowed databases will be included.',
+    placeholder: 'group_pattern',
+    type: FieldType.LIST,
+    buttonLabel: 'Add pattern',
+    fieldPath: schemaAllowFieldPathGroup,
+    rules: null,
+    section: 'Group',
+    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
+        setListValuesOnRecipe(recipe, values, schemaAllowFieldPathGroup),
+};
+
+const schemaDenyFieldPathGroup = 'source.config.groups_pattern.deny';
+export const GROUP_DENY: RecipeField = {
+    name: 'groups.deny',
+    label: 'Deny Patterns',
+    tooltip:
+        'Exclude specific schemas by providing the name of a schema, or a regular expression (regex). If not provided, all schemas inside allowed databases will be included. Deny patterns always take precedence over allow patterns.',
+    placeholder: 'user_pattern',
+    type: FieldType.LIST,
+    buttonLabel: 'Add pattern',
+    fieldPath: schemaDenyFieldPathGroup,
+    rules: null,
+    section: 'Group',
+    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
+        setListValuesOnRecipe(recipe, values, schemaDenyFieldPathGroup),
+};
+
+const schemaAllowFieldPathUser = 'source.config.users_pattern.allow';
+export const USER_ALLOW: RecipeField = {
+    name: 'user.allow',
+    label: 'Allow Patterns',
+    tooltip:
+        'Exclude specific schemas by providing the name of a schema, or a regular expression (regex). If not provided, all schemas inside allowed databases will be included. Deny patterns always take precedence over allow patterns.',
+    placeholder: 'user_pattern',
+    type: FieldType.LIST,
+    buttonLabel: 'Add pattern',
+    fieldPath: schemaAllowFieldPathUser,
+    rules: null,
+    section: 'User',
+    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
+        setListValuesOnRecipe(recipe, values, schemaAllowFieldPathUser),
+};
+
+const schemaDenyFieldPathUser = 'source.config.users_pattern.deny';
+export const USER_DENY: RecipeField = {
+    name: 'user.deny',
+    label: 'Deny Patterns',
+    tooltip:
+        'Exclude specific schemas by providing the name of a schema, or a regular expression (regex). If not provided, all schemas inside allowed databases will be included. Deny patterns always take precedence over allow patterns.',
+    placeholder: 'user_pattern',
+    type: FieldType.LIST,
+    buttonLabel: 'Add pattern',
+    fieldPath: schemaDenyFieldPathUser,
+    rules: null,
+    section: 'User',
+    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
+        setListValuesOnRecipe(recipe, values, schemaDenyFieldPathUser),
+};

datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts

@@ -83,7 +83,7 @@ import {
     PROJECT_NAME,
 } from './lookml';
 import { PRESTO, PRESTO_HOST_PORT, PRESTO_DATABASE, PRESTO_USERNAME, PRESTO_PASSWORD } from './presto';
-import { BIGQUERY_BETA, CSV, DBT_CLOUD, MYSQL, POWER_BI, UNITY_CATALOG, VERTICA } from '../constants';
+import { AZURE, BIGQUERY_BETA, CSV, DBT_CLOUD, MYSQL, OKTA, POWER_BI, UNITY_CATALOG, VERTICA } from '../constants';
 import { BIGQUERY_BETA_PROJECT_ID, DATASET_ALLOW, DATASET_DENY, PROJECT_ALLOW, PROJECT_DENY } from './bigqueryBeta';
 import { MYSQL_HOST_PORT, MYSQL_PASSWORD, MYSQL_USERNAME } from './mysql';
 import { MSSQL, MSSQL_DATABASE, MSSQL_HOST_PORT, MSSQL_PASSWORD, MSSQL_USERNAME } from './mssql';
@@ -141,6 +141,8 @@ import {
     INCLUDE_PROJECTIONS_LINEAGE,
 } from './vertica';
 import { CSV_ARRAY_DELIMITER, CSV_DELIMITER, CSV_FILE_URL, CSV_WRITE_SEMANTICS } from './csv';
+import { INCLUDE_DEPROVISIONED_USERS, INCLUDE_SUSPENDED_USERS, INGEST_GROUPS, INGEST_USERS, OKTA_API_TOKEN, OKTA_DOMAIN_URL, POFILE_TO_GROUP, POFILE_TO_GROUP_REGX_ALLOW, POFILE_TO_GROUP_REGX_DENY, POFILE_TO_USER, POFILE_TO_USER_REGX_ALLOW, POFILE_TO_USER_REGX_DENY } from './okta';
+import { AZURE_AUTHORITY_URL, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_GRAPH_URL, AZURE_INGEST_GROUPS, AZURE_INGEST_USERS, AZURE_REDIRECT_URL, AZURE_TENANT_ID, AZURE_TOKEN_URL, GROUP_ALLOW, GROUP_DENY, USER_ALLOW, USER_DENY } from './azure';
 
 export enum RecipeSections {
     Connection = 0,
@@ -459,6 +461,16 @@ export const RECIPE_FIELDS: RecipeFields = {
         filterFields: [],
         advancedFields: [CSV_ARRAY_DELIMITER, CSV_DELIMITER, CSV_WRITE_SEMANTICS],
     },
+    [OKTA]: {
+        fields: [OKTA_DOMAIN_URL,OKTA_API_TOKEN,POFILE_TO_USER,POFILE_TO_GROUP],
+        filterFields: [POFILE_TO_USER_REGX_ALLOW,POFILE_TO_USER_REGX_DENY,POFILE_TO_GROUP_REGX_ALLOW,POFILE_TO_GROUP_REGX_DENY],
+        advancedFields: [INGEST_USERS,INGEST_GROUPS,INCLUDE_DEPROVISIONED_USERS,INCLUDE_SUSPENDED_USERS],
+    },
+    [AZURE]: {
+        fields: [AZURE_CLIENT_ID,AZURE_TENANT_ID,AZURE_CLIENT_SECRET,AZURE_REDIRECT_URL,AZURE_AUTHORITY_URL,AZURE_TOKEN_URL,AZURE_GRAPH_URL],
+        filterFields: [GROUP_ALLOW,GROUP_DENY,USER_ALLOW,USER_DENY],
+        advancedFields: [AZURE_INGEST_USERS,AZURE_INGEST_GROUPS],
+    },
 };
 
 export const CONNECTORS_WITH_FORM = new Set(Object.keys(RECIPE_FIELDS));

datahub-web-react/src/app/ingest/source/builder/RecipeForm/okta.ts

@@ -0,0 +1,162 @@
+import { RecipeField, FieldType, setListValuesOnRecipe } from './common';
+
+const validateURL = (fieldName) => {
+    return {
+        validator(_, value) {
+            const URLPattern = new RegExp(/^(?:http(s)?:\/\/)?[\w.-]+(?:\.[\w.-]+)+[\w\-._~:/?#[\]@!$&'()*+,;=.]+$/);
+            const isURLValid = URLPattern.test(value);
+            if (!value || isURLValid) {
+                return Promise.resolve();
+            }
+            return Promise.reject(new Error(`A valid ${fieldName} is required.`));
+        },
+    };
+};
+
+export const OKTA_DOMAIN_URL: RecipeField = {
+    name: 'okta_domain',
+    label: 'Okta Domain URL',
+    tooltip: 'The location of your Okta Domain, without a protocol.',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.okta_domain',
+    placeholder: 'dev-35531955.okta.com',
+    required: true,
+    rules: [() => validateURL('Okta Domain URL')],
+};
+
+export const OKTA_API_TOKEN: RecipeField = {
+    name: 'credential.project_id',
+    label: 'Token',
+    tooltip: 'An API token generated for the DataHub application inside your Okta Developer Console.',
+    type: FieldType.SECRET,
+    fieldPath: 'source.config.okta_api_token',
+    placeholder: 'd0121d0000882411234e11166c6aaa23ed5d74e0',
+    rules: null,
+    required: true,
+};
+
+export const POFILE_TO_USER: RecipeField = {
+    name: 'okta_profile_to_username_attr',
+    label: 'Okta Profile to Username attribute',
+    tooltip: 'Which Okta User Profile attribute to use as input to DataHub username mapping. Common values used are - login, email.',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.okta_profile_to_username_attr',
+    placeholder: 'usename',
+    rules: null,
+};
+
+export const POFILE_TO_GROUP: RecipeField = {
+    name: 'okta_profile_to_group_name_attr',
+    label: 'Okta Profile to group name attribute',
+    tooltip: 'Which Okta Group Profile attribute to use as input to DataHub group name mapping.',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.okta_profile_to_group_name_attr',
+    placeholder: 'Group name',
+    rules: null,
+};
+
+
+const schemaAllowFieldPath = 'source.config.okta_profile_to_username_attr_regex.allow';
+export const POFILE_TO_USER_REGX_ALLOW: RecipeField = {
+    name: 'user.allow',
+    label: 'Allow Patterns',
+    tooltip:
+        'Only include specific schemas by providing the name of a schema, or a regular expression (regex) to include specific schemas. If not provided, all schemas inside allowed databases will be included.',
+    placeholder: 'user_pattern',
+    type: FieldType.LIST,
+    buttonLabel: 'Add pattern',
+    fieldPath: schemaAllowFieldPath,
+    rules: null,
+    section: 'Profile To User Attribute',
+    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
+        setListValuesOnRecipe(recipe, values, schemaAllowFieldPath),
+};
+
+
+const schemaDenyFieldPath = 'source.config.okta_profile_to_username_attr_regex.deny';
+export const POFILE_TO_USER_REGX_DENY: RecipeField = {
+    name: 'user.deny',
+    label: 'Deny Patterns',
+    tooltip:
+        'Only include specific schemas by providing the name of a schema, or a regular expression (regex) to include specific schemas. If not provided, all schemas inside allowed databases will be included.',
+    placeholder: 'user_pattern',
+    type: FieldType.LIST,
+    buttonLabel: 'Add pattern',
+    fieldPath: schemaDenyFieldPath,
+    rules: null,
+    section: 'Profile To User Attribute',
+    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
+        setListValuesOnRecipe(recipe, values, schemaDenyFieldPath),
+};
+
+
+const schemaAllowFieldPathForGroup = 'source.config.okta_profile_to_group_name_regex.allow';
+export const POFILE_TO_GROUP_REGX_ALLOW: RecipeField = {
+    name: 'group.allow',
+    label: 'Allow Patterns',
+    tooltip:
+        'Only include specific schemas by providing the name of a schema, or a regular expression (regex) to include specific schemas. If not provided, all schemas inside allowed databases will be included.',
+    placeholder: 'group_pattern',
+    type: FieldType.LIST,
+    buttonLabel: 'Add pattern',
+    fieldPath: schemaAllowFieldPathForGroup,
+    rules: null,
+    section: 'Profile To Group Attribute',
+    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
+        setListValuesOnRecipe(recipe, values, schemaAllowFieldPathForGroup),
+};
+
+const schemaDenyFieldPathForGroup = 'source.config.okta_profile_to_group_name_regex.deny';
+export const POFILE_TO_GROUP_REGX_DENY: RecipeField = {
+    name: 'group.deny',
+    label: 'Deny Patterns',
+    tooltip:
+        'Only include specific schemas by providing the name of a schema, or a regular expression (regex) to include specific schemas. If not provided, all schemas inside allowed databases will be included.',
+    placeholder: 'group_pattern',
+    type: FieldType.LIST,
+    buttonLabel: 'Add pattern',
+    fieldPath: schemaDenyFieldPathForGroup,
+    rules: null,
+    section: 'Profile To Group Attribute',
+    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
+        setListValuesOnRecipe(recipe, values, schemaDenyFieldPathForGroup),
+};
+
+
+
+export const INGEST_USERS: RecipeField = {
+    name: 'ingest_users',
+    label: 'Ingest Users',
+    tooltip: 'Whether users should be ingested into DataHub.',
+    type: FieldType.BOOLEAN,
+    fieldPath: 'source.config.ingest_users',
+    rules: null,
+};
+
+export const INGEST_GROUPS: RecipeField = {
+    name: 'ingest_groups',
+    label: 'Ingest Groups',
+    tooltip: 'Whether groups should be ingested into DataHub.',
+    type: FieldType.BOOLEAN,
+    fieldPath: 'source.config.ingest_groups',
+    rules: null,
+};
+
+
+export const INCLUDE_DEPROVISIONED_USERS: RecipeField = {
+    name: 'include_deprovisioned_users',
+    label: 'Include deprovisioned users',
+    tooltip: 'Whether to ingest users in the DEPROVISIONED state from Okta.',
+    type: FieldType.BOOLEAN,
+    fieldPath: 'source.config.include_deprovisioned_users',
+    rules: null,
+};
+export const INCLUDE_SUSPENDED_USERS: RecipeField = {
+    name: 'include_suspended_users',
+    label: 'Include suspended users',
+    tooltip: 'Whether to ingest users in the SUSPENDED state from Okta.',
+    type: FieldType.BOOLEAN,
+    fieldPath: 'source.config.include_suspended_users',
+    rules: null,
+};
+