feat(ingest): add ingestion source for SAP Analytics Cloud

datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts

@@ -83,7 +83,7 @@ import {
     PROJECT_NAME,
 } from './lookml';
 import { PRESTO, PRESTO_HOST_PORT, PRESTO_DATABASE, PRESTO_USERNAME, PRESTO_PASSWORD } from './presto';
-import { AZURE, BIGQUERY_BETA, CSV, DBT_CLOUD, MYSQL, OKTA, POWER_BI, UNITY_CATALOG, VERTICA } from '../constants';
+import { AZURE, BIGQUERY_BETA, CSV, DBT_CLOUD, MYSQL, OKTA, POWER_BI, SAC, UNITY_CATALOG, VERTICA } from '../constants';
 import { BIGQUERY_BETA_PROJECT_ID, DATASET_ALLOW, DATASET_DENY, PROJECT_ALLOW, PROJECT_DENY } from './bigqueryBeta';
 import { MYSQL_HOST_PORT, MYSQL_PASSWORD, MYSQL_USERNAME } from './mysql';
 import { MSSQL, MSSQL_DATABASE, MSSQL_HOST_PORT, MSSQL_PASSWORD, MSSQL_USERNAME } from './mssql';
@@ -171,6 +171,20 @@ import {
     USER_ALLOW,
     USER_DENY,
 } from './azure';
+import {
+    SAC_TENANT_URL,
+    SAC_TOKEN_URL,
+    SAC_CLIENT_ID,
+    SAC_CLIENT_SECRET,
+    INGEST_STORIES,
+    INGEST_APPLICATIONS,
+    RESOURCE_ID_ALLOW,
+    RESOURCE_ID_DENY,
+    RESOURCE_NAME_ALLOW,
+    RESOURCE_NAME_DENY,
+    FOLDER_ALLOW,
+    FOLDER_DENY,
+} from './sac';
 
 export enum RecipeSections {
     Connection = 0,
@@ -519,8 +533,29 @@ export const RECIPE_FIELDS: RecipeFields = {
         filterFields: [GROUP_ALLOW, GROUP_DENY, USER_ALLOW, USER_DENY],
         advancedFields: [AZURE_INGEST_USERS, AZURE_INGEST_GROUPS, STATEFUL_INGESTION_ENABLED, SKIP_USERS_WITHOUT_GROUP],
     },
+    [SAC]: {
+        fields: [SAC_TENANT_URL, SAC_TOKEN_URL, SAC_CLIENT_ID, SAC_CLIENT_SECRET],
+        filterFields: [
+            INGEST_STORIES,
+            INGEST_APPLICATIONS,
+            RESOURCE_ID_ALLOW,
+            RESOURCE_ID_DENY,
+            RESOURCE_NAME_ALLOW,
+            RESOURCE_NAME_DENY,
+            FOLDER_ALLOW,
+            FOLDER_DENY,
+        ],
+        advancedFields: [STATEFUL_INGESTION_ENABLED],
+    },
 };
 
 export const CONNECTORS_WITH_FORM = new Set(Object.keys(RECIPE_FIELDS));
 
-export const CONNECTORS_WITH_TEST_CONNECTION = new Set([SNOWFLAKE, LOOKER, BIGQUERY_BETA, BIGQUERY, UNITY_CATALOG]);
+export const CONNECTORS_WITH_TEST_CONNECTION = new Set([
+    SNOWFLAKE,
+    LOOKER,
+    BIGQUERY_BETA,
+    BIGQUERY,
+    UNITY_CATALOG,
+    SAC,
+]);

datahub-web-react/src/app/ingest/source/builder/RecipeForm/sac.ts

@@ -0,0 +1,161 @@
+import { RecipeField, FieldType, setListValuesOnRecipe } from './common';
+
+export const SAC_TENANT_URL: RecipeField = {
+    name: 'tenant_url',
+    label: 'Tenant URL',
+    tooltip: 'The URL of the SAP Analytics Cloud tenant.',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.tenant_url',
+    placeholder: 'https://company.eu10.sapanalytics.cloud',
+    required: true,
+    rules: null,
+};
+
+export const SAC_TOKEN_URL: RecipeField = {
+    name: 'token_url',
+    label: 'Token URL',
+    tooltip: 'The OAuth 2.0 Token Service URL.',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.token_url',
+    placeholder: 'https://company.eu10.hana.ondemand.com/oauth/token',
+    required: true,
+    rules: null,
+};
+
+export const SAC_CLIENT_ID: RecipeField = {
+    name: 'client_id',
+    label: 'Client ID',
+    tooltip: 'Client ID.',
+    type: FieldType.SECRET,
+    fieldPath: 'source.config.client_id',
+    placeholder: 'client_id',
+    required: true,
+    rules: null,
+};
+
+export const SAC_CLIENT_SECRET: RecipeField = {
+    name: 'client_secret',
+    label: 'Client Secret',
+    tooltip: 'Client Secret.',
+    type: FieldType.SECRET,
+    fieldPath: 'source.config.client_secret',
+    placeholder: 'client_secret',
+    required: true,
+    rules: null,
+};
+
+export const INGEST_STORIES: RecipeField = {
+    name: 'ingest_stories',
+    label: 'Ingest Stories',
+    tooltip: 'Whether stories should be ingested into DataHub.',
+    type: FieldType.BOOLEAN,
+    fieldPath: 'source.config.ingest_stories',
+    rules: null,
+    section: 'Stories and Applications',
+};
+
+export const INGEST_APPLICATIONS: RecipeField = {
+    name: 'ingest_applications',
+    label: 'Ingest Applications',
+    tooltip: 'Whether applications should be ingested into DataHub.',
+    type: FieldType.BOOLEAN,
+    fieldPath: 'source.config.ingest_applications',
+    rules: null,
+    section: 'Stories and Applications',
+};
+
+const resourceIdAllowFieldPath = 'source.config.resource_id_pattern.allow';
+export const RESOURCE_ID_ALLOW: RecipeField = {
+    name: 'resource_id_pattern.allow',
+    label: 'Resource Id Allow Patterns',
+    tooltip:
+        'Only include specific Stories and Applications by providing the id of the ressource, or a Regular Expression (REGEX). If not provided, all Stories and Applications will be included.',
+    type: FieldType.LIST,
+    buttonLabel: 'Add pattern',
+    fieldPath: resourceIdAllowFieldPath,
+    rules: null,
+    section: 'Stories and Applications',
+    placeholder: 'LXTH4JCE36EOYLU41PIINLYPU9XRYM26',
+    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
+        setListValuesOnRecipe(recipe, values, resourceIdAllowFieldPath),
+};
+
+const resourceIdDenyFieldPath = 'source.config.resource_id_pattern.deny';
+export const RESOURCE_ID_DENY: RecipeField = {
+    name: 'resource_id_pattern.deny',
+    label: 'Resource Id Deny Patterns',
+    tooltip:
+        'Exclude specific Stories and Applications by providing the id of the resource, or a Regular Expression (REGEX). If not provided, all Stories and Applications will be included. Deny patterns always take precendence over Allow patterns.',
+    type: FieldType.LIST,
+    buttonLabel: 'Add pattern',
+    fieldPath: resourceIdDenyFieldPath,
+    rules: null,
+    section: 'Stories and Applications',
+    placeholder: 'LXTH4JCE36EOYLU41PIINLYPU9XRYM26',
+    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
+        setListValuesOnRecipe(recipe, values, resourceIdDenyFieldPath),
+};
+
+const resourceNameAllowFieldPath = 'source.config.resource_id_pattern.allow';
+export const RESOURCE_NAME_ALLOW: RecipeField = {
+    name: 'resource_name_pattern.allow',
+    label: 'Resource Name Allow Patterns',
+    tooltip:
+        'Only include specific Stories and Applications by providing the name of the ressource, or a Regular Expression (REGEX). If not provided, all Stories and Applications will be included.',
+    type: FieldType.LIST,
+    buttonLabel: 'Add pattern',
+    fieldPath: resourceNameAllowFieldPath,
+    rules: null,
+    section: 'Stories and Applications',
+    placeholder: 'Name of the story',
+    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
+        setListValuesOnRecipe(recipe, values, resourceNameAllowFieldPath),
+};
+
+const resourceNameDenyFieldPath = 'source.config.resource_name_pattern.deny';
+export const RESOURCE_NAME_DENY: RecipeField = {
+    name: 'resource_name_pattern.deny',
+    label: 'Resource Name Deny Patterns',
+    tooltip:
+        'Exclude specific Stories and Applications by providing the name of the resource, or a Regular Expression (REGEX). If not provided, all Stories and Applications will be included. Deny patterns always take precendence over Allow patterns.',
+    type: FieldType.LIST,
+    buttonLabel: 'Add pattern',
+    fieldPath: resourceNameDenyFieldPath,
+    rules: null,
+    section: 'Stories and Applications',
+    placeholder: 'Name of the story',
+    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
+        setListValuesOnRecipe(recipe, values, resourceNameDenyFieldPath),
+};
+
+const folderAllowFieldPath = 'source.config.resource_id_pattern.allow';
+export const FOLDER_ALLOW: RecipeField = {
+    name: 'folder_pattern.allow',
+    label: 'Folder Allow Patterns',
+    tooltip:
+        'Only include specific Stories and Applications by providing the folder containing the resources, or a Regular Expression (REGEX). If not provided, all Stories and Applications will be included.',
+    type: FieldType.LIST,
+    buttonLabel: 'Add pattern',
+    fieldPath: folderAllowFieldPath,
+    rules: null,
+    section: 'Stories and Applications',
+    placeholder: 'Folder of the story',
+    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
+        setListValuesOnRecipe(recipe, values, folderAllowFieldPath),
+};
+
+const folderDenyFieldPath = 'source.config.folder_pattern.deny';
+export const FOLDER_DENY: RecipeField = {
+    name: 'folder_pattern.deny',
+    label: 'Folder Deny Patterns',
+    tooltip:
+        'Exclude specific Stories and Applications by providing the folder containing the resources, or a Regular Expression (REGEX). If not provided, all Stories and Applications will be included. Deny patterns always take precendence over Allow patterns.',
+    type: FieldType.LIST,
+    buttonLabel: 'Add pattern',
+    fieldPath: folderDenyFieldPath,
+    rules: null,
+    section: 'Stories and Applications',
+    placeholder: 'Folder of the story',
+    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
+        setListValuesOnRecipe(recipe, values, folderDenyFieldPath),
+};

datahub-web-react/src/app/ingest/source/builder/SelectTemplateStep.tsx

@@ -104,6 +104,18 @@ export const SelectTemplateStep = ({ state, updateState, goTo, cancel, ingestion
             source.name.toLocaleLowerCase().includes(searchFilter.toLocaleLowerCase()),
     );
 
+    filteredSources.sort((a, b) => {
+        if (a.name === 'custom') {
+            return 1;
+        }
+
+        if (b.name === 'custom') {
+            return -1;
+        }
+
+        return a.displayName.localeCompare(b.displayName);
+    });
+
     return (
         <Container>
             <Section>

datahub-web-react/src/app/ingest/source/builder/constants.ts

@@ -34,6 +34,7 @@ import fivetranLogo from '../../../../images/fivetranlogo.png';
 import csvLogo from '../../../../images/csv-logo.png';
 import qlikLogo from '../../../../images/qliklogo.png';
 import sigmaLogo from '../../../../images/sigmalogo.png';
+import sacLogo from '../../../../images/saclogo.svg';
 
 export const ATHENA = 'athena';
 export const ATHENA_URN = `urn:li:dataPlatform:${ATHENA}`;
@@ -122,6 +123,8 @@ export const QLIK_SENSE = 'qlik-sense';
 export const QLIK_SENSE_URN = `urn:li:dataPlatform:${QLIK_SENSE}`;
 export const SIGMA = 'sigma';
 export const SIGMA_URN = `urn:li:dataPlatform:${SIGMA}`;
+export const SAC = 'sac';
+export const SAC_URN = `urn:li:dataPlatform:${SAC}`;
 
 export const PLATFORM_URN_TO_LOGO = {
     [ATHENA_URN]: athenaLogo,
@@ -161,6 +164,7 @@ export const PLATFORM_URN_TO_LOGO = {
     [CSV_URN]: csvLogo,
     [QLIK_SENSE_URN]: qlikLogo,
     [SIGMA_URN]: sigmaLogo,
+    [SAC_URN]: sacLogo,
 };
 
 export const SOURCE_TO_PLATFORM_URN = {

datahub-web-react/src/app/ingest/source/builder/sources.json

@@ -287,6 +287,14 @@
         "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/csv'",
         "recipe": "source: \n    type: csv-enricher \n    config: \n        # URL of your csv file to ingest \n        filename: \n        array_delimiter: '|' \n        delimiter: ',' \n        write_semantics: PATCH"
     },
+    {
+        "urn": "urn:li:dataPlatform:sac",
+        "name": "sac",
+        "displayName": "SAP Analytics Cloud",
+        "description": "Import Stories, Applications and Models from SAP Analytics Cloud.",
+        "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/sac/",
+        "recipe": "source:\n    type: sac\n    config:\n        tenant_url: # Your SAP Analytics Cloud tenant URL, e.g. https://company.eu10.sapanalytics.cloud or https://company.eu10.hcs.cloud.sap\n        token_url: # The Token URL of your SAP Analytics Cloud tenant, e.g. https://company.eu10.hana.ondemand.com/oauth/token.\n\n        # Add secret in Secrets Tab with relevant names for each variable\n        client_id: \"${SAC_CLIENT_ID}\" # Your SAP Analytics Cloud client id\n        client_secret: \"${SAC_CLIENT_SECRET}\" # Your SAP Analytics Cloud client secret"
+    },
     {
         "urn": "urn:li:dataPlatform:custom",
         "name": "custom",

datahub-web-react/src/app/ingest/source/conf/sac/sac.ts

@@ -0,0 +1,26 @@
+import { SourceConfig } from '../types';
+import sacLogo from '../../../../../images/saclogo.svg';
+
+const placeholderRecipe = `\
+source:
+    type: sac
+    config:
+        tenant_url: # Your SAP Analytics Cloud tenant URL, e.g. https://company.eu10.sapanalytics.cloud or https://company.eu10.hcs.cloud.sap
+        token_url: # The Token URL of your SAP Analytics Cloud tenant, e.g. https://company.eu10.hana.ondemand.com/oauth/token.
+
+        # Add secret in Secrets Tab with relevant names for each variable
+        client_id: "\${SAC_CLIENT_ID}" # Your SAP Analytics Cloud client id
+        client_secret: "\${SAC_CLIENT_SECRET}" # Your SAP Analytics Cloud client secret
+`;
+
+export const SAC = 'sac';
+
+const sacConfig: SourceConfig = {
+    type: SAC,
+    placeholderRecipe,
+    displayName: 'SAP Analytics Cloud',
+    docsUrl: 'https://datahubproject.io/docs/generated/ingestion/sources/sac/',
+    logoUrl: sacLogo,
+};
+
+export default sacConfig;

datahub-web-react/src/app/ingest/source/conf/sources.tsx

@@ -17,6 +17,7 @@ import hiveConfig from './hive/hive';
 import oracleConfig from './oracle/oracle';
 import tableauConfig from './tableau/tableau';
 import csvConfig from './csv/csv';
+import sacConfig from './sac/sac';
 
 const baseUrl = window.location.origin;
 
@@ -48,6 +49,7 @@ export const SOURCE_TEMPLATE_CONFIGS: Array<SourceConfig> = [
     oracleConfig,
     hiveConfig,
     csvConfig,
+    sacConfig,
     {
         type: 'custom',
         placeholderRecipe: DEFAULT_PLACEHOLDER_RECIPE,