diff --git a/datahub-web-react/src/app/ingest/source/builder/DefineRecipeStep.tsx b/datahub-web-react/src/app/ingest/source/builder/DefineRecipeStep.tsx index 9182c110123eea..08ab29b3172d6b 100644 --- a/datahub-web-react/src/app/ingest/source/builder/DefineRecipeStep.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/DefineRecipeStep.tsx @@ -6,6 +6,8 @@ import { getSourceConfigs, jsonToYaml, yamlToJson } from '../utils'; import { YamlEditor } from './YamlEditor'; import { ANTD_GRAY } from '../../../entity/shared/constants'; import { IngestionSourceBuilderStep } from './steps'; +import RecipeBuilder from './RecipeBuilder'; +import { CONNECTORS_WITH_FORM } from './RecipeForm/utils'; const LOOKML_DOC_LINK = 'https://datahubproject.io/docs/generated/ingestion/sources/looker#module-lookml'; @@ -37,17 +39,19 @@ const ControlsContainer = styled.div` export const DefineRecipeStep = ({ state, updateState, goTo, prev }: StepProps) => { const existingRecipeJson = state.config?.recipe; const existingRecipeYaml = existingRecipeJson && jsonToYaml(existingRecipeJson); + const { type } = state; + const sourceConfigs = getSourceConfigs(type as string); - const [stagedRecipeYml, setStagedRecipeYml] = useState(existingRecipeYaml || ''); + const [stagedRecipeYml, setStagedRecipeYml] = useState(existingRecipeYaml || sourceConfigs.placeholderRecipe); useEffect(() => { - setStagedRecipeYml(existingRecipeYaml || ''); + if (existingRecipeYaml) { + setStagedRecipeYml(existingRecipeYaml); + } }, [existingRecipeYaml]); const [stepComplete, setStepComplete] = useState(false); - const { type } = state; - const sourceConfigs = getSourceConfigs(type as string); const isEditing: boolean = prev === undefined; const displayRecipe = stagedRecipeYml || sourceConfigs.placeholderRecipe; const sourceDisplayName = sourceConfigs.displayName; @@ -85,6 +89,19 @@ export const DefineRecipeStep = ({ state, updateState, goTo, prev }: StepProps) goTo(IngestionSourceBuilderStep.CREATE_SCHEDULE); }; + if (type && CONNECTORS_WITH_FORM.has(type)) { + return ( + + ); + } + return ( <>
diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeBuilder.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeBuilder.tsx new file mode 100644 index 00000000000000..daf8b9bbed8085 --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeBuilder.tsx @@ -0,0 +1,103 @@ +import { Button, message } from 'antd'; +import React, { useState } from 'react'; +import YAML from 'yamljs'; +import { CodeOutlined, FormOutlined } from '@ant-design/icons'; +import styled from 'styled-components/macro'; +import { ANTD_GRAY } from '../../../entity/shared/constants'; +import { YamlEditor } from './YamlEditor'; +import RecipeForm from './RecipeForm/RecipeForm'; + +export const ControlsContainer = styled.div` + display: flex; + justify-content: space-between; + margin-top: 8px; +`; + +const BorderedSection = styled.div` + display: flex; + flex-direction: column; + padding-bottom: 16px; + border: solid ${ANTD_GRAY[4]} 0.5px; +`; + +const StyledButton = styled(Button)<{ isSelected: boolean }>` + ${(props) => + props.isSelected && + ` + color: #1890ff; + &:focus { + color: #1890ff; + } + `} +`; + +const ButtonsWrapper = styled.div` + display: flex; + justify-content: flex-end; + margin-bottom: 10px; +`; + +interface Props { + type: string; + isEditing: boolean; + displayRecipe: string; + setStagedRecipe: (recipe: string) => void; + onClickNext: () => void; + goToPrevious?: () => void; +} + +function RecipeBuilder(props: Props) { + const { type, isEditing, displayRecipe, setStagedRecipe, onClickNext, goToPrevious } = props; + + const [isViewingForm, setIsViewingForm] = useState(true); + + function switchViews(isFormView: boolean) { + try { + YAML.parse(displayRecipe); + setIsViewingForm(isFormView); + } catch (e) { + const messageText = (e as any).parsedLine + ? `Fix line ${(e as any).parsedLine} in your recipe` + : 'Please fix your recipe'; + message.warn(`Found invalid YAML. ${messageText} in order to switch views.`); + } + } + + return ( +
+ + switchViews(true)}> + Form + + switchViews(false)}> + YAML + + + {isViewingForm && ( + + )} + {!isViewingForm && ( + <> + + + + + + + + + )} +
+ ); +} + +export default RecipeBuilder; diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/FormField.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/FormField.tsx new file mode 100644 index 00000000000000..7e3d601726a67d --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/FormField.tsx @@ -0,0 +1,115 @@ +import { Button, Checkbox, Form, Input, Tooltip } from 'antd'; +import React from 'react'; +import styled from 'styled-components/macro'; +import { MinusCircleOutlined, PlusOutlined, QuestionCircleOutlined } from '@ant-design/icons'; +import { FieldType, RecipeField } from './utils'; +import { ANTD_GRAY } from '../../../../entity/shared/constants'; + +const Label = styled.div` + font-weight: bold; + padding-bottom: 8px; +`; + +const StyledButton = styled(Button)` + color: ${ANTD_GRAY[7]}; + width: 80%; +`; + +const StyledQuestion = styled(QuestionCircleOutlined)` + color: rgba(0, 0, 0, 0.45); + margin-left: 4px; +`; + +const StyledRemoveIcon = styled(MinusCircleOutlined)` + font-size: 14px; + margin-left: 10px; +`; + +const StyledFormItem = styled(Form.Item)<{ alignLeft: boolean; removeMargin: boolean }>` + margin-bottom: ${(props) => (props.removeMargin ? '0' : '16px')}; + + ${(props) => + props.alignLeft && + ` + .ant-form-item { + flex-direction: row; + + } + + .ant-form-item-label { + padding: 0; + margin-right: 10px; + } + `} +`; + +const ListWrapper = styled.div<{ removeMargin: boolean }>` + margin-bottom: ${(props) => (props.removeMargin ? '0' : '16px')}; +`; + +interface ListFieldProps { + field: RecipeField; + removeMargin?: boolean; +} + +function ListField({ field, removeMargin }: ListFieldProps) { + return ( + + {(fields, { add, remove }) => ( + + + {fields.map((item) => ( + + + + + remove(item.name)} /> + + ))} + add()} style={{ width: '80%' }} icon={}> + Add pattern + + + )} + + ); +} + +interface Props { + field: RecipeField; + removeMargin?: boolean; +} + +function FormField(props: Props) { + const { field, removeMargin } = props; + + if (field.type === FieldType.LIST) return ; + + const isBoolean = field.type === FieldType.BOOLEAN; + const input = isBoolean ? : ; + const valuePropName = isBoolean ? 'checked' : 'value'; + const getValueFromEvent = isBoolean ? undefined : (e) => (e.target.value === '' ? null : e.target.value); + + return ( + + {input} + + ); +} + +export default FormField; diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/RecipeForm.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/RecipeForm.tsx new file mode 100644 index 00000000000000..87bd26583eb685 --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/RecipeForm.tsx @@ -0,0 +1,148 @@ +import { Button, Collapse, Form, message, Typography } from 'antd'; +import React from 'react'; +import { get } from 'lodash'; +import YAML from 'yamljs'; +import { ApiOutlined, FilterOutlined, SettingOutlined } from '@ant-design/icons'; +import styled from 'styled-components/macro'; +import { jsonToYaml } from '../../utils'; +import { RecipeField, RECIPE_FIELDS, setFieldValueOnRecipe } from './utils'; +import FormField from './FormField'; + +export const ControlsContainer = styled.div` + display: flex; + justify-content: space-between; + margin-top: 12px; +`; + +const StyledCollapse = styled(Collapse)` + margin-bottom: 16px; + + .ant-collapse-header { + font-size: 14px; + font-weight: bold; + padding: 12px 0; + } +`; + +const HeaderTitle = styled.span` + margin-left: 8px; +`; + +const MarginWrapper = styled.div` + margin-left: 20px; +`; + +function getInitialValues(displayRecipe: string, allFields: any[]) { + const initialValues = {}; + let recipeObj; + try { + recipeObj = YAML.parse(displayRecipe); + } catch (e) { + message.warn('Found invalid YAML. Please check your recipe configuration.'); + return {}; + } + if (recipeObj) { + allFields.forEach((field) => { + initialValues[field.name] = + field.getValueFromRecipeOverride?.(recipeObj) || get(recipeObj, field.fieldPath); + }); + } + + return initialValues; +} + +function SectionHeader({ icon, text }: { icon: any; text: string }) { + return ( + + {icon} + {text} + + ); +} + +function shouldRenderFilterSectionHeader(field: RecipeField, index: number, filterFields: RecipeField[]) { + if (index === 0 && field.section) return true; + if (field.section && filterFields[index - 1].section !== field.section) return true; + return false; +} + +interface Props { + type: string; + isEditing: boolean; + displayRecipe: string; + setStagedRecipe: (recipe: string) => void; + onClickNext: () => void; + goToPrevious?: () => void; +} + +function RecipeForm(props: Props) { + const { type, isEditing, displayRecipe, setStagedRecipe, onClickNext, goToPrevious } = props; + const { fields, advancedFields, filterFields } = RECIPE_FIELDS[type]; + const allFields = [...fields, ...advancedFields, ...filterFields]; + + function updateFormValues(changedValues: any) { + let updatedValues = YAML.parse(displayRecipe); + + Object.keys(changedValues).forEach((fieldName) => { + const recipeField = allFields.find((f) => f.name === fieldName); + if (recipeField) { + updatedValues = + recipeField.setValueOnRecipeOverride?.(updatedValues, changedValues[fieldName]) || + setFieldValueOnRecipe(updatedValues, changedValues[fieldName], recipeField.fieldPath); + } + }); + + const stagedRecipe = jsonToYaml(JSON.stringify(updatedValues)); + setStagedRecipe(stagedRecipe); + } + + return ( +
+ + } text="Connection" />} key="0"> + {fields.map((field, i) => ( + + ))} + + + + } text="Filter" />} key="1"> + {filterFields.map((field, i) => ( + <> + {shouldRenderFilterSectionHeader(field, i, filterFields) && ( + {field.section} + )} + + + + + ))} + + + + } text="Advanced" />} + key="2" + > + {advancedFields.map((field, i) => ( + + ))} + + + + + + +
+ ); +} + +export default RecipeForm; diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/__tests__/utils.test.ts b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/__tests__/utils.test.ts new file mode 100644 index 00000000000000..64604ea350d283 --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/__tests__/utils.test.ts @@ -0,0 +1,120 @@ +import { setFieldValueOnRecipe, setListValuesOnRecipe } from '../utils'; + +describe('setFieldValueOnRecipe', () => { + const accountIdFieldPath = 'source.config.account_id'; + const profilingEnabledFieldPath = 'source.config.profiling.enabled'; + + it('should set the field value on a recipe object when it was not defined', () => { + const recipe = { source: { config: {} } }; + const updatedRecipe = setFieldValueOnRecipe(recipe, 'test', accountIdFieldPath); + expect(updatedRecipe).toMatchObject({ source: { config: { account_id: 'test' } } }); + }); + + it('should update the field value on a recipe object when it was defined', () => { + const recipe = { source: { config: { account_id: 'test' } } }; + const updatedRecipe = setFieldValueOnRecipe(recipe, 'edited!', accountIdFieldPath); + expect(updatedRecipe).toMatchObject({ source: { config: { account_id: 'edited!' } } }); + }); + + it('should update the field value on a recipe without changing any other fields', () => { + const recipe = { source: { config: { test: 'test' } } }; + const updatedRecipe = setFieldValueOnRecipe(recipe, 'edited!', accountIdFieldPath); + expect(updatedRecipe).toMatchObject({ source: { config: { test: 'test', account_id: 'edited!' } } }); + }); + + it('should clear the key: value pair when passing in null', () => { + const recipe = { source: { config: { existingField: true, account_id: 'test' } } }; + const updatedRecipe = setFieldValueOnRecipe(recipe, null, accountIdFieldPath); + expect(updatedRecipe).toMatchObject({ source: { config: { existingField: true } } }); + }); + + it('should return the original recipe when passing in undefined', () => { + const recipe = { source: { config: { test: 'test' } } }; + const updatedRecipe = setFieldValueOnRecipe(recipe, undefined, accountIdFieldPath); + expect(updatedRecipe).toMatchObject(recipe); + }); + + it('should set the field value on a recipe object when it was not defined and has a parent', () => { + const recipe = { source: { config: {} } }; + const updatedRecipe = setFieldValueOnRecipe(recipe, true, profilingEnabledFieldPath); + expect(updatedRecipe).toMatchObject({ source: { config: { profiling: { enabled: true } } } }); + }); + + it('should update the field value on a recipe object when it was defined and has a parent', () => { + const recipe = { source: { config: { profiling: { enabled: true } } } }; + const updatedRecipe = setFieldValueOnRecipe(recipe, false, profilingEnabledFieldPath); + expect(updatedRecipe).toMatchObject({ source: { config: { profiling: { enabled: false } } } }); + }); + + it('should update the field value with a parent on a recipe without changing any other fields', () => { + const recipe = { source: { config: { test: 'test' } } }; + const updatedRecipe = setFieldValueOnRecipe(recipe, false, profilingEnabledFieldPath); + expect(updatedRecipe).toMatchObject({ source: { config: { test: 'test', profiling: { enabled: false } } } }); + }); + + it('should clear the field and its parent when passing in null and field is only child of parent', () => { + const recipe = { source: { config: { test: 'test', profiling: { enabled: true } } } }; + const updatedRecipe = setFieldValueOnRecipe(recipe, null, profilingEnabledFieldPath); + expect(updatedRecipe).toMatchObject({ source: { config: { test: 'test' } } }); + }); + + it('should clear the field but not its parent when passing in null and parent has other children', () => { + const recipe = { source: { config: { test: 'test', profiling: { enabled: true, testing: 'hello' } } } }; + const updatedRecipe = setFieldValueOnRecipe(recipe, null, 'source.config.profiling.testing'); + expect(updatedRecipe).toMatchObject({ source: { config: { test: 'test', profiling: { enabled: true } } } }); + }); +}); + +describe('setListValuesOnRecipe', () => { + const tableAllowFieldPath = 'source.config.table_pattern.allow'; + + it('should update list values on a recipe when it was not defined', () => { + const recipe = { source: { config: {} } }; + const updatedRecipe = setListValuesOnRecipe(recipe, ['*test_pattern'], tableAllowFieldPath); + expect(updatedRecipe).toMatchObject({ source: { config: { table_pattern: { allow: ['*test_pattern'] } } } }); + }); + + it('should update list values on a recipe when it was defined', () => { + const recipe = { source: { config: { table_pattern: { allow: ['*test_pattern'] } } } }; + const updatedRecipe = setListValuesOnRecipe(recipe, ['*test_pattern(edit)'], tableAllowFieldPath); + expect(updatedRecipe).toMatchObject({ + source: { config: { table_pattern: { allow: ['*test_pattern(edit)'] } } }, + }); + }); + + it('should append list values on a recipe', () => { + const recipe = { source: { config: { table_pattern: { allow: ['*test_pattern'] } } } }; + const updatedRecipe = setListValuesOnRecipe(recipe, ['*test_pattern', 'new'], tableAllowFieldPath); + expect(updatedRecipe).toMatchObject({ + source: { config: { table_pattern: { allow: ['*test_pattern', 'new'] } } }, + }); + }); + + it('should remove list values on a recipe', () => { + const recipe = { source: { config: { table_pattern: { allow: ['*test_pattern', 'remove_me'] } } } }; + const updatedRecipe = setListValuesOnRecipe(recipe, ['*test_pattern'], tableAllowFieldPath); + expect(updatedRecipe).toMatchObject({ + source: { config: { table_pattern: { allow: ['*test_pattern'] } } }, + }); + }); + + it('should remove empty values from the list when updating a recipe', () => { + const recipe = { source: { config: { table_pattern: { allow: ['*test_pattern'] } } } }; + const updatedRecipe = setListValuesOnRecipe(recipe, ['*test_pattern', '', '', ''], tableAllowFieldPath); + expect(updatedRecipe).toMatchObject({ + source: { config: { table_pattern: { allow: ['*test_pattern'] } } }, + }); + }); + + it('should clear the value and its parent when passing in empty list and parent has no other children', () => { + const recipe = { source: { config: { existingField: true, table_pattern: { allow: ['*test_pattern'] } } } }; + const updatedRecipe = setListValuesOnRecipe(recipe, [], tableAllowFieldPath); + expect(updatedRecipe).toMatchObject({ source: { config: { existingField: true } } }); + }); + + it('should clear the value but not its parent when passing in empty list and parent has other children', () => { + const recipe = { source: { config: { table_pattern: { allow: ['*test_pattern'], deny: ['test_deny'] } } } }; + const updatedRecipe = setListValuesOnRecipe(recipe, [], tableAllowFieldPath); + expect(updatedRecipe).toMatchObject({ source: { config: { table_pattern: { deny: ['test_deny'] } } } }); + }); +}); diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/utils.ts b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/utils.ts new file mode 100644 index 00000000000000..f80d74815ad498 --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/utils.ts @@ -0,0 +1,287 @@ +import { set, get } from 'lodash'; +import { SNOWFLAKE } from '../../conf/snowflake/snowflake'; + +export enum FieldType { + TEXT, + BOOLEAN, + LIST, +} + +export interface RecipeField { + name: string; + label: string; + tooltip: string; + type: FieldType; + fieldPath: string; + rules: any[] | null; + section?: string; + getValueFromRecipeOverride?: (recipe: any) => any; + setValueOnRecipeOverride?: (recipe: any, value: any) => any; +} + +function clearFieldAndParents(recipe: any, fieldPath: string) { + set(recipe, fieldPath, undefined); + + const updatedFieldPath = fieldPath.split('.').slice(0, -1).join('.'); // remove last item from fieldPath + if (updatedFieldPath) { + const parentKeys = Object.keys(get(recipe, updatedFieldPath)); + + // only child left is what we just set as undefined + if (parentKeys.length === 1) { + clearFieldAndParents(recipe, updatedFieldPath); + } + } + return recipe; +} + +export function setFieldValueOnRecipe(recipe: any, value: any, fieldPath: string) { + const updatedRecipe = { ...recipe }; + if (value !== undefined) { + if (value === null) { + clearFieldAndParents(updatedRecipe, fieldPath); + return updatedRecipe; + } + set(updatedRecipe, fieldPath, value); + } + return updatedRecipe; +} + +export function setListValuesOnRecipe(recipe: any, values: string[] | undefined, fieldPath: string) { + const updatedRecipe = { ...recipe }; + if (values !== undefined) { + const filteredValues: string[] | undefined = values.filter((v) => !!v); + return filteredValues.length + ? setFieldValueOnRecipe(updatedRecipe, filteredValues, fieldPath) + : setFieldValueOnRecipe(updatedRecipe, null, fieldPath); + } + return updatedRecipe; +} + +export const SNOWFLAKE_ACCOUNT_ID: RecipeField = { + name: 'account_id', + label: 'Account ID', + tooltip: 'Snowflake account. e.g. abc48144', + type: FieldType.TEXT, + fieldPath: 'source.config.account_id', + rules: null, +}; + +export const SNOWFLAKE_WAREHOUSE: RecipeField = { + name: 'warehouse', + label: 'Warehouse', + tooltip: 'Snowflake warehouse.', + type: FieldType.TEXT, + fieldPath: 'source.config.warehouse', + rules: null, +}; + +export const SNOWFLAKE_USERNAME: RecipeField = { + name: 'username', + label: 'Username', + tooltip: 'Snowflake username.', + type: FieldType.TEXT, + fieldPath: 'source.config.username', + rules: null, +}; + +export const SNOWFLAKE_PASSWORD: RecipeField = { + name: 'password', + label: 'Password', + tooltip: 'Snowflake password.', + type: FieldType.TEXT, + fieldPath: 'source.config.password', + rules: null, +}; + +export const SNOWFLAKE_ROLE: RecipeField = { + name: 'role', + label: 'Role', + tooltip: 'Snowflake role.', + type: FieldType.TEXT, + fieldPath: 'source.config.role', + rules: null, +}; + +const includeLineageFieldPathA = 'source.config.include_table_lineage'; +const includeLineageFieldPathB = 'source.config.include_view_lineage'; +export const INCLUDE_LINEAGE: RecipeField = { + name: 'include_lineage', + label: 'Include Lineage', + tooltip: 'Include Table and View lineage in your ingestion.', + type: FieldType.BOOLEAN, + fieldPath: 'source.config.include_table_lineage', + rules: null, + getValueFromRecipeOverride: (recipe: any) => + get(recipe, includeLineageFieldPathA) && get(recipe, includeLineageFieldPathB), + setValueOnRecipeOverride: (recipe: any, value: boolean) => { + let updatedRecipe = setFieldValueOnRecipe(recipe, value, includeLineageFieldPathA); + updatedRecipe = setFieldValueOnRecipe(updatedRecipe, value, includeLineageFieldPathB); + return updatedRecipe; + }, +}; + +export const IGNORE_START_TIME_LINEAGE: RecipeField = { + name: 'ignore_start_time_lineage', + label: 'Ignore Start Time Lineage', + tooltip: 'Get all lineage by ignoring the start_time field. It is suggested to set to true initially.', + type: FieldType.BOOLEAN, + fieldPath: 'source.config.ignore_start_time_lineage', + rules: null, +}; + +export const CHECK_ROLE_GRANTS: RecipeField = { + name: 'check_role_grants', + label: 'Check Role Grants', + tooltip: + 'If set to True then checks role grants at the beginning of the ingestion run. To be used for debugging purposes. If you think everything is working fine then set it to False. In some cases this can take long depending on how many roles you might have.', + type: FieldType.BOOLEAN, + fieldPath: 'source.config.check_role_grants', + rules: null, +}; + +export const PROFILING_ENABLED: RecipeField = { + name: 'profiling.enabled', + label: 'Enable Profiling', + tooltip: 'Whether profiling should be done.', + type: FieldType.BOOLEAN, + fieldPath: 'source.config.profiling.enabled', + rules: null, +}; + +export const STATEFUL_INGESTION_ENABLED: RecipeField = { + name: 'stateful_ingestion.enabled', + label: 'Enable Stateful Ingestion', + tooltip: 'Enable the type of the ingestion state provider registered with datahub.', + type: FieldType.BOOLEAN, + fieldPath: 'source.config.stateful_ingestion.enabled', + rules: null, +}; + +const databaseAllowFieldPath = 'source.config.database_pattern.allow'; +export const DATABASE_ALLOW: RecipeField = { + name: 'database_pattern.allow', + label: 'Allow Patterns', + tooltip: 'Use regex here.', + type: FieldType.LIST, + fieldPath: 'source.config.database_pattern.allow', + rules: null, + section: 'Databases', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, databaseAllowFieldPath), +}; + +const databaseDenyFieldPath = 'source.config.database_pattern.deny'; +export const DATABASE_DENY: RecipeField = { + name: 'database_pattern.deny', + label: 'Deny Patterns', + tooltip: 'Use regex here.', + type: FieldType.LIST, + fieldPath: 'source.config.database_pattern.deny', + rules: null, + section: 'Databases', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, databaseDenyFieldPath), +}; + +const schemaAllowFieldPath = 'source.config.schema_pattern.allow'; +export const SCHEMA_ALLOW: RecipeField = { + name: 'schema_pattern.allow', + label: 'Allow Patterns', + tooltip: 'Use regex here.', + type: FieldType.LIST, + fieldPath: 'source.config.schema_pattern.allow', + rules: null, + section: 'Schemas', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, schemaAllowFieldPath), +}; + +const schemaDenyFieldPath = 'source.config.schema_pattern.deny'; +export const SCHEMA_DENY: RecipeField = { + name: 'schema_pattern.deny', + label: 'Deny Patterns', + tooltip: 'Use regex here.', + type: FieldType.LIST, + fieldPath: 'source.config.schema_pattern.deny', + rules: null, + section: 'Schemas', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, schemaDenyFieldPath), +}; + +const viewAllowFieldPath = 'source.config.view_pattern.allow'; +export const VIEW_ALLOW: RecipeField = { + name: 'view_pattern.allow', + label: 'Allow Patterns', + tooltip: 'Use regex here.', + type: FieldType.LIST, + fieldPath: 'source.config.view_pattern.allow', + rules: null, + section: 'Views', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, viewAllowFieldPath), +}; + +const viewDenyFieldPath = 'source.config.view_pattern.deny'; +export const VIEW_DENY: RecipeField = { + name: 'view_pattern.deny', + label: 'Deny Patterns', + tooltip: 'Use regex here.', + type: FieldType.LIST, + fieldPath: 'source.config.view_pattern.deny', + rules: null, + section: 'Views', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, viewDenyFieldPath), +}; + +const tableAllowFieldPath = 'source.config.table_pattern.allow'; +export const TABLE_ALLOW: RecipeField = { + name: 'table_pattern.allow', + label: 'Allow Patterns', + tooltip: 'Use regex here.', + type: FieldType.LIST, + fieldPath: 'source.config.table_pattern.allow', + rules: null, + section: 'Tables', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, tableAllowFieldPath), +}; + +const tableDenyFieldPath = 'source.config.table_pattern.deny'; +export const TABLE_DENY: RecipeField = { + name: 'table_pattern.deny', + label: 'Deny Patterns', + tooltip: 'Use regex here.', + type: FieldType.LIST, + fieldPath: 'source.config.table_pattern.deny', + rules: null, + section: 'Tables', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, tableDenyFieldPath), +}; + +export const RECIPE_FIELDS = { + [SNOWFLAKE]: { + fields: [SNOWFLAKE_ACCOUNT_ID, SNOWFLAKE_WAREHOUSE, SNOWFLAKE_USERNAME, SNOWFLAKE_PASSWORD, SNOWFLAKE_ROLE], + advancedFields: [ + INCLUDE_LINEAGE, + IGNORE_START_TIME_LINEAGE, + CHECK_ROLE_GRANTS, + PROFILING_ENABLED, + STATEFUL_INGESTION_ENABLED, + ], + filterFields: [ + TABLE_ALLOW, + TABLE_DENY, + DATABASE_ALLOW, + DATABASE_DENY, + SCHEMA_ALLOW, + SCHEMA_DENY, + VIEW_ALLOW, + VIEW_DENY, + ], + }, +}; + +export const CONNECTORS_WITH_FORM = new Set(Object.keys(RECIPE_FIELDS)); diff --git a/datahub-web-react/src/app/ingest/source/builder/__tests__/DefineRecipeStep.test.tsx b/datahub-web-react/src/app/ingest/source/builder/__tests__/DefineRecipeStep.test.tsx new file mode 100644 index 00000000000000..ce1e9d6d7184dc --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/builder/__tests__/DefineRecipeStep.test.tsx @@ -0,0 +1,35 @@ +import { render } from '@testing-library/react'; +import React from 'react'; +import { DefineRecipeStep } from '../DefineRecipeStep'; + +describe('DefineRecipeStep', () => { + it('should render the RecipeBuilder if the type is in CONNECTORS_WITH_FORM', () => { + const { getByText, queryByText } = render( + {}} + goTo={() => {}} + submit={() => {}} + cancel={() => {}} + />, + ); + + expect(getByText('Connection')).toBeInTheDocument(); + expect(queryByText('For more information about how to configure a recipe')).toBeNull(); + }); + + it('should not render the RecipeBuilder if the type is not in CONNECTORS_WITH_FORM', () => { + const { getByText, queryByText } = render( + {}} + goTo={() => {}} + submit={() => {}} + cancel={() => {}} + />, + ); + + expect(getByText('Configure Postgres Recipe')).toBeInTheDocument(); + expect(queryByText('Connection')).toBeNull(); + }); +}); diff --git a/datahub-web-react/src/app/ingest/source/conf/snowflake/snowflake.ts b/datahub-web-react/src/app/ingest/source/conf/snowflake/snowflake.ts index 811d116809306f..e89a1187a91ef8 100644 --- a/datahub-web-react/src/app/ingest/source/conf/snowflake/snowflake.ts +++ b/datahub-web-react/src/app/ingest/source/conf/snowflake/snowflake.ts @@ -5,52 +5,23 @@ const placeholderRecipe = `\ source: type: snowflake config: - # Uncomment this section to provision the role required for ingestion - # provision_role: - # enabled: true - # dry_run: false - # run_ingestion: true - # admin_username: "\${SNOWFLAKE_ADMIN_USER}" - # admin_password: "\${SNOWFLAKE_ADMIN_PASS}" - - # Your Snowflake account name - # e.g. if URL is example48144.us-west-2.snowflakecomputing.com then use "example48144" - account_id: "example48144" - warehouse: # Your Snowflake warehouse name, e.g. "PROD_WH" - - # Credentials - username: "\${SNOWFLAKE_USER}" # Create a secret SNOWFLAKE_USER in secrets Tab - password: "\${SNOWFLAKE_PASS}" # Create a secret SNOWFLAKE_PASS in secrets Tab + account_id: "example_id" + warehouse: "example_warehouse" role: "datahub_role" - - # Suggest to have this set to true initially to get all lineage ignore_start_time_lineage: true - - # This is an alternative option to specify the start_time for lineage - # if you don't want to look back since beginning - # start_time: '2022-03-01T00:00:00Z' - - # Uncomment and change to only allow some database metadata to be ingested - # database_pattern: - # allow: - # - "^ACCOUNTING_DB$" - # - "^MARKETING_DB$" - - # Uncomment and change to deny some metadata from few schemas - # schema_pattern: - # deny: - # - "information_schema.*" - - # If you want to ingest only few tables with name revenue and sales - # table_pattern: - # allow: - # - ".*revenue" - # - ".*sales" - + include_table_lineage: true + include_view_lineage: true + check_role_grants: true + profiling: + enabled: true + stateful_ingestion: + enabled: true `; +export const SNOWFLAKE = 'snowflake'; + const snowflakeConfig: SourceConfig = { - type: 'snowflake', + type: SNOWFLAKE, placeholderRecipe, displayName: 'Snowflake', docsUrl: 'https://datahubproject.io/docs/generated/ingestion/sources/snowflake/',