feat: add support from structured datasets (#801)

* feat: add support from structured datasets Signed-off-by: Carina Ursu <[email protected]> * chore: don't enfore format Signed-off-by: Carina Ursu <[email protected]> * chore: fix tests Signed-off-by: Carina Ursu <[email protected]> --------- Signed-off-by: Carina Ursu <[email protected]>
flyteorg · Jul 25, 2023 · 16abb34 · 16abb34
1 parent 3eb0fa7
commit 16abb34
Show file tree

Hide file tree

Showing 18 changed files with 177 additions and 9 deletions.
diff --git a/packages/console/package.json b/packages/console/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@flyteorg/console",
-  "version": "0.0.43",
+  "version": "0.0.44",
   "description": "Flyteconsole main app module",
   "main": "./dist/index.js",
   "module": "./lib/index.js",

diff --git a/...onents/Executions/contextProvider/NodeExecutionDetails/WorkflowNodeExecutionsProvider.tsx b/...onents/Executions/contextProvider/NodeExecutionDetails/WorkflowNodeExecutionsProvider.tsx
@@ -209,7 +209,6 @@ export const WorkflowNodeExecutionsProvider = ({
           setShouldUpdate(true);
         }
 
-        console.log('carina newNodes', { newNodes });
         return newNodes;
       });
     },

diff --git a/.../console/src/components/Launch/LaunchForm/LaunchFormComponents/StructuredDatasetInput.tsx b/.../console/src/components/Launch/LaunchForm/LaunchFormComponents/StructuredDatasetInput.tsx
@@ -0,0 +1,84 @@
+import React, { FC, useEffect, useMemo, useState } from 'react';
+import { TextField } from '@material-ui/core';
+import { makeStyles, Theme } from '@material-ui/core/styles';
+import t from '../strings';
+import { DatasetValue, InputProps } from '../types';
+import { getLaunchInputId } from '../utils';
+import { StyledCard } from './StyledCard';
+import { getHelperForInput } from '../inputHelpers/getHelperForInput';
+
+const useStyles = makeStyles((theme: Theme) => ({
+  formatInput: {
+    flex: '1 1 auto',
+  },
+  inputContainer: {
+    marginTop: theme.spacing(1),
+    paddingLeft: theme.spacing(1),
+  },
+  metadataContainer: {
+    display: 'flex',
+    marginTop: theme.spacing(1),
+    width: '100%',
+  },
+}));
+
+/** A micro form for entering the values related to a Structured Dataset Literal */
+export const StructuredDatasetInput: FC<InputProps> = props => {
+  const styles = useStyles();
+  const {
+    error,
+    name,
+    onChange,
+    value: propValue,
+    typeDefinition,
+    label,
+  } = props;
+
+  const helper = useMemo(
+    () => getHelperForInput(typeDefinition.type),
+    [typeDefinition],
+  );
+
+  const [datasetValue, setDatasetValue] = useState<DatasetValue>(
+    propValue ||
+      (helper.typeDefinitionToDefaultValue(typeDefinition) as DatasetValue),
+  );
+
+  const handleChange = (input: Partial<DatasetValue>) => {
+    const value = { ...datasetValue, ...input } as DatasetValue;
+    setDatasetValue(value);
+  };
+
+  useEffect(() => {
+    onChange(datasetValue);
+  }, [datasetValue]);
+
+  const inputId = getLaunchInputId(`${name}-dsd`);
+
+  return (
+    <StyledCard error={error} label={label}>
+      <div className={styles.inputContainer}>
+        <TextField
+          id={`${inputId}-uri`}
+          helperText={t('sdsUriHelperText')}
+          fullWidth={true}
+          label="uri"
+          onChange={e => handleChange({ uri: e.target.value })}
+          value={datasetValue?.uri}
+          variant="outlined"
+        />
+        <div className={styles.metadataContainer}>
+          <TextField
+            className={styles.formatInput}
+            id={`${inputId}-format`}
+            helperText={t('sdsFormatHelperText')}
+            label="format"
+            onChange={e => handleChange({ format: e.target.value })}
+            value={datasetValue?.format}
+            variant="outlined"
+          />
+        </div>
+      </div>
+    </StyledCard>
+  );
+};
diff --git a/...es/console/src/components/Launch/LaunchForm/LaunchFormComponents/getComponentForInput.tsx b/...es/console/src/components/Launch/LaunchForm/LaunchFormComponents/getComponentForInput.tsx
@@ -9,6 +9,7 @@ import { MapInput } from '../MapInput';
 import { UnsupportedInput } from './UnsupportedInput';
 import { NoneInput } from './NoneInput';
 import { SimpleInput } from './SimpleInput';
+import { StructuredDatasetInput } from './StructuredDatasetInput';
 
 export function getComponentForInput(input: InputProps, showErrors: boolean) {
   const onChange = (newValue: InputValue) => {
@@ -36,6 +37,8 @@ export function getComponentForInput(input: InputProps, showErrors: boolean) {
       return <CollectionInput {...props} />;
     case InputType.Struct:
       return <StructInput {...props} />;
+    case InputType.StructuredDataset:
+      return <StructuredDatasetInput {...props} />;
     case InputType.Map:
       return <MapInput {...props} />;
     case InputType.Unknown:

diff --git a/packages/console/src/components/Launch/LaunchForm/UnsupportedRequiredInputsError.tsx b/packages/console/src/components/Launch/LaunchForm/UnsupportedRequiredInputsError.tsx
@@ -1,8 +1,8 @@
+import * as React from 'react';
 import { makeStyles, Theme } from '@material-ui/core/styles';
 import ErrorOutline from '@material-ui/icons/ErrorOutline';
 import { NonIdealState } from 'components/common/NonIdealState';
 import { useCommonStyles } from 'components/common/styles';
-import * as React from 'react';
 import t from './strings';
 import { ParsedInput } from './types';
 

diff --git a/packages/console/src/components/Launch/LaunchForm/constants.ts b/packages/console/src/components/Launch/LaunchForm/constants.ts
@@ -33,6 +33,7 @@ export const typeLabels: { [k in InputType]: string } = {
   [InputType.Schema]: 'schema - uri',
   [InputType.String]: 'string',
   [InputType.Struct]: 'struct',
+  [InputType.StructuredDataset]: 'structured dataset',
   [InputType.Union]: 'union',
   [InputType.Unknown]: 'unknown',
 };

diff --git a/packages/console/src/components/Launch/LaunchForm/getInputs.ts b/packages/console/src/components/Launch/LaunchForm/getInputs.ts
@@ -43,7 +43,7 @@ export function getInputsForWorkflow(
       : typeLabel;
     const inputKey = createInputCacheKey(name, typeDefinition);
     const defaultVaue =
-      parameter.default != null ? parameter.default : undefined;
+      parameter.default !== null ? parameter.default : undefined;
     // TODO: fill default value if initial value is not set
     const initialValue = initialValues.has(inputKey)
       ? initialValues.get(inputKey)

diff --git a/packages/console/src/components/Launch/LaunchForm/inputHelpers/blob.ts b/packages/console/src/components/Launch/LaunchForm/inputHelpers/blob.ts
@@ -76,7 +76,7 @@ function validate({ value, required }: InputValidatorParams) {
       `Unknown blob dimensionality value: ${blobValue.dimensionality}`,
     );
   }
-  if (blobValue.format != null && typeof blobValue.format !== 'string') {
+  if (!!blobValue.format && typeof blobValue.format !== 'string') {
     throw new Error('Blob format must be a string');
   }
 }

diff --git a/packages/console/src/components/Launch/LaunchForm/inputHelpers/getHelperForInput.ts b/packages/console/src/components/Launch/LaunchForm/inputHelpers/getHelperForInput.ts
@@ -11,6 +11,7 @@ import { noneHelper } from './none';
 import { schemaHelper } from './schema';
 import { stringHelper } from './string';
 import { structHelper } from './struct';
+import { structuredDataSetHelper } from './structuredDataSet';
 import { unionHelper } from './union';
 import { InputHelper } from './types';
 
@@ -33,6 +34,7 @@ const inputHelpers: Record<InputType, InputHelper> = {
   [InputType.Schema]: schemaHelper,
   [InputType.String]: stringHelper,
   [InputType.Struct]: structHelper,
+  [InputType.StructuredDataset]: structuredDataSetHelper,
   [InputType.Union]: unionHelper,
   [InputType.Unknown]: unsupportedHelper,
 };

diff --git a/packages/console/src/components/Launch/LaunchForm/inputHelpers/structuredDataSet.ts b/packages/console/src/components/Launch/LaunchForm/inputHelpers/structuredDataSet.ts
@@ -0,0 +1,60 @@
+import { Core } from '@flyteorg/flyteidl-types';
+import { isObject } from 'lodash';
+import { DatasetValue, InputTypeDefinition } from '../types';
+import { ConverterInput, InputHelper, InputValidatorParams } from './types';
+
+function fromLiteral(
+  literal: Core.ILiteral,
+  typeDefinition: InputTypeDefinition,
+): DatasetValue {
+  const { structuredDataset } = literal?.scalar || {};
+
+  const { metadata, uri } = structuredDataset || {};
+  const { format } = metadata?.structuredDatasetType || {};
+  return {
+    uri: uri!,
+    format: format!,
+  };
+}
+
+function toLiteral({ value, typeDefinition }: ConverterInput): Core.ILiteral {
+  const dsValue = value as DatasetValue;
+  return {
+    scalar: {
+      structuredDataset: {
+        metadata: {
+          structuredDatasetType: {
+            format: dsValue.format,
+          },
+        },
+        uri: dsValue.uri,
+      },
+    },
+  };
+}
+
+function validate({ value, required }: InputValidatorParams) {
+  if (!isObject(value)) {
+    throw new Error('Invalid structured datased value');
+  }
+  const dsValue = value as DatasetValue;
+  if (required && (!dsValue?.uri || typeof dsValue?.uri !== 'string')) {
+    throw new Error('Dataset uri is required');
+  }
+
+  if (!!dsValue.format && typeof dsValue.format !== 'string') {
+    throw new Error('Dataset format must be a string');
+  }
+}
+
+export const structuredDataSetHelper: InputHelper = {
+  fromLiteral,
+  toLiteral,
+  validate,
+  typeDefinitionToDefaultValue: typeDefinition => {
+    return {
+      format: undefined,
+      uri: undefined,
+    } as DatasetValue;
+  },
+};
diff --git a/packages/console/src/components/Launch/LaunchForm/inputHelpers/utils.ts b/packages/console/src/components/Launch/LaunchForm/inputHelpers/utils.ts
@@ -60,6 +60,7 @@ export function typeIsSupported(typeDefinition: InputTypeDefinition): boolean {
     case InputType.Schema:
     case InputType.String:
     case InputType.Struct:
+    case InputType.StructuredDataset:
       return true;
     case InputType.Union:
       if (listOfSubTypes?.length) {

diff --git a/packages/console/src/components/Launch/LaunchForm/strings.ts b/packages/console/src/components/Launch/LaunchForm/strings.ts
@@ -12,6 +12,8 @@ const str = {
   taskUnsupportedRequiredInputsString: `This Task version contains one or more required inputs which are not supported by Flyte Console.\n\nYou can launch this Task version with the Flyte CLI instead.\n\nThe required inputs are :`,
   blobUriHelperText: '(required) location of the data',
   blobFormatHelperText: '(optional) csv, parquet, etc...',
+  sdsUriHelperText: '(required) location of the data',
+  sdsFormatHelperText: '(optional) parquet, feather, csv, etc...',
   correctInputErrors:
     'Some inputs have errors. Please correct them before submitting.',
   noneInputTypeDescription: 'The value of none type is empty',

diff --git a/packages/console/src/components/Launch/LaunchForm/types.ts b/packages/console/src/components/Launch/LaunchForm/types.ts
@@ -202,6 +202,7 @@ export enum InputType {
   Schema = 'SCHEMA',
   String = 'STRING',
   Struct = 'STRUCT',
+  StructuredDataset = 'STRUCTUREDDS',
   Union = 'Union',
   Unknown = 'UNKNOWN',
 }
@@ -219,6 +220,11 @@ export interface BlobValue {
   uri: string;
 }
 
+export interface DatasetValue {
+  uri?: string;
+  format?: string;
+}
+
 export interface UnionValue {
   value: InputValue;
   typeDefinition: InputTypeDefinition;
@@ -231,6 +237,7 @@ export type InputValue =
   | number
   | boolean
   | Date
+  | DatasetValue
   | BlobValue
   | UnionValue
   | NoneValue;

diff --git a/packages/console/src/components/Launch/LaunchForm/useLaunchWorkflowFormState.ts b/packages/console/src/components/Launch/LaunchForm/useLaunchWorkflowFormState.ts
@@ -153,9 +153,11 @@ async function loadInputs(
     defaultInputValues,
   );
 
+  const unsupportedRequiredInputs = getUnsupportedRequiredInputs(parsedInputs);
+
   return {
     parsedInputs,
-    unsupportedRequiredInputs: getUnsupportedRequiredInputs(parsedInputs),
+    unsupportedRequiredInputs,
   };
 }
 

diff --git a/packages/console/src/components/Launch/LaunchForm/utils.ts b/packages/console/src/components/Launch/LaunchForm/utils.ts
@@ -175,6 +175,8 @@ export function getInputDefintionForLiteralType(
     result.subtype = getInputDefintionForLiteralType(literalType.mapValueType);
   } else if (literalType.schema) {
     result.type = InputType.Schema;
+  } else if (literalType.structuredDatasetType) {
+    result.type = InputType.StructuredDataset;
   } else if (literalType.simple) {
     result.type = simpleTypeToInputType[literalType.simple];
   } else if (literalType.enumType) {

diff --git a/packages/console/src/models/Common/types.ts b/packages/console/src/models/Common/types.ts
@@ -21,6 +21,10 @@ export type SchemaColumnType = Core.SchemaType.SchemaColumn.SchemaColumnType;
 export const SchemaColumnType = Core.SchemaType.SchemaColumn.SchemaColumnType;
 export type MessageFormat = Core.TaskLog.MessageFormat;
 export const MessageFormat = Core.TaskLog.MessageFormat;
+export type StructuredDatasetType = Core.StructuredDatasetType;
+export const StructuredDatasetType = Core.StructuredDatasetType;
+export type DatasetColumn = Core.StructuredDatasetType.DatasetColumn;
+export const DatasetColumn = Core.StructuredDatasetType.DatasetColumn;
 /* eslint-enable @typescript-eslint/no-redeclare */
 
 export type Alias = Core.IAlias;
@@ -154,6 +158,7 @@ export interface LiteralType extends Core.ILiteralType {
   metadata?: ProtobufStruct;
   schema?: SchemaType;
   simple?: SimpleType;
+  structuredDatasetType?: StructuredDatasetType;
   enumType?: EnumType;
 }
 

diff --git a/website/package.json b/website/package.json
@@ -37,7 +37,7 @@
   },
   "dependencies": {
     "@flyteorg/common": "^0.0.4",
-    "@flyteorg/console": "^0.0.43",
+    "@flyteorg/console": "^0.0.44",
     "long": "^4.0.0",
     "protobufjs": "~6.11.3",
     "react-ga4": "^1.4.1",

diff --git a/yarn.lock b/yarn.lock
@@ -2020,7 +2020,7 @@ __metadata:
   resolution: "@flyteconsole/client-app@workspace:website"
   dependencies:
     "@flyteorg/common": ^0.0.4
-    "@flyteorg/console": ^0.0.43
+    "@flyteorg/console": ^0.0.44
     "@types/long": ^3.0.32
     long: ^4.0.0
     protobufjs: ~6.11.3
@@ -2059,7 +2059,7 @@ __metadata:
   languageName: unknown
   linkType: soft
 
-"@flyteorg/console@^0.0.43, @flyteorg/console@workspace:packages/console":
+"@flyteorg/console@^0.0.44, @flyteorg/console@workspace:packages/console":
   version: 0.0.0-use.local
   resolution: "@flyteorg/console@workspace:packages/console"
   dependencies:
-Original file line number
+Diff line change
@@ Expand Up / @@ -209,7 +209,6 @@ export const WorkflowNodeExecutionsProvider = ({ @@
               setShouldUpdate(true);
             }
-            console.log('carina newNodes', { newNodes });
             return newNodes;
           });
         },
@@ Expand Down @@