diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 9c71a3bf..11c1f3c3 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -19,7 +19,7 @@ jobs: with: node-version: 16.x cache: 'yarn' - - run: yarn + - run: yarn - run: yarn test - - run: yarn lint || true # TODO dont ignore lint errors + - run: yarn lint - run: yarn build diff --git a/README.md b/README.md index 83fbb070..3dbe5c45 100644 --- a/README.md +++ b/README.md @@ -76,24 +76,27 @@ The configuration file contains paths to input files which are included in the z The workflow configuration file consists out of 2 parts: 1. Global parameters, which are available to engine and each node. -2. Table with parameters for each node the workflow should run. +2. Tables with parameters for each node the workflow should run. -TOML does not allow for tables with same name. So any node that needs be run multiple times should have a index appened to the table name. +TOML does not allow for tables with same name. So any node that occurs multiple times should have a index appened to the table name. ### Catalog The catalog is a YAML formatted file which tells the app what nodes are available. In has the following info: -1. Description of global parameters +1. global: Description of global parameters * schema: What parameters are valid. Formatted as JSON schema draft 7. * uiSchema: How the form for filling the parameters should be rendered. -2. Description of available nodes. +2. nodes: Description of available nodes. * id: Identifier of node, for computers * label: Label of node, for humans * category: Category to which node belongs * description: Text describing what node needs, does and produces. * schema: What parameters are valid. Formatted as JSON schema draft 7. * uiSchema: How the form for filling the parameters should be rendered. -3. Descriptions of node categories -4. Title and link to example workflows -5. Title of the catalog +3. catagories: Descriptions of node categories + * name: Name of category + * description: Description of category +4. examples: Title and link to example workflows + * map with title as key and link as value +5. title: Title of the catalog diff --git a/package.json b/package.json index e589e98f..a10e4a7f 100644 --- a/package.json +++ b/package.json @@ -43,5 +43,10 @@ }, "engines": { "node": "~16" + }, + "ts-standard": { + "ignore": [ + "vite.config.ts" + ] } } diff --git a/src/App.tsx b/src/App.tsx index 2d532fd9..1e74c86c 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -6,22 +6,25 @@ import 'react-toastify/dist/ReactToastify.css' import './App.css' import { CatalogPanel } from './CatalogPanel' -import { StepPanel } from './StepPanel' +import { NodePanel } from './NodePanel' import { WorkflowPanel } from './WorkflowPanel' import { Header } from './Header' +import { ErrorBoundary } from './ErrorBoundary' function App (): JSX.Element { return ( - Loading...}> - -
-
- - - -
- + + Loading...}> + +
+
+ + + +
+ + ) } diff --git a/src/CatalogCategory.tsx b/src/CatalogCategory.tsx index e4b66dd8..742767da 100644 --- a/src/CatalogCategory.tsx +++ b/src/CatalogCategory.tsx @@ -2,7 +2,7 @@ import { useCatalog } from './store' import { ICategory } from './types' import { CatalogNode } from './CatalogNode' -export const CatalogCategory = ({ name }: ICategory) => { +export const CatalogCategory = ({ name }: ICategory): JSX.Element => { const catalog = useCatalog() return (
  • diff --git a/src/CatalogNode.tsx b/src/CatalogNode.tsx index bb5152ef..d7c209ad 100644 --- a/src/CatalogNode.tsx +++ b/src/CatalogNode.tsx @@ -1,7 +1,7 @@ import { useWorkflow } from './store' -import { INode } from './types' +import { ICatalogNode } from './types' -export const CatalogNode = ({ id, label }: INode) => { +export const CatalogNode = ({ id, label }: ICatalogNode): JSX.Element => { const { addNodeToWorkflow } = useWorkflow() return (
  • diff --git a/src/CatalogPanel.tsx b/src/CatalogPanel.tsx index f3c8f185..5e81d600 100644 --- a/src/CatalogPanel.tsx +++ b/src/CatalogPanel.tsx @@ -1,23 +1,24 @@ import React from 'react' + import { CatalogPicker } from './CatalogPicker' import { CatalogCategory } from './CatalogCategory' import { useCatalog, useWorkflow } from './store' import { Example } from './Example' -export const CatalogPanel = () => { +export const CatalogPanel = (): JSX.Element => { const catalog = useCatalog() const { toggleGlobalEdit } = useWorkflow() return (
    - Step catalog + Catalog Loading catalog...}> -

    Nodes

    +

    Nodes

      {catalog?.categories.map((category) => )}
    -

    Examples

    +

    Examples

    Workflow examples that can be loaded as a starting point.
      {Object.entries(catalog?.examples).map(([name, workflow]) => )} diff --git a/src/CatalogPicker.tsx b/src/CatalogPicker.tsx index 7999b7c1..b9d271ce 100644 --- a/src/CatalogPicker.tsx +++ b/src/CatalogPicker.tsx @@ -2,9 +2,9 @@ import { useRecoilState } from 'recoil' import { catalogURLchoices } from './constants' import { catalogURLState } from './store' -export const CatalogPicker = () => { +export const CatalogPicker = (): JSX.Element => { const [url, setURL] = useRecoilState(catalogURLState) - // TODO clear workflow when switching catalogs + // TODO clear? workflow when switching catalogs return ( - + {selectedPanel} - +
    ) diff --git a/src/WorkflowUpload.tsx b/src/WorkflowUpload.tsx new file mode 100644 index 00000000..862fc642 --- /dev/null +++ b/src/WorkflowUpload.tsx @@ -0,0 +1,48 @@ +import { useRef } from 'react' +import { toast } from 'react-toastify' +import { useWorkflow } from './store' + +export const WorkflowUpload = (): JSX.Element => { + const uploadRef = useRef(null) + const { loadWorkflowArchive } = useWorkflow() + + async function uploadWorkflow (event: React.ChangeEvent): Promise { + if (event.target.files == null) { + return + } + const file = event.target.files[0] + const url = URL.createObjectURL(file) + await toast.promise( + async () => { + try { + await loadWorkflowArchive(url) + } finally { + URL.revokeObjectURL(url) + } + }, + { + pending: 'Loading workfow ...', + success: 'Workflow loaded', + error: { + render ({ data }) { + console.error(data) + return 'Workflow archive failed to load. See DevTools (F12) console for errors.' + } + } + } + ) + } + + return ( + + ) +} diff --git a/src/archive.ts b/src/archive.ts index be2677e0..3937533a 100644 --- a/src/archive.ts +++ b/src/archive.ts @@ -10,15 +10,15 @@ import { ZipWriter } from '@zip.js/zip.js' import { saveAs } from 'file-saver' -import { IStep, INode, IFiles, IParameters } from './types' +import { IWorkflowNode, ICatalogNode, IFiles, IParameters } from './types' import { workflow2tomltext } from './toml' import { workflowArchiveFilename, workflowFilename } from './constants' async function createZip ( - steps: IStep[], + nodes: IWorkflowNode[], global: IParameters, files: IFiles -) { +): Promise { const writer = new ZipWriter(new BlobWriter('application/zip')) // add data URL content to file in archive @@ -27,18 +27,18 @@ async function createZip ( await writer.add(fn, new Data64URIReader(dataURL)) ) ) - const text = workflow2tomltext(steps, global) + const text = workflow2tomltext(nodes, global) await writer.add(workflowFilename, new TextReader(text)) return await writer.close() } export async function saveArchive ( - steps: IStep[], + nodes: IWorkflowNode[], global: IParameters, files: IFiles -) { - const zip: Blob = await createZip(steps, global, files) +): Promise { + const zip: Blob = await createZip(nodes, global, files) saveAs(zip, workflowArchiveFilename) } @@ -47,7 +47,7 @@ export function injectFilenameIntoDataURL (filename: string, unnamedDataURL: str return unnamedDataURL.replace('data:;base64,', `data:${mimeType};name=${filename};base64,`) } -export async function readArchive (archiveURL: string, nodes: INode[]): Promise<{ +export async function readArchive (archiveURL: string, nodes: ICatalogNode[]): Promise<{ tomlstring: string files: IFiles }> { @@ -70,13 +70,11 @@ export async function readArchive (archiveURL: string, nodes: INode[]): Promise< } else if (entry.directory) { // Skip directories } else { - // TODO add mime type to Data64Uri const writer = new Data64URIWriter() const dataURL = await entry.getData(writer) files[entry.filename] = injectFilenameIntoDataURL(entry.filename, dataURL) } } - // TODO complain when there is no workflowFilename in archive if (tomlstring === '') { throw new Error('No workflow.cfg file found in workflow archive file') } diff --git a/src/catalog.ts b/src/catalog.ts new file mode 100644 index 00000000..f0a622f2 --- /dev/null +++ b/src/catalog.ts @@ -0,0 +1,36 @@ +import { load } from 'js-yaml' +import { ICatalog, IGlobal } from './types' +import { validateCatalog, ValidationError } from './validate' + +export async function fetchCatalog (catalogUrl: string): Promise { + const response = await fetch(catalogUrl) + if (!response.ok) { + throw new Error('Error retrieving catalog') + } + const body = await response.text() + const catalog = load(body) + if (!isCatalog(catalog)) { + throw new Error('Retrieved catalog is malformed') + } + const errors = validateCatalog(catalog) + if (errors.length > 0) { + throw new ValidationError('Invalid catalog loaded', errors) + } + return catalog +} + +export function isCatalog (catalog: unknown): catalog is ICatalog { + return typeof catalog === 'object' && + catalog !== null && + 'global' in catalog && + 'nodes' in catalog + // TODO add more checks +} + +export function globalParameterKeys (global: IGlobal): Set { + let keys: string[] = [] + if (global?.schema.properties != null) { + keys = Object.keys(global.schema.properties) + } + return new Set(keys) +} diff --git a/src/constants.tsx b/src/constants.tsx index a7b14cb6..fb902bbe 100644 --- a/src/constants.tsx +++ b/src/constants.tsx @@ -3,8 +3,6 @@ export const workflowFilename = 'workflow.cfg' export const workflowArchiveFilename = 'workflow.zip' -// TODO choices should be loaded from a yaml file -// so a workflow-builder distribution dir can be combined with a catalogy index and catalog item files to make an app. export const catalogURLchoices = [ ['haddock3basic', new URL('/haddock3.basic.catalog.yaml', import.meta.url).href], ['haddock3intermediate', new URL('/haddock3.intermediate.catalog.yaml', import.meta.url).href], diff --git a/src/dataurls.ts b/src/dataurls.ts index 6a21e2fc..fe65eb89 100644 --- a/src/dataurls.ts +++ b/src/dataurls.ts @@ -1,16 +1,16 @@ -import { IFiles } from './types' -import { walk } from './searchreplace' +import { IFiles, IParameters } from './types' +import { walk } from './utils/searchreplace' -export function isDataURL (value: string) { +export function isDataURL (value: string): boolean { return value.startsWith('data:') } -export function dataURL2filename (value: string) { +export function dataURL2filename (value: string): string { // rjsf creates data URLs like `data:text/markdown;name=README.md;base64,Rm9.....` return value.split(';')[1].split('=')[1] } -export function externalizeDataUrls (data: unknown, files: IFiles) { +export function externalizeDataUrls (data: IParameters, files: IFiles): IParameters { return walk(data, isDataURL, (d: string) => { const fn = dataURL2filename(d) files[fn] = d @@ -18,7 +18,7 @@ export function externalizeDataUrls (data: unknown, files: IFiles) { }) } -export function internalizeDataUrls (data: unknown, files: IFiles) { +export function internalizeDataUrls (data: IParameters, files: IFiles): IParameters { return walk( data, (d: string) => d in files, diff --git a/src/store.ts b/src/store.ts index b66a0715..c55715a7 100644 --- a/src/store.ts +++ b/src/store.ts @@ -1,12 +1,13 @@ -import { load } from 'js-yaml' import { atom, selector, useRecoilState, useRecoilValue } from 'recoil' + import { externalizeDataUrls } from './dataurls' -import { readArchive, saveArchive } from './archive' -import { ICatalog, IStep, IFiles, IParameters } from './types' -import { parseWorkflow, workflow2tomltext } from './toml' +import { saveArchive } from './archive' +import { ICatalog, IWorkflowNode, IFiles, IParameters, ICatalogNode } from './types' +import { workflow2tomltext } from './toml' import { catalogURLchoices } from './constants' -import { validateWorkflow, validateCatalog } from './validate' -import { toast } from 'react-toastify' +import { moveItem, removeItemAtIndex, replaceItemAtIndex } from './utils/array' +import { fetchCatalog } from './catalog' +import { dropUnusedFiles, loadWorkflowArchive } from './workflow' export const catalogURLState = atom({ key: 'catalogURL', @@ -17,23 +18,7 @@ const catalogState = selector({ key: 'catalog', get: async ({ get }) => { const catalogUrl = get(catalogURLState) - try { - const response = await fetch(catalogUrl) - const body = await response.text() - const catalog = load(body) - const errors = validateCatalog(catalog) - if (errors.length > 0) { - // TODO notify user of bad catalog - toast.error('Loading catalog failed') - return {} as ICatalog - } - // TODO Only report success when user initiated catalog loading, not when page is loaded - // toast.success('Loading catalog completed') - return catalog as ICatalog - } catch (error) { - toast.error('Loading catalog failed') - return {} as ICatalog - } + return await fetchCatalog(catalogUrl) } }) @@ -41,18 +26,6 @@ export function useCatalog (): ICatalog { return useRecoilValue(catalogState) } -const globalKeysState = selector>({ - key: 'globalKeys', - get: ({ get }) => { - const { global } = get(catalogState) - let keys: string[] = [] - if (global?.schema.properties != null) { - keys = Object.keys(global.schema.properties) - } - return new Set(keys) - } -}) - const globalParametersState = atom({ key: 'global', default: {} @@ -63,152 +36,163 @@ const editingGlobalParametersState = atom({ default: false }) -const stepsState = atom({ - key: 'steps', +const workflowNodesState = atom({ + key: 'workflowNodes', default: [] }) -const selectedStepIndexState = atom({ - key: 'selectedStepIndex', +const selectedNodeIndexState = atom({ + key: 'selectedNodeIndex', default: -1 }) +export function useSelectNodeIndex (): number { + return useRecoilValue(selectedNodeIndexState) +} + const filesState = atom({ key: 'files', default: {} }) -function replaceItemAtIndex (arr: V[], index: number, newValue: V) { - return [...arr.slice(0, index), newValue, ...arr.slice(index + 1)] +const selectedNodeState = selector({ + key: 'selectedNode', + get: ({ get }) => { + const index = get(selectedNodeIndexState) + const nodes = get(workflowNodesState) + if (index in nodes) { + return nodes[index] + } + return undefined + } +}) + +export function useSelectedNode (): IWorkflowNode | undefined { + return useRecoilValue(selectedNodeState) +} + +const selectedCatalogNodeState = selector({ + key: 'selectedNodeCatalogState', + get: ({ get }) => { + const node = get(selectedNodeState) + if (node === undefined) { + return undefined + } + const catalog = get(catalogState) + const catalogNode = catalog.nodes.find((n) => n.id === node.id) + if (catalogNode === undefined) { + return undefined + } + return catalogNode + } +}) + +export function useSelectedCatalogNode (): ICatalogNode | undefined { + return useRecoilValue(selectedCatalogNodeState) } -function removeItemAtIndex (arr: V[], index: number) { - return [...arr.slice(0, index), ...arr.slice(index + 1)] +interface UseWorkflow { + nodes: IWorkflowNode[] + editingGlobal: boolean + global: IParameters + toggleGlobalEdit: () => void + addNodeToWorkflow: (nodeId: string) => void + setGlobalParameters: (inlinedParameters: IParameters) => void + setNodeParameters: (inlinedParameters: IParameters) => void + loadWorkflowArchive: (archiveURL: string) => Promise + save: () => Promise + deleteNode: (nodeIndex: number) => void + selectNode: (nodeIndex: number) => void + clearNodeSelection: () => void + moveNodeDown: (nodeIndex: number) => void + moveNodeUp: (nodeIndex: number) => void } -export function useWorkflow () { - const [steps, setSteps] = useRecoilState(stepsState) +export function useWorkflow (): UseWorkflow { + const [nodes, setNodes] = useRecoilState(workflowNodesState) const [global, setGlobal] = useRecoilState(globalParametersState) const [editingGlobal, setEditingGlobal] = useRecoilState(editingGlobalParametersState) - const [selectedStepIndex, setSelectedStepIndex] = useRecoilState(selectedStepIndexState) - const { files, setFiles } = useFiles() - const { global: globalDescription,nodes } = useCatalog() - const globalKeys = useRecoilValue(globalKeysState); + const [selectedNodeIndex, setSelectedNodeIndex] = useRecoilState(selectedNodeIndexState) + const [files, setFiles] = useRecoilState(filesState) + const catalog = useCatalog() return { - steps, - selectedStep: selectedStepIndex, // TODO rename to selectedStepIndex + nodes, editingGlobal, global, toggleGlobalEdit () { setEditingGlobal(!editingGlobal) - setSelectedStepIndex(-1) + setSelectedNodeIndex(-1) }, addNodeToWorkflow (nodeId: string) { - setSteps([...steps, { id: nodeId, parameters: {} }]) - if (selectedStepIndex === -1 && !editingGlobal) { - setSelectedStepIndex(steps.length) + setNodes([...nodes, { id: nodeId, parameters: {} }]) + if (selectedNodeIndex === -1 && !editingGlobal) { + setSelectedNodeIndex(nodes.length) } }, - selectStep: (stepIndex: number) => { + selectNode: (nodeIndex: number) => { if (editingGlobal) { setEditingGlobal(false) } - setSelectedStepIndex(stepIndex) + setSelectedNodeIndex(nodeIndex) }, - deleteStep (stepIndex: number) { - if (stepIndex === selectedStepIndex) { - setSelectedStepIndex(-1) + deleteNode (nodeIndex: number) { + if (nodeIndex === selectedNodeIndex) { + setSelectedNodeIndex(-1) } - const newSteps = removeItemAtIndex(steps, stepIndex) - setSteps(newSteps) + const newNodes = removeItemAtIndex(nodes, nodeIndex) + const newFiles = dropUnusedFiles(global, newNodes, files) + setFiles(newFiles) + setNodes(newNodes) }, - clearStepSelection: () => setSelectedStepIndex(-1), - setParameters (inlinedParameters: unknown) { + clearNodeSelection: () => setSelectedNodeIndex(-1), + setGlobalParameters (inlinedParameters: IParameters) { const newFiles = { ...files } - // TODO forget files that are no longer refered to in parameters const parameters = externalizeDataUrls(inlinedParameters, newFiles) - if (editingGlobal) { - setGlobal(parameters) - } else { - const newStep = { ...steps[selectedStepIndex], parameters } - const newSteps = replaceItemAtIndex(steps, selectedStepIndex, newStep) - setSteps(newSteps as any) - } - setFiles(newFiles) + const newUsedFiles = dropUnusedFiles(parameters, nodes, newFiles) + setGlobal(parameters) + setFiles(newUsedFiles) + }, + setNodeParameters (inlinedParameters: IParameters) { + const newFiles = { ...files } + const parameters = externalizeDataUrls(inlinedParameters, newFiles) + const newNode = { ...nodes[selectedNodeIndex], parameters } + const newNodes = replaceItemAtIndex(nodes, selectedNodeIndex, newNode) + const newUsedFiles = dropUnusedFiles(global, newNodes, newFiles) + setNodes(newNodes) + setFiles(newUsedFiles) }, async loadWorkflowArchive (archiveURL: string) { - try { - const { tomlstring, files: newFiles } = await readArchive(archiveURL, nodes) - const { steps: newSteps, global: newGlobal } = parseWorkflow(tomlstring, globalKeys) - const errors = validateWorkflow({ - global: newGlobal, - steps: newSteps - }, { - global: globalDescription, - nodes: nodes - }) - if (errors.length > 0) { - // give feedback to users about errors - toast.error('Workflow archive is invalid. See DevTools console for errors') - console.error(errors) - } else { - setSteps(newSteps) - setFiles(newFiles) - setGlobal(newGlobal) - } - } catch (error) { - toast.error('Workflow archive is failed to load. See DevTools console for errors') - console.error(error) - } + const r = await loadWorkflowArchive(archiveURL, catalog) + setNodes(r.nodes) + setFiles(r.files) + setGlobal(r.global) }, async save () { - await saveArchive(steps, global, files) + await saveArchive(nodes, global, files) }, - moveStepDown (stepIndex: number) { - if (stepIndex + 1 < steps.length) { - const newSteps = moveStep(steps, stepIndex, 1) - setSelectedStepIndex(-1) - setSteps(newSteps) + moveNodeDown (nodeIndex: number) { + if (nodeIndex + 1 < nodes.length) { + const newNodes = moveItem(nodes, nodeIndex, 1) + setSelectedNodeIndex(-1) + setNodes(newNodes) } }, - moveStepUp (stepIndex: number) { - if (stepIndex > 0) { - const newSteps = moveStep(steps, stepIndex, -1) - setSelectedStepIndex(-1) - setSteps(newSteps) + moveNodeUp (nodeIndex: number) { + if (nodeIndex > 0) { + const newNodes = moveItem(nodes, nodeIndex, -1) + setSelectedNodeIndex(-1) + setNodes(newNodes) } } } } -export function useFiles () { - const [files, setFiles] = useRecoilState(filesState) - - return { - files, - setFiles - } -} - -function moveStep (steps: IStep[], stepIndex: number, direction: number) { - const step = steps[stepIndex] - const swappedIndex = stepIndex + direction - const swappedStep = steps[swappedIndex] - const newSteps = replaceItemAtIndex( - replaceItemAtIndex(steps, stepIndex, swappedStep), - swappedIndex, - step - ) - return newSteps -} - -export function useText () { - const { steps, global } = useWorkflow() - return workflow2tomltext(steps, global) +export function useFiles (): IFiles { + return useRecoilValue(filesState) } -export function useTextUrl () { - const text = useText() - return 'data:application/json;base64,' + btoa(text) +export function useText (): string { + const { nodes, global } = useWorkflow() + return workflow2tomltext(nodes, global) } diff --git a/src/toml.test.ts b/src/toml.test.ts index c8b60326..81c874aa 100644 --- a/src/toml.test.ts +++ b/src/toml.test.ts @@ -1,10 +1,9 @@ -import { expect, describe, it } from 'vitest'; -import { parseWorkflow, workflow2tomltext } from './toml'; -import { INode } from './types'; +import { expect, describe, it } from 'vitest' +import { parseWorkflow, workflow2tomltext } from './toml' -describe('steps2tomltext()', () => { +describe('nodes2tomltext()', () => { it('should write list of dicts as array of tables', () => { - const steps = [{ + const nodes = [{ id: 'somenode', parameters: { foo: [{ @@ -15,7 +14,7 @@ describe('steps2tomltext()', () => { } }] - const result = workflow2tomltext(steps, {}) + const result = workflow2tomltext(nodes, {}) const expected = ` [somenode] @@ -30,7 +29,7 @@ bar = 'fizzz' expect(result).toEqual(expected) }) it('should index repeated nodes', () => { - const steps = [ + const nodes = [ { id: 'somenode', parameters: { @@ -45,7 +44,7 @@ bar = 'fizzz' } ] - const result = workflow2tomltext(steps, {}) + const result = workflow2tomltext(nodes, {}) const expected = ` [somenode] @@ -74,7 +73,7 @@ foo = 'bar' global: { myglobalvar: 'something' }, - steps: [ + nodes: [ { id: 'somenode', parameters: { @@ -99,7 +98,7 @@ foo = 'fizz' const result = parseWorkflow(workflow, new Set()) const expected = { global: {}, - steps: [ + nodes: [ { id: 'somenode', parameters: { diff --git a/src/toml.ts b/src/toml.ts index 0368939f..f7a17bd7 100644 --- a/src/toml.ts +++ b/src/toml.ts @@ -1,42 +1,42 @@ import { Section, stringify, parse } from '@ltd/j-toml' -import { IStep, IParameters } from './types' +import { IWorkflowNode, IParameters, IWorkflow } from './types' function isObject (o: unknown): boolean { return typeof o === 'object' && Object.prototype.toString.call(o) === '[object Object]' } -function steps2tomltable (steps: IStep[]) { +function nodes2tomltable (nodes: IWorkflowNode[]): Record { const table: Record = {} const track: Record = {} - for (const step of steps) { - if (!(step.id in track)) { - track[step.id] = 0 + for (const node of nodes) { + if (!(node.id in track)) { + track[node.id] = 0 } - track[step.id]++ + track[node.id]++ const section = - track[step.id] > 1 ? `${step.id}.${track[step.id]}` : step.id - const stepParameters: Record = {} + track[node.id] > 1 ? `${node.id}.${track[node.id]}` : node.id + const nodeParameters: Record = {} // TODO make recursive so `items.input.items.hisd: nesting` is also applied - Object.entries(step.parameters).forEach(([k, v]) => { + Object.entries(node.parameters).forEach(([k, v]) => { if (Array.isArray(v) && v.length > 0 && isObject(v[0])) { // A value that is an array of objects will have each of its objects as a section - stepParameters[k] = v.map(d => Section(d)) + nodeParameters[k] = v.map(d => Section(d)) } else { - stepParameters[k] = v + nodeParameters[k] = v } }) - table[section] = Section(stepParameters as any) + table[section] = Section(nodeParameters as any) } return table } export function workflow2tomltext ( - steps: IStep[], + nodes: IWorkflowNode[], global: IParameters -) { +): string { const table = { - ...steps2tomltable(steps), + ...nodes2tomltable(nodes), ...global } const text = stringify(table as any, { @@ -46,22 +46,22 @@ export function workflow2tomltext ( return text } -export function parseWorkflow (workflow: string, globalKeys: Set) { +export function parseWorkflow (workflow: string, globalKeys: Set): IWorkflow { const table = parse(workflow, { bigint: false }) - const global: Record = {} - const steps: IStep[] = [] + const global: IParameters = {} + const nodes: IWorkflowNode[] = [] const sectionwithindex = /\.\d+$/ Object.entries(table).forEach(([k, v]) => { const section = k.replace(sectionwithindex, '') if (globalKeys.has(section)) { global[k] = v } else { - steps.push({ + nodes.push({ id: section, parameters: v as IParameters }) } }) - // TODO validate steps and global parameters against schemas in catalog - return { steps, global } + // TODO validate nodes and global parameters against schemas in catalog + return { nodes, global } } diff --git a/src/types.ts b/src/types.ts index 23cd8224..2ab465be 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,7 +1,7 @@ import { JSONSchema7 } from 'json-schema' import { UiSchema } from '@rjsf/core' -export interface INode { +export interface ICatalogNode { id: string label: string schema: JSONSchema7 @@ -23,13 +23,13 @@ export interface ICatalog { title: string global: IGlobal categories: ICategory[] - nodes: INode[] + nodes: ICatalogNode[] examples: Record } export type IParameters = Record -export interface IStep { +export interface IWorkflowNode { id: string parameters: IParameters } @@ -38,10 +38,10 @@ export type IFiles = Record export interface IWorkflow { global: IParameters - steps: IStep[] + nodes: IWorkflowNode[] } export interface IWorkflowSchema { global: IGlobal - nodes: INode[] + nodes: ICatalogNode[] } diff --git a/src/utils/array.ts b/src/utils/array.ts new file mode 100644 index 00000000..e3fd611f --- /dev/null +++ b/src/utils/array.ts @@ -0,0 +1,19 @@ +export function replaceItemAtIndex (arr: V[], index: number, newValue: V): V[] { + return [...arr.slice(0, index), newValue, ...arr.slice(index + 1)] +} + +export function removeItemAtIndex (arr: V[], index: number): V[] { + return [...arr.slice(0, index), ...arr.slice(index + 1)] +} + +export function moveItem (arr: V[], index: number, offset: number): V[] { + const item = arr[index] + const swappedIndex = index + offset + const swappedNode = arr[swappedIndex] + const newNodes = replaceItemAtIndex( + replaceItemAtIndex(arr, index, swappedNode), + swappedIndex, + item + ) + return newNodes +} diff --git a/src/searchreplace.test.ts b/src/utils/searchreplace.test.ts similarity index 100% rename from src/searchreplace.test.ts rename to src/utils/searchreplace.test.ts diff --git a/src/searchreplace.ts b/src/utils/searchreplace.ts similarity index 100% rename from src/searchreplace.ts rename to src/utils/searchreplace.ts diff --git a/src/validate.test.ts b/src/validate.test.ts index 21a100a2..74ef671d 100644 --- a/src/validate.test.ts +++ b/src/validate.test.ts @@ -1,7 +1,7 @@ import { beforeEach, describe, expect, it } from 'vitest' import { JSONSchema7 } from 'json-schema' import { validateCatalog, validateWorkflow } from './validate' -import { IWorkflowSchema } from './types' +import { ICatalog, IWorkflowSchema } from './types' describe('validateWorkflow()', () => { describe('given a workflow with only global parameters', () => { @@ -31,7 +31,7 @@ describe('validateWorkflow()', () => { global: { run_dir: 'run1' }, - steps: [] + nodes: [] } const errors = validateWorkflow(workflow, schemas) @@ -41,7 +41,7 @@ describe('validateWorkflow()', () => { it('should return run_dir required error when empty global parameters is given', () => { const workflow = { global: {}, - steps: [] + nodes: [] } const errors = validateWorkflow(workflow, schemas) @@ -99,7 +99,7 @@ describe('validateWorkflow()', () => { it('should return no errors when parameters are valid against node schema', () => { const workflow = { global: {}, - steps: [ + nodes: [ { id: 'mynode', parameters: { @@ -116,7 +116,7 @@ describe('validateWorkflow()', () => { it('should return autohis required error when empty node parameters is given', () => { const workflow = { global: {}, - steps: [ + nodes: [ { id: 'mynode', parameters: {} @@ -134,17 +134,17 @@ describe('validateWorkflow()', () => { missingProperty: 'autohis' }, schemaPath: '#/required', - workflowPath: 'step[0]' + workflowPath: 'node[0]' } ] expect(errors).toEqual(expected) }) - describe('given step without absent node', () => { + describe('given node without absent node', () => { it('should give a node schema not found error', () => { const workflow = { global: {}, - steps: [ + nodes: [ { id: 'myothernode', parameters: {} @@ -162,7 +162,7 @@ describe('validateWorkflow()', () => { node: 'myothernode' }, schemaPath: '', - workflowPath: 'step[0]' + workflowPath: 'node[0]' } ] expect(errors).toEqual(expected) @@ -174,11 +174,16 @@ describe('validateWorkflow()', () => { describe('validateCatalog', () => { describe('given catalog with JSON schemas', () => { it('should return zero errors', () => { - const catalog = { + const catalog: ICatalog = { title: 'Test catalog', global: { schema: { - type: 'string' + type: 'object', + properties: { + someprop: { + type: 'string' + } + } } }, nodes: [{ @@ -187,9 +192,16 @@ describe('validateCatalog', () => { description: 'My node description', category: 'My category', schema: { - type: 'string' + type: 'object', + properties: { + someprop: { + type: 'string' + } + } } - }] + }], + examples: {}, + categories: [] } const errors = validateCatalog(catalog) @@ -198,33 +210,24 @@ describe('validateCatalog', () => { }) }) - describe('given empty object', () => { - it('should return errors about all missing fields', () => { - const catalog = {} - - const errors = validateCatalog(catalog) - - const expected = [{ - message: 'catalog malformed or missing fields', - instancePath: '', - schemaPath: '', - keyword: '', - params: {} - }] - expect(errors).toEqual(expected) - }) - }) - describe('given global schema with wrong type', () => { it('should return errors about that schema', () => { - const catalog = { + const catalog: ICatalog = { title: 'Test catalog', global: { schema: { - type: 'nonexistingtype' + type: 'object', + properties: { + someprop: { + // @ts-expect-error + type: 'nonexistingtype' + } + } } }, - nodes: [] + nodes: [], + examples: {}, + categories: [] } const errors = validateCatalog(catalog) @@ -235,11 +238,16 @@ describe('validateCatalog', () => { describe('given node schema with wrong type', () => { it('should return errors about that schema', () => { - const catalog = { + const catalog: ICatalog = { title: 'Test catalog', global: { schema: { - type: 'string' + type: 'object', + properties: { + someprop: { + type: 'string' + } + } } }, nodes: [{ @@ -248,7 +256,13 @@ describe('validateCatalog', () => { description: 'My node description', category: 'My category', schema: { - type: 'nonexistingtype' + type: 'object', + properties: { + someprop: { + // @ts-expect-error + type: 'nonexistingtype' + } + } } }] } diff --git a/src/validate.ts b/src/validate.ts index b93629db..a7539931 100644 --- a/src/validate.ts +++ b/src/validate.ts @@ -2,7 +2,7 @@ import Ajv from 'ajv' import type { ErrorObject } from 'ajv' import addFormats from 'ajv-formats' import { JSONSchema7 } from 'json-schema' -import type { ICatalog, INode, IParameters, IStep, IWorkflow, IWorkflowSchema } from './types' +import type { ICatalogNode, IParameters, IWorkflowNode, IWorkflow, IWorkflowSchema, ICatalog } from './types' const ajv = new Ajv() addFormats(ajv) @@ -13,6 +13,13 @@ interface IvresseErrorObject extends ErrorObject, un export type Errors = IvresseErrorObject[] +export class ValidationError extends Error { + constructor (message: string, public errors: IvresseErrorObject[] = []) { + super(message) + Object.setPrototypeOf(this, new.target.prototype) + } +} + export function validateWorkflow (workflow: IWorkflow, schemas: IWorkflowSchema): Errors { const globalErrors = validateParameters( workflow.global, @@ -21,38 +28,38 @@ export function validateWorkflow (workflow: IWorkflow, schemas: IWorkflowSchema) globalErrors.forEach(e => { e.workflowPath = 'global' }) - const stepValidator = validateStep(schemas.nodes) - const stepsErrors = workflow.steps.map(stepValidator) + const nodeValidator = validateNode(schemas.nodes) + const nodesErrors = workflow.nodes.map(nodeValidator) // TODO validate files, // that all file paths in keys of files object are mentioned in parameters // and all filled `type:path` fields have entry in files object - return [...globalErrors, ...stepsErrors.flat(1)] + return [...globalErrors, ...nodesErrors.flat(1)] } -function validateStep (nodes: INode[]): (value: IStep, index: number, array: IStep[]) => Errors { - return (step, stepIndex) => { - const node = nodes.find((n) => n.id === step.id) - if (node != null) { - const stepErrors = validateParameters( - step.parameters, - node.schema +function validateNode (catalogNodes: ICatalogNode[]): (value: IWorkflowNode, index: number, array: IWorkflowNode[]) => Errors { + return (node, nodeIndex) => { + const catalogNode = catalogNodes.find((n) => n.id === node.id) + if (catalogNode != null) { + const nodeErrors = validateParameters( + node.parameters, + catalogNode.schema ) - stepErrors.forEach(e => { - e.workflowPath = `step[${stepIndex}]` + nodeErrors.forEach(e => { + e.workflowPath = `node[${nodeIndex}]` }) - return stepErrors + return nodeErrors } else { - // Node belonging to step could not be found + // Node belonging to node could not be found return [{ message: 'must have node name belonging to known nodes', params: { - node: step.id + node: node.id }, instancePath: '', schemaPath: '', keyword: 'schema', - workflowPath: `step[${stepIndex}]` + workflowPath: `node[${nodeIndex}]` }] } } @@ -66,31 +73,13 @@ function validateParameters (parameters: IParameters, schema: JSONSchema7): Erro } function validateSchema (schema: JSONSchema7): Errors { - if (!ajv.validateSchema(schema) && ajv.errors !== undefined && ajv.errors !== null) { + if (!(ajv.validateSchema(schema) as boolean) && ajv.errors !== undefined && ajv.errors !== null) { return ajv.errors } return [] } -function isCatalog (catalog: unknown): catalog is ICatalog { - return typeof catalog === 'object' && - catalog !== null && - 'global' in catalog && - 'nodes' in catalog - // TODO add more checks -} - -export function validateCatalog (catalog: unknown): Errors { - if (!isCatalog(catalog)) { - return [{ - message: 'catalog malformed or missing fields', - instancePath: '', - schemaPath: '', - keyword: '', - params: {} - }] - } - +export function validateCatalog (catalog: ICatalog): Errors { // Validate global schema const globalErrors = validateSchema(catalog.global.schema) globalErrors.forEach(e => { @@ -98,14 +87,14 @@ export function validateCatalog (catalog: unknown): Errors { }) // Validate node schemas - const stepsErrors = catalog.nodes.map((n, nodeIndex) => { - const stepErrors = validateSchema(n.schema) - stepErrors.forEach(e => { + const nodesErrors = catalog.nodes.map((n, nodeIndex) => { + const nodeErrors = validateSchema(n.schema) + nodeErrors.forEach(e => { e.workflowPath = `node[${nodeIndex}]` }) - return stepErrors + return nodeErrors }) // TODO validate non schema fields - return [...globalErrors, ...stepsErrors.flat(1)] + return [...globalErrors, ...nodesErrors.flat(1)] } diff --git a/src/vite-env.d.ts b/src/vite-env.d.ts deleted file mode 100644 index 11f02fe2..00000000 --- a/src/vite-env.d.ts +++ /dev/null @@ -1 +0,0 @@ -/// diff --git a/src/workflow.ts b/src/workflow.ts new file mode 100644 index 00000000..8e1f9037 --- /dev/null +++ b/src/workflow.ts @@ -0,0 +1,56 @@ +import { readArchive } from './archive' +import { globalParameterKeys } from './catalog' +import { parseWorkflow } from './toml' +import { ICatalog, IFiles, IParameters, IWorkflow, IWorkflowNode } from './types' +import { walk } from './utils/searchreplace' +import { validateWorkflow, ValidationError } from './validate' + +interface ILoadedworkflow extends IWorkflow { + files: IFiles +} + +export async function loadWorkflowArchive (archiveURL: string, catalog: ICatalog): Promise { + const { tomlstring, files } = await readArchive(archiveURL, catalog.nodes) + const globalKeys = globalParameterKeys(catalog.global) + const { nodes, global } = parseWorkflow(tomlstring, globalKeys) + const errors = validateWorkflow({ + global, + nodes + }, { + global: catalog.global, + nodes: catalog.nodes + }) + if (errors.length > 0) { + throw new ValidationError('Invalid workflow loaded', errors) + } + return { + global, + nodes, + files + } +} + +export function dropUnusedFiles (global: IParameters, nodes: IWorkflowNode[], files: IFiles): IFiles { + const newFiles = { ...files } + const parameters = [global, ...nodes.map(n => n.parameters)] + // Find out which files are mentioned in parameters + const filenamesInParameters: Set = new Set() + walk( + parameters, + (d: string) => d in newFiles, + (d: string) => { + filenamesInParameters.add(d) + return d + } + ) + // Remove any files not mentioned in parameters + const filenames = Object.keys(newFiles) + for (const filename of filenames) { + if (!filenamesInParameters.has(filename)) { + // TODO convert files to Map get rid of eslint-disable + /* eslint-disable @typescript-eslint/no-dynamic-delete */ + delete newFiles[filename] + } + } + return newFiles +}